1{ config, lib, pkgs, ... }:
2let
3 dashboard = pkgs.callPackage ./dashboard.nix { };
4 dashboardPort = "7072";
5 internalPort = 8448;
6 proxyToken = "/var/lib/studio/dashboard-proxy.token";
7 hostTools = lib.fileset.toSource {
8 root = ../.;
9 fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../tools/vms.py ../tools/dashboard-run.py ../tools/release.py ../service/keycloak/api.py ];
10 };
11 nativePkl = pkgs.callPackage ./pkl.nix { };
12in
13{
14 nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" ];
15
16 networking.firewall = {
17 enable = true;
18 allowedTCPPorts = [ 22 ];
19 interfaces.podman0.allowedTCPPorts = [ 443 internalPort 10428 15432 18428 ];
20 interfaces.podman0.allowedTCPPortRanges = [{ from = 20000; to = 32000; }];
21 interfaces.tailscale0.allowedTCPPorts = [ 80 443 445 8081 8082 ];
22 };
23
24 services.openssh.enable = true;
25 services.caddy = {
26 enable = true;
27 package = pkgs.caddy.withPlugins {
28 plugins = [ "github.com/caddyserver/replace-response@v0.0.0-20250618171559-80962887e4c6" ];
29 hash = "sha256-2tcvl9ZDpe7SAy3tZCPNwmw0KoIIakqggVaRtVAg9R0=";
30 };
31 extraConfig = "import /var/lib/caddy/routes.caddy";
32 virtualHosts = {
33 ":8082".extraConfig = "reverse_proxy 127.0.0.1:4646";
34 };
35 };
36 systemd.services.caddy.environment = {
37 OTEL_SERVICE_NAME = "studio-caddy";
38 OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = "http://127.0.0.1:10428/insert/opentelemetry/v1/traces";
39 OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
40 OTEL_TRACES_SAMPLER_ARG = "0.25";
41 };
42
43 boot.supportedFilesystems = [ "zfs" ];
44 virtualisation.podman.enable = true;
45 virtualisation.libvirtd.enable = true;
46 systemd.services.podman.environment.LOGGING = "--log-level=warn";
47
48 services.nomad = {
49 enable = true;
50 enableDocker = false;
51 dropPrivileges = false;
52 extraSettingsPlugins = [ pkgs.nomad-driver-podman ];
53 extraPackages = [ pkgs.cni-plugins ];
54 settings = {
55 datacenter = "clover";
56 bind_addr = "0.0.0.0";
57 addresses.http = "127.0.0.1";
58 advertise.http = "127.0.0.1";
59 server = {
60 enabled = true;
61 bootstrap_expect = 1;
62 };
63 acl.enabled = true;
64 client = {
65 enabled = true;
66 host_network.loopback.cidr = "127.0.0.1/8";
67 };
68 plugin.nomad-driver-podman.config.socket_path = "unix:///run/podman/podman.sock";
69 telemetry.collection_interval = "15s";
70 ui.enabled = true;
71 };
72 };
73
74 systemd.services.studio-log-shipper = {
75 wantedBy = [ "multi-user.target" ];
76 after = [ "nomad.service" ];
77 wants = [ "nomad.service" ];
78 unitConfig.ConditionPathExists = "/opt/studio/current/tools/log-shipper.py";
79 path = [ pkgs.systemd ];
80 serviceConfig = {
81 ExecStart = "${pkgs.python3}/bin/python3 /opt/studio/current/tools/log-shipper.py";
82 Restart = "always";
83 RestartSec = 5;
84 };
85 };
86
87 systemd.services.nomad = {
88 after = [ "podman.socket" ];
89 requires = [ "podman.socket" ];
90 };
91
92 systemd.services.studio-router = {
93 wantedBy = [ "multi-user.target" ];
94 wants = [ "nomad.service" "caddy.service" ];
95 after = [ "nomad.service" "caddy.service" ];
96 unitConfig.ConditionPathExists = "/var/lib/studio/router.token";
97 environment = {
98 STUDIO_DOMAIN = config.environment.variables.STUDIO_DOMAIN;
99 STUDIO_DASHBOARD_PORT = dashboardPort;
100 STUDIO_INTERNAL_PORT = toString internalPort;
101 STUDIO_PROXY_TOKEN_FILE = proxyToken;
102 };
103 serviceConfig = {
104 ExecStart = "${pkgs.python3}/bin/python3 ${../tools/router.py}";
105 Restart = "always";
106 RestartSec = 5;
107 };
108 };
109
110 users.groups.studio-dashboard = { };
111 users.users.studio-dashboard = {
112 isSystemUser = true;
113 group = "studio-dashboard";
114 };
115 systemd.services.studio-host = {
116 wantedBy = [ "multi-user.target" ];
117 wants = [ "podman.socket" ];
118 after = [ "podman.socket" ];
119 path = [ pkgs.zfs pkgs.util-linux pkgs.python3 pkgs.libvirt pkgs.qemu pkgs.podman pkgs.nomad pkgs.systemd pkgs.rsync pkgs.nixos-rebuild nativePkl ]
120 ++ lib.optional (builtins.elem "nvidia" config.services.xserver.videoDrivers) (lib.getBin config.hardware.nvidia.package);
121 environment = (lib.filterAttrs (name: _: lib.hasPrefix "STUDIO_" name) config.environment.variables) // {
122 STUDIO_POOL = lib.attrByPath [ "STUDIO_POOL" ] "studio-demo" config.environment.variables;
123 STUDIO_DASHBOARD_USER = "studio-dashboard";
124 STUDIO_VM_ACCEL = lib.attrByPath [ "STUDIO_VM_ACCEL" ] "kvm" config.environment.variables;
125 };
126 serviceConfig = {
127 ExecStart = "${pkgs.python3}/bin/python3 ${hostTools}/tools/dashboard-host.py";
128 Type = "notify";
129 Group = "studio-dashboard";
130 StateDirectory = "studio/host";
131 StateDirectoryMode = "0700";
132 RuntimeDirectory = "studio-host";
133 RuntimeDirectoryMode = "0750";
134 RuntimeDirectoryPreserve = "yes";
135 UMask = "0077";
136 ProtectSystem = "strict";
137 ReadWritePaths = [ "/srv/vm" ];
138 ProtectHome = true;
139 PrivateTmp = true;
140 NoNewPrivileges = true;
141 CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_READ_SEARCH" ];
142 MemoryMax = "256M";
143 TasksMax = 32;
144 DevicePolicy = "closed";
145 DeviceAllow = [ "/dev/zfs rw" ] ++ lib.optional (builtins.elem "nvidia" config.services.xserver.videoDrivers) "char-nvidia* rw";
146 RestrictAddressFamilies = [ "AF_UNIX" ];
147 IPAddressDeny = "any";
148 Restart = "always";
149 RestartSec = 2;
150 };
151 };
152 systemd.services.studio-dashboard = let
153 environment = (lib.filterAttrs (name: _: lib.hasPrefix "STUDIO_" name) config.environment.variables) // {
154 STUDIO_DATA_DIR = "/data";
155 STUDIO_PROXY_TOKEN_FILE = "/run/secrets/dashboard-proxy.token";
156 STUDIO_NOMAD_TOKEN_FILE = "/run/secrets/dashboard-nomad.token";
157 STUDIO_CA_BUNDLE = "/run/secrets/ca-bundle.crt";
158 STUDIO_INDEX_DIR = "/data/index";
159 STUDIO_INTERNAL_URL = "https://dashboard.internal.${config.environment.variables.STUDIO_DOMAIN}:${toString internalPort}";
160 STUDIO_FILES_URL = "https://file.${config.environment.variables.STUDIO_DOMAIN}";
161 STUDIO_PUBLIC_ORIGIN = "https://snowglobe.${config.environment.variables.STUDIO_DOMAIN}";
162 STUDIO_KEYCLOAK_URL = "https://auth.${config.environment.variables.STUDIO_DOMAIN}";
163 STUDIO_JELLYFIN_URL = "https://jelly.${config.environment.variables.STUDIO_DOMAIN}";
164 STUDIO_SHALE_URL = "https://shale.${config.environment.variables.STUDIO_DOMAIN}";
165 STUDIO_PUBLISHED_ROOT = "/srv/clover/Published";
166 STUDIO_DOMAIN = config.environment.variables.STUDIO_DOMAIN;
167 STUDIO_POOL = lib.attrByPath [ "STUDIO_POOL" ] "studio-demo" config.environment.variables;
168 STUDIO_NODE_NAME = config.networking.hostName;
169 STUDIO_MEDIA_READ_ONLY = lib.attrByPath [ "STUDIO_MEDIA_READ_ONLY" ] "false" config.environment.variables;
170 STUDIO_CLOVER_READ_ONLY = lib.attrByPath [ "STUDIO_CLOVER_READ_ONLY" ] "false" config.environment.variables;
171 };
172 in {
173 inherit environment;
174 wantedBy = [ "multi-user.target" ];
175 wants = [ "nomad.service" "studio-router.service" "studio-host.service" ];
176 after = [ "nomad.service" "studio-host.service" ];
177 unitConfig.ConditionPathExists = [ "/var/lib/studio/dashboard.token" "/var/lib/studio/ca-bundle.crt" "/opt/studio/current" ];
178 path = [ pkgs.podman pkgs.coreutils pkgs.curl pkgs.openssl pkgs.systemd ];
179 serviceConfig = {
180 Type = "notify";
181 NotifyAccess = "all";
182 ExecStart = pkgs.writeShellScript "studio-dashboard-start" ''
183 uid=$(id -u studio-dashboard)
184 gid=$(id -g studio-dashboard)
185 optional=()
186 if test -d /srv/prod/ytdl/data && test -d '/srv/clover/Documents/Config/Youtube Downloader'; then
187 optional+=(--volume=/srv/prod/ytdl/data:/srv/prod/ytdl/data:rw
188 --env=STUDIO_YT_STATE=/srv/prod/ytdl/data
189 '--env=STUDIO_YT_CONFIG=/srv/clover/Documents/Config/Youtube Downloader'
190 --env=STUDIO_YT_MEDIA=/srv/clover/Media)
191 fi
192 exec podman run --rm --replace --name=studio-dashboard --pull=never \
193 --sdnotify=conmon --cgroups=no-conmon --log-driver=journald \
194 --user="$uid:$gid" \
195 --group-add=${toString config.users.groups.chloe.gid} --network=podman \
196 --read-only --read-only-tmpfs=false --cap-drop=ALL --security-opt=no-new-privileges \
197 --memory=2g --cpus=4 --pids-limit=256 --cpu-shares=1024 \
198 --publish=127.0.0.1:${dashboardPort}:7072 \
199 --tmpfs=/tmp:rw,noexec,nosuid,nodev,size=256m,mode=1777 \
200 --tmpfs=/run:rw,noexec,nosuid,nodev,size=16m,mode=755 \
201 --env=HOME=/data --env=XDG_CACHE_HOME=/data/cache \
202 --volume=/var/lib/studio/dashboard:/data:rw \
203 --volume=/run/studio-host:/run/studio-host:ro \
204 --volume=${proxyToken}:/run/secrets/dashboard-proxy.token:ro \
205 --volume=/var/lib/studio/dashboard.token:/run/secrets/dashboard-nomad.token:ro \
206 --volume=/var/lib/studio/ca-bundle.crt:/run/secrets/ca-bundle.crt:ro \
207 --mount=type=bind,src=/srv/clover,dst=/srv/clover,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_CLOVER_READ_ONLY" \
208 --mount=type=bind,src=/srv/clover/Media,dst=/srv/clover/Media,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_MEDIA_READ_ONLY" \
209 ${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--env=${name}") (builtins.attrNames environment)} \
210 ${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--add-host=${name}.${config.environment.variables.STUDIO_DOMAIN}:host-gateway") [ "dashboard.internal" "auth" "keycloak" "jelly" "db" "shale" ]} \
211 "''${optional[@]}" ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}
212 '';
213 ExecStop = "${pkgs.podman}/bin/podman stop --ignore --time=8 studio-dashboard";
214 ExecStopPost = "${pkgs.podman}/bin/podman rm --ignore --force studio-dashboard";
215 CPUWeight = 1000;
216 KillMode = "mixed";
217 TimeoutStartSec = 180;
218 TimeoutStopSec = 30;
219 Restart = "always";
220 RestartSec = 5;
221 };
222 preStart = ''
223 umask 0077
224 if ! test -e ${proxyToken}; then
225 openssl rand -hex 32 > ${proxyToken}.pending
226 mv ${proxyToken}.pending ${proxyToken}
227 fi
228 chown root:studio-dashboard ${proxyToken} /var/lib/studio/dashboard.token
229 chmod 0640 ${proxyToken} /var/lib/studio/dashboard.token
230 chown -R studio-dashboard:studio-dashboard /var/lib/studio/dashboard
231 systemd-tmpfiles --create --prefix=/srv/prod/ytdl/data
232 if ! podman image exists ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}; then
233 podman load --quiet --input ${dashboard.image}
234 fi
235 '';
236 postStart = ''
237 for attempt in $(seq 1 30); do
238 if printf 'header = "Studio-Proxy-Token: %s"\n' "$(cat ${proxyToken})" | curl --config - --fail --silent http://127.0.0.1:${dashboardPort}/ >/dev/null; then
239 exit 0
240 fi
241 sleep 1
242 done
243 exit 1
244 '';
245 };
246
247 systemd.tmpfiles.rules = [
248 "d /srv/prod 0755 root root -"
249 "d /srv/staging 0755 root root -"
250 "d /srv/vm 0755 root root -"
251 "d /srv/logs 0770 clo chloe -"
252 "A+ /srv/prod/ytdl/data - - - - g:chloe:rwX,m::rwX,d:g:chloe:rwx,d:m::rwx"
253 "d /var/lib/studio 0700 root root -"
254 "d /var/lib/studio/dashboard 0700 studio-dashboard studio-dashboard -"
255 "d /var/lib/studio/routes 0700 root root -"
256 "d /var/lib/caddy/studio 0755 caddy caddy -"
257 "f /var/lib/caddy/routes.caddy 0640 caddy caddy -"
258 ];
259
260 environment.variables.STUDIO_NODE_NAME = config.networking.hostName;
261 environment.variables.STUDIO_DOMAIN = lib.mkDefault "studio.test";
262 environment.systemPackages = with pkgs; [ acl curl jq openssl python3 rsync zfs qemu ] ++ [ nativePkl ];
263}