1#!/usr/bin/env python3
2import argparse
3from datetime import datetime, timezone
4import fcntl
5import hashlib
6import json
7import os
8from pathlib import Path
9import re
10import subprocess
11import time
12
13
14ROOT = Path("/opt/studio")
15RELEASES = ROOT / "releases"
16STATE = Path("/var/lib/studio")
17HISTORY = STATE / "deployments.json"
18SOURCES = ("config", "service", "tools", "nixos", "dashboard", "flake.nix", "flake.lock", "readme.md")
19RELEASE_ID = re.compile(r"[0-9a-f]{16}\Z")
20STAGE_ID = re.compile(r"[a-z][a-z0-9-]*\Z")
21
22
23def excluded_services(root):
24 path = root / "config/excluded-services.json"
25 return set(json.loads(path.read_text())) if path.exists() else set()
26
27
28def files(root):
29 excluded = excluded_services(root)
30 for name in SOURCES:
31 source = root / name
32 if name == "service" and source.is_dir():
33 paths = []
34 for directory, directories, filenames in os.walk(source):
35 directories[:] = [entry for entry in directories if entry not in excluded]
36 paths.extend(Path(directory) / entry for entry in [*directories, *filenames]
37 if not (entry.endswith(".pkl") and entry[:-4] in excluded))
38 else:
39 paths = source.rglob("*") if source.is_dir() else [source]
40 for path in sorted(paths):
41 relative = path.relative_to(root)
42 if relative.parts[:2] in (
43 ("dashboard", "node_modules"), ("dashboard", "dist"),
44 ("dashboard", ".cache"), ("dashboard", "data"), ("dashboard", "target"),
45 ):
46 continue
47 if any(part in {".DS_Store", "__pycache__", ".identities.lock", "identities.pending"} or part.startswith("._") or part.endswith(".pyc") for part in relative.parts):
48 continue
49 if path.is_symlink():
50 raise ValueError(f"unsupported release file: {relative}")
51 if path.is_dir():
52 continue
53 if not path.is_file():
54 raise ValueError(f"unsupported release file: {relative}")
55 yield path, relative
56
57
58def _hash_contents(digest, path):
59 with path.open("rb") as source:
60 while chunk := source.read(65536):
61 digest.update(chunk)
62
63
64def tree_digest(root):
65 digest = hashlib.sha256()
66 for path, relative in files(root):
67 digest.update(str(relative).encode() + b"\0")
68 _hash_contents(digest, path)
69 return digest.hexdigest()
70
71
72def host_digest(root):
73 digest = hashlib.sha256()
74 paths = sorted(path for path, relative in files(root)
75 if relative.parts[0] in {"nixos", "dashboard", "config", "service"}
76 or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py"})
77 for path in paths:
78 digest.update(str(path.relative_to(root)).encode())
79 _hash_contents(digest, path)
80 return digest.hexdigest()
81
82
83def check_release(release, legacy=False):
84 if not RELEASE_ID.fullmatch(release):
85 raise ValueError("invalid release ID")
86 path = RELEASES / release
87 if not path.is_dir():
88 raise ValueError(f"release is missing: {release}")
89 manifest = path / ".studio-release.json"
90 if not manifest.exists():
91 if not legacy:
92 raise ValueError(f"release has no manifest: {release}")
93 else:
94 details = json.loads(manifest.read_text())
95 if details.get("id") != release or details.get("digest") != tree_digest(path) or details.get("version", 1) not in (1, 2):
96 raise ValueError(f"release contents changed: {release}")
97 return path
98
99
100def current_release(link="current"):
101 current = ROOT / link
102 if not current.is_symlink():
103 return None
104 target = current.resolve()
105 if target.parent != RELEASES or not RELEASE_ID.fullmatch(target.name):
106 raise ValueError(f"{link} points outside releases: {target}")
107 return target.name
108
109
110def main_release(version=None):
111 version = version or current_release("main")
112 if version is None:
113 return None
114 if not RELEASE_ID.fullmatch(version):
115 raise ValueError("invalid release ID")
116 details = json.loads((RELEASES / version / ".studio-release.json").read_text())
117 revision = details.get("main")
118 if (not isinstance(revision, dict) or not isinstance(revision.get("commit"), str)
119 or not re.fullmatch(r"[0-9a-f]{40}", revision["commit"])
120 or not isinstance(revision.get("description"), str) or not revision["description"].strip()):
121 raise ValueError("This release did not come from a described main commit. Publish main first.")
122 return {"release": version, "commit": revision["commit"], "description": revision["description"]}
123
124
125def history():
126 return json.loads(HISTORY.read_text()) if HISTORY.exists() else []
127
128
129def save_history(entries):
130 pending = HISTORY.with_suffix(".pending")
131 pending.write_text(json.dumps(entries, indent=2) + "\n")
132 pending.replace(HISTORY)
133
134
135def jobs(path):
136 rendered = subprocess.run(
137 ["python3", str(path / "tools/studio.py"), "render"],
138 check=True, capture_output=True, text=True,
139 ).stdout
140 result = set(re.findall(r'^job "([a-z][a-z0-9-]*)" \{$', rendered, re.MULTILINE))
141 if not result:
142 raise ValueError(f"no Nomad jobs rendered by {path}")
143 return result
144
145
146def activate(release, legacy=False, initial=False):
147 path = check_release(release, legacy)
148 script = path / "tools/studio.py"
149 managed = STATE / "managed-jobs.json"
150 current = current_release()
151 previous = set(json.loads(managed.read_text())) if managed.exists() else (
152 jobs(RELEASES / current) if current and not initial else set()
153 )
154 digest = host_digest(path)
155 hostname = os.uname().nodename.split(".", 1)[0]
156 configuration = "vm" if hostname == "clover-demo" else hostname
157 recorded = ROOT / "host.digest"
158 old_digest = recorded.read_text().strip() if recorded.exists() else None
159 if old_digest != digest:
160 subprocess.run(["nixos-rebuild", "dry-build", "--flake", f"path:{path}#{configuration}"], check=True)
161 subprocess.run(["python3", str(script), "preflight"], check=True)
162 backup = None
163 if current and not initial:
164 backup_script = ROOT / "data.py"
165 if not backup_script.is_file():
166 backup_script = path / "tools/data.py"
167 result = subprocess.run(
168 ["python3", str(backup_script), "backup", current, release],
169 check=True, capture_output=True, text=True,
170 )
171 print(result.stdout.strip(), flush=True)
172 backup = result.stdout.split("backup=", 1)[1].split()[0]
173 if old_digest != digest:
174 if current is None:
175 (ROOT / "current").symlink_to(path)
176 subprocess.run(["nixos-rebuild", "switch", "--flake", f"path:{path}#{configuration}"], check=True)
177 next_link = ROOT / ("next-" + release)
178 next_link.unlink(missing_ok=True)
179 next_link.symlink_to(path)
180 next_link.replace(ROOT / "current")
181 if old_digest != digest:
182 recorded.write_text(digest + "\n")
183 for _ in range(60):
184 response = subprocess.run(
185 ["curl", "--fail", "--silent", "--max-time", "2", "http://127.0.0.1:4646/v1/status/leader"],
186 capture_output=True, text=True,
187 )
188 if response.returncode == 0 and json.loads(response.stdout):
189 break
190 time.sleep(2)
191 else:
192 raise RuntimeError("Nomad API did not become ready")
193 subprocess.run(["systemctl", "restart", "studio-router.service"], check=True)
194 if (path / "nixos/dashboard.nix").exists():
195 subprocess.run(["systemctl", "start", "studio-dashboard.service"], check=True)
196 subprocess.run(["systemctl", "is-active", "--quiet", "studio-dashboard.service"], check=True)
197 subprocess.run(["python3", str(script), "pool"], check=True)
198 subprocess.run(["python3", str(script), "deploy"], check=True)
199 desired = jobs(path)
200 for name in sorted(previous - desired):
201 subprocess.run(
202 ["nomad", "job", "stop", "-purge", "-yes", name], check=True,
203 env={**os.environ, "NOMAD_TOKEN": (STATE / "nomad.token").read_text().strip()},
204 )
205 pending = managed.with_suffix(".pending")
206 pending.write_text(json.dumps(sorted(desired)) + "\n")
207 pending.replace(managed)
208 if (path / "tools/log-shipper.py").exists():
209 subprocess.run(["systemctl", "restart", "studio-log-shipper.service"], check=True)
210 manifest = path / ".studio-release.json"
211 if not legacy and json.loads(manifest.read_text()).get("version", 1) >= 2:
212 subprocess.run(["python3", str(script), "check"], check=True)
213 return backup
214
215
216def main():
217 parser = argparse.ArgumentParser(description="Deploy main or roll back a home server release")
218 parser.add_argument("mode", choices=["publish", "deploy", "rollback", "history", "bootstrap", "verify"])
219 parser.add_argument("target", nargs="?")
220 args = parser.parse_args()
221 STATE.mkdir(parents=True, exist_ok=True)
222 if args.mode == "verify":
223 if not args.target:
224 parser.error("verify requires a release ID")
225 check_release(args.target)
226 return
227 if args.mode == "history":
228 if args.target:
229 parser.error("history takes no target")
230 current = current_release()
231 entries = history()
232 for index, entry in enumerate(entries[-12:], start=max(0, len(entries) - 12)):
233 marker = "*" if index == len(entries) - 1 and entry["release"] == current else " "
234 when = datetime.fromtimestamp(entry["time"], timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
235 print(f"{marker} {entry['release']} {when} {entry['source']}")
236 if current and (not entries or entries[-1]["release"] != current):
237 print(f"* {current} deployment incomplete")
238 return
239 with (STATE / "release.lock").open("w") as lock:
240 fcntl.flock(lock, fcntl.LOCK_EX)
241 if args.mode == "publish":
242 if not args.target:
243 parser.error("publish requires a main release ID")
244 path = check_release(args.target)
245 main_release(args.target)
246 pending = ROOT / "main.pending"
247 pending.unlink(missing_ok=True)
248 pending.symlink_to(path)
249 pending.replace(ROOT / "main")
250 print(f"main={args.target}")
251 return
252 entries = history()
253 current = current_release()
254 if current and not entries and args.mode != "bootstrap":
255 entries.append({"release": current, "source": "previous", "time": int(time.time()), "legacy": not (RELEASES / current / ".studio-release.json").exists()})
256 save_history(entries)
257 if args.mode == "deploy":
258 candidate = main_release()
259 if not candidate or args.target != candidate["release"]:
260 raise ValueError("main changed or is unavailable. Publish main and retry deployment.")
261 release = candidate["release"]
262 source = "main"
263 legacy = False
264 elif args.mode == "bootstrap":
265 if not args.target or not RELEASE_ID.fullmatch(args.target) or entries:
266 parser.error("bootstrap requires a release ID and no successful deployment")
267 release, source, legacy = args.target, "bootstrap", False
268 main_release(release)
269 else:
270 if args.target:
271 match = next((item for item in reversed(entries) if item["release"] == args.target), None)
272 if not match:
273 parser.error("rollback target is not in production history")
274 else:
275 match = next((item for item in reversed(entries) if item["release"] != current), None)
276 if not match:
277 parser.error("no earlier production release")
278 release, source, legacy = match["release"], "rollback", match.get("legacy", False)
279 if release == current and entries and entries[-1]["release"] == current:
280 print(f"already running {release}")
281 return
282 backup = activate(release, legacy, args.mode == "bootstrap")
283 entries.append({"release": release, "source": source, "time": int(time.time()), "legacy": legacy, "backup": backup})
284 save_history(entries)
285 print(f"production={release} previous={current or ''}")
286
287
288if __name__ == "__main__":
289 main()