1#!/usr/bin/env python3
2"""Builds, signs and publishes Snowbound for each desktop platform; see tools/RELEASE.md."""
3import argparse
4import base64
5from datetime import datetime
6import hashlib
7import json
8import os
9from pathlib import Path
10import re
11import shutil
12import subprocess
13import sys
14import tempfile
15import time
16import zipfile
17from zoneinfo import ZoneInfo
18
19ROOT = Path(__file__).resolve().parents[1]
20PUBLISHED = Path('/Volumes/clover/Documents/Public/Snowbound')
21URL = 'https://file.paperclover.net/shr/snowbound/'
22KEY = Path.home() / '.config/snowbound/release-key'
23# Its public half, which the app checks updates against.
24MINISIGN = ROOT / 'minisign.pub'
25ZONE = ZoneInfo('America/Los_Angeles')
26PLATFORMS = ['macos-aarch64', 'macos-x86_64', 'macos-10.6', 'linux-x86_64', 'linux-aarch64',
27 'windows-x86_64', 'windows-aarch64']
28# Windows 7 to 11 on x86_64 (nightly's tier-3 win7 target), and Windows 11 on Arm.
29WINDOWS = {'x86_64': 'x86_64-win7-windows-gnu', 'aarch64': 'aarch64-pc-windows-gnullvm'}
30# Clover's Developer ID Application certificate, by its SHA-1 hash: its name is the account
31# holder's legal name, which nothing here prints or stores.
32IDENTITY = 'BA308AA3591299E053E8824CEF1651F686F8908E'
33# The App Store Connect API key that notarizes it: {"key": P8 PATH, "key_id": ID, "issuer": ID}.
34NOTARY = Path('~/.config/snowbound/notary.json').expanduser()
35# The first published build's commit, where changes start when no build was published before.
36FIRST = '354f001dec3d731a4d1a6fac0a25d9e28550d781'
37KINDS = {'feat': 'feature', 'fix': 'fix'}
38
39
40def derive(release, commits):
41 """The version of the commit made at `release`, given when each commit leading to it was
42 made, itself included: its day in Los Angeles, and how many of those were made that day."""
43 day = release.astimezone(ZONE).date()
44 return day.isoformat(), sum(1 for made in commits if made.astimezone(ZONE).date() == day)
45
46
47def name(version):
48 return f'{version[0]}-r{version[1]}'
49
50
51def folder(version):
52 """The build's top-level folder."""
53 return f'{version[0]}.r{version[1]}'
54
55
56def parse(text):
57 date, revision = text.split('-r')
58 return date, int(revision)
59
60
61def newest(latest, archives, version):
62 """`latest.json`'s map with each of `archives`' platforms moved to `version` where that is
63 newer than what it names."""
64 return {**latest, **{platform: name(version) for platform in archives
65 if platform not in latest or parse(latest[platform]) < version}}
66
67
68def builds(published):
69 """The versions of the builds `published` holds, oldest first."""
70 return sorted(parse(entry.name.replace('.r', '-r')) for entry in published.iterdir()
71 if re.fullmatch(r'\d{4}-\d{2}-\d{2}\.r\d+', entry.name))
72
73
74def jj(*args):
75 return subprocess.check_output(['jj', *args], cwd=ROOT, text=True)
76
77
78def clean(dry_run, moment):
79 """Refuses to go on, or in a dry run warns, where the working copy isn't `main`."""
80 changed = jj('diff', '--from', 'main', '--to', '@', '--summary').strip()
81 if changed and not dry_run:
82 sys.exit(f'The working copy differs from main {moment}:\n{changed}')
83 if changed:
84 print(f'Dry run: the working copy differs from main {moment}.', file=sys.stderr)
85
86
87def run(command, **kwargs):
88 print('+', ' '.join(map(str, command)), flush=True)
89 subprocess.run(command, cwd=ROOT, check=True, **kwargs)
90
91
92def history():
93 """Every commit on `main` by id: its parents' ids, when it was made, and its message."""
94 template = ('commit_id ++ "\\x1f" ++ parents.map(|parent| parent.commit_id()).join(" ") ++ "\\x1f" ++ '
95 'committer.timestamp().utc().format("%Y-%m-%dT%H:%M:%S+00:00") ++ "\\x1f" ++ description ++ "\\x1e"')
96 commits = {}
97 for record in jj('log', '--no-graph', '-r', '::main', '-T', template).split('\x1e')[:-1]:
98 commit, parents, made, description = record.split('\x1f')
99 commits[commit] = (parents.split(), datetime.fromisoformat(made), description)
100 return commits
101
102
103def ancestors(commits, commit):
104 """`commit` and every commit before it."""
105 found, stack = set(), [commit]
106 while stack:
107 commit = stack.pop()
108 if commit not in found:
109 found.add(commit)
110 stack.extend(commits[commit][0])
111 return found
112
113
114def version_of(commits, commit):
115 return derive(commits[commit][1], [commits[each][1] for each in ancestors(commits, commit)])
116
117
118def entries(description):
119 """What a commit brings, as (kind, title): one entry of its prefix's kind, or one per item where
120 its body has a top-level bulleted list, each of the prefix's kind."""
121 subject, _, body = description.strip().partition('\n')
122 prefix = re.match(r'(\w+)(\([^)]*\))?!?:\s*', subject)
123 kind = KINDS.get(prefix[1].lower(), 'other') if prefix else 'other'
124 items, open_item = [], False
125 for line in body.splitlines():
126 if line.startswith(('- ', '* ')):
127 items.append(line[2:].strip())
128 open_item = True
129 elif open_item and line[:1].isspace() and line.strip():
130 items[-1] += ' ' + line.strip()
131 else:
132 open_item = False
133 titles = [title.rstrip('.') for title in items or [subject[prefix.end():] if prefix else subject]]
134 # Capitalized as a sentence, except a word like macOS or iCloud.
135 return [(kind, title if re.match(r'\S+[A-Z]', title) else title[:1].upper() + title[1:])
136 for title in titles if title]
137
138
139def changes(commits, commit, since):
140 """Every entry the commits after `since` up to `commit` bring, oldest first, each with the
141 version of the commit that brought it."""
142 versions = {each: version_of(commits, each)
143 for each in ancestors(commits, commit) - ancestors(commits, since)}
144 return [{'version': name(versions[each]), 'kind': kind, 'title': title}
145 for each in sorted(versions, key=versions.get) for kind, title in entries(commits[each][2])]
146
147
148def sign(files):
149 output = subprocess.check_output(
150 ['cargo', 'run', '--quiet', '--release', '-p', 'snowbound', '--example', 'release_sign', '--', KEY, *files],
151 cwd=ROOT, text=True)
152 return output.split()
153
154
155def minisign(files):
156 """Writes FILE.minisig beside each of `files` as `minisign -S` would with the release key:
157 a signature of the file's BLAKE2b-512, then one of that and the trusted comment."""
158 key_id = base64.b64decode(MINISIGN.read_text().splitlines()[1])[2:10]
159 comments = [f'timestamp:{int(time.time())}\tfile:{file.name}\thashed' for file in files]
160 with tempfile.TemporaryDirectory() as scratch:
161 def signed(messages):
162 paths = [Path(scratch) / str(index) for index in range(len(messages))]
163 for path, message in zip(paths, messages):
164 path.write_bytes(message)
165 return [bytes.fromhex(signature) for signature in sign(paths)]
166 signatures = signed([hashlib.blake2b(file.read_bytes()).digest() for file in files])
167 global_signatures = signed([signature + comment.encode() for signature, comment in zip(signatures, comments)])
168 for file, signature, comment, global_signature in zip(files, signatures, comments, global_signatures):
169 Path(f'{file}.minisig').write_text(
170 'untrusted comment: signature from the Snowbound release key\n'
171 f'{base64.b64encode(b"ED" + key_id + signature).decode()}\n'
172 f'trusted comment: {comment}\n{base64.b64encode(global_signature).decode()}\n')
173
174
175def split_debug(executable, debug):
176 """Moves `executable`'s debug info to `debug`, which its debug link then names."""
177 sysroot = subprocess.check_output(['rustc', '--print', 'sysroot'], text=True).strip()
178 host = re.search(r'^host: (\S+)$', subprocess.check_output(['rustc', '-vV'], text=True), re.M)[1]
179 objcopy = Path(sysroot) / 'lib/rustlib' / host / 'bin/rust-objcopy'
180 run([objcopy, '--only-keep-debug', executable, debug])
181 run([objcopy, '--strip-debug', f'--add-gnu-debuglink={debug}', executable])
182
183
184def zip_bundle(bundle, archive):
185 run(['ditto', '-c', '-k', '--norsrc', '--noextattr', '--noqtn', '--noacl', '--keepParent', bundle, archive])
186
187
188def notary():
189 """notarytool's credential arguments from NOTARY, if they sign in."""
190 if not NOTARY.exists():
191 return None
192 key = json.loads(NOTARY.read_text())
193 arguments = ['--key', str(Path(key['key']).expanduser()), '--key-id', key['key_id'], '--issuer', key['issuer']]
194 signs_in = subprocess.run(['xcrun', 'notarytool', 'history', *arguments], capture_output=True).returncode == 0
195 return arguments if signs_in else None
196
197
198def build_mac(platform, folder, developer_id, notarize, symbols):
199 """The zipped app, which build_macos.py signs; 10.6's stays unsigned, as it predates Developer ID.
200 Its zipped dSYM goes to `symbols`."""
201 bundle = folder / 'Snowbound.app'
202 if platform == 'macos-10.6':
203 signing = ['--snow-leopard']
204 elif developer_id:
205 signing = ['--sign', 'developer-id', '--sign-identity', IDENTITY]
206 else:
207 signing = ['--sign', 'ad-hoc']
208 if platform != 'macos-10.6':
209 signing += ['--arch', platform.removeprefix('macos-')]
210 dsym = folder / 'Snowbound.dSYM'
211 run([sys.executable, ROOT / 'tools/canvas/build_macos.py', '--release', '--output', bundle, '--dsym', dsym, *signing])
212 zip_bundle(dsym, symbols)
213 archive = folder / 'archive.zip'
214 if notarize and platform != 'macos-10.6':
215 zip_bundle(bundle, archive)
216 run(['xcrun', 'notarytool', 'submit', archive, *notarize, '--wait'])
217 run(['xcrun', 'stapler', 'staple', bundle])
218 archive.unlink()
219 zip_bundle(bundle, archive)
220 return archive
221
222
223def build_linux(architectures):
224 """The executables, each all of Snowbound for its architecture."""
225 run(['sh', ROOT / 'crates/snowbound/linux/package.sh', *architectures])
226 return {f'linux-{arch}': ROOT / f'target/{arch}-unknown-linux-gnu/release/snowbound' for arch in architectures}
227
228
229def build_windows(architectures):
230 """The executables, each all of Snowbound for its architecture, one running on every
231 Windows it supports."""
232 built = {}
233 for arch in architectures:
234 run(['sh', ROOT / 'platform/windows/cargo.sh', arch, 'build', '--release', '-p', 'snowbound'])
235 built[f'windows-{arch}'] = ROOT / f'target/windows/{WINDOWS[arch]}/release/snowbound.exe'
236 return built
237
238
239def main():
240 parser = argparse.ArgumentParser(description=__doc__)
241 parser.add_argument('--dry-run', action='store_true',
242 help='Publish into a new temporary folder instead, even from a changed working copy')
243 parser.add_argument('--platforms', nargs='+', choices=PLATFORMS, default=PLATFORMS)
244 parser.add_argument('--ad-hoc', action='store_true',
245 help='Sign the macOS app ad hoc instead of with Developer ID, unnotarized')
246 args = parser.parse_args()
247 developer_id = not args.ad_hoc and any(platform in ('macos-aarch64', 'macos-x86_64') for platform in args.platforms)
248 identities = subprocess.run(['security', 'find-identity', '-v', '-p', 'codesigning'],
249 capture_output=True, text=True).stdout
250 if developer_id and IDENTITY not in identities:
251 sys.exit(f'The keychain has no signing identity {IDENTITY}; release with --ad-hoc, or add it.')
252 notarize = notary() if developer_id else None
253 if developer_id and not notarize:
254 print(f'Not notarizing: no API key in {NOTARY} that signs in (see tools/RELEASE.md).',
255 file=sys.stderr)
256
257 commit = jj('log', '--no-graph', '-r', 'main', '-T', 'commit_id').strip()
258 clean(args.dry_run, 'to release it')
259 commits = history()
260 version = version_of(commits, commit)
261 print(f'Snowbound build {version[0]} revision {version[1]}, commit {commit}', flush=True)
262
263 published = Path(tempfile.mkdtemp(prefix='snowbound-release-')) if args.dry_run else PUBLISHED
264 if not published.is_dir():
265 sys.exit(f'{published} is not mounted.')
266 target = published / folder(version)
267 if target.exists():
268 build = json.loads((target / 'build.json').read_text())
269 if build['commit'] != commit:
270 sys.exit(f'{target} holds commit {build["commit"]}, not {commit}.')
271 print(f'{target} is already published.')
272 else:
273 run([sys.executable, ROOT / 'tools/ci.py', '--rev', commit])
274 clean(args.dry_run, 'after the checks')
275 stage = ROOT / 'target/release-stage' / name(version)
276 shutil.rmtree(stage, ignore_errors=True)
277 stage.mkdir(parents=True)
278 # The app reads its version from this as it compiles.
279 os.environ['SNOWBOUND_BUILD'] = name(version)
280 # For the symbol files; the executables shed it.
281 os.environ['CARGO_PROFILE_RELEASE_DEBUG'] = 'line-tables-only'
282 built = {}
283 symbols = []
284 for platform in args.platforms:
285 if platform.startswith('macos'):
286 work = stage / platform
287 work.mkdir()
288 symbols.append(stage / f'Snowbound-{name(version)}-{platform}.dSYM.zip')
289 built[platform] = build_mac(platform, work, developer_id, notarize, symbols[-1])
290 linux = [platform.removeprefix('linux-') for platform in args.platforms if platform.startswith('linux')]
291 if linux:
292 built |= build_linux(linux)
293 windows = [platform.removeprefix('windows-') for platform in args.platforms if platform.startswith('windows')]
294 if windows:
295 built |= build_windows(windows)
296 clean(args.dry_run, 'after the build')
297 files = {}
298 for platform, source in built.items():
299 prefix = 'Snowbound' if platform.startswith('macos') else 'snowbound'
300 files[platform] = stage / f'{prefix}-{name(version)}-{platform}{source.suffix}'
301 shutil.copy2(source, files[platform])
302 if not platform.startswith('macos'):
303 debug = stage / f'{prefix}-{name(version)}-{platform}.debug'
304 split_debug(files[platform], debug)
305 symbols.append(debug.with_name(f'{debug.name}.zip'))
306 with zipfile.ZipFile(symbols[-1], 'w', zipfile.ZIP_DEFLATED) as archive:
307 archive.write(debug, debug.name)
308 signatures = sign(files.values())
309 # A dry run's changes too start after the newest build the share holds.
310 before = [each for each in builds(PUBLISHED) if each < version] if PUBLISHED.is_dir() else []
311 since = json.loads((PUBLISHED / folder(before[-1]) / 'build.json').read_text())['commit'] if before else FIRST
312 build = {
313 'version': name(version),
314 'commit': commit,
315 'published': datetime.now(ZONE).isoformat(timespec='seconds'),
316 'changes': changes(commits, commit, since),
317 'archives': {platform: {
318 'file': file.name,
319 'size': file.stat().st_size,
320 'sha256': hashlib.sha256(file.read_bytes()).hexdigest(),
321 # Older apps check it; newer ones trust the sha256 build.json.sig vouches for.
322 'signature': signature,
323 } for (platform, file), signature in zip(files.items(), signatures)},
324 }
325 (stage / 'build.json').write_text(json.dumps(build, indent=2) + '\n')
326 (stage / 'build.json.sig').write_text(sign([stage / 'build.json'])[0] + '\n')
327 downloads = [*files.values(), *symbols, stage / 'build.json']
328 minisign(downloads)
329 partial = target.with_name(f'.{target.name}.partial')
330 shutil.rmtree(partial, ignore_errors=True)
331 partial.mkdir()
332 for file in [*downloads, *(Path(f'{file}.minisig') for file in downloads), stage / 'build.json.sig']:
333 # copy() keeps the Linux executables executable for anyone running them off the share.
334 shutil.copy(file, partial / file.name)
335 partial.rename(target)
336 shutil.rmtree(stage)
337 print(f'Published {target}')
338
339 history_file = published / 'history.json'
340 partial = history_file.with_name('.history.json.partial')
341 partial.write_text(json.dumps([name(each) for each in builds(published)], indent=2) + '\n')
342 os.replace(partial, history_file)
343 latest_file = published / 'latest.json'
344 latest = json.loads(latest_file.read_text()) if latest_file.exists() else {}
345 build = json.loads((target / 'build.json').read_text())
346 moved = newest(latest, build['archives'], version)
347 if moved != latest:
348 partial = latest_file.with_name('.latest.json.partial')
349 partial.write_text(json.dumps(moved, indent=2, sort_keys=True) + '\n')
350 os.replace(partial, latest_file)
351 print(f'{latest_file}: {json.dumps(moved, sort_keys=True)}')
352 # Stable names for the readme's download links, always the newest build of each platform.
353 downloads = published / 'latest'
354 downloads.mkdir(exist_ok=True)
355 for platform, newest_name in moved.items():
356 if newest_name != name(version):
357 continue
358 file = build['archives'][platform]['file']
359 for published_name in (file, f'{file}.minisig'):
360 stable = downloads / published_name.replace(f'-{name(version)}', '')
361 partial = stable.with_name(f'.{stable.name}.partial')
362 shutil.copy(target / published_name, partial)
363 os.replace(partial, stable)
364 if not args.dry_run:
365 print(f'{URL}{folder(version)}/')
366
367
368if __name__ == '__main__':
369 main()