← Back to benchmarkrender_splicebench.py
#!/usr/bin/env python3
"""Render the local Splicebench from completed, matching benchmark reports."""
import argparse
import hashlib
import html
import json
import shutil
import statistics
from pathlib import Path
ROOT = Path(__file__).resolve().parent
SITE = ROOT / 'splicebench-preview'
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('stores', type=Path)
parser.add_argument('deltas', type=Path)
parser.add_argument('--packed-git', type=Path, required=True)
parser.add_argument('--mercurial', type=Path, required=True)
parser.add_argument('--chg', type=Path, required=True)
parser.add_argument('--pijul', type=Path, required=True)
parser.add_argument('--startup', type=Path, required=True)
parser.add_argument('--nab', type=Path, required=True)
parser.add_argument('--pack-results', type=Path, required=True)
parser.add_argument('--wiki', type=Path)
parser.add_argument('--wiki-pijul', type=Path)
parser.add_argument('--wiki-manifest', type=Path)
parser.add_argument('--linux', type=Path)
parser.add_argument('--linux-pijul', type=Path)
parser.add_argument('--linux-manifest', type=Path)
args = parser.parse_args()
stores = json.loads(args.stores.read_text())
deltas = json.loads(args.deltas.read_text())
startup = json.loads(args.startup.read_text())
if startup['source_sha256'] != hashlib.sha256(args.deltas.read_bytes()).hexdigest():
raise ValueError('startup baseline does not match delta report')
git = json.loads(args.packed_git.read_text())
pijul = json.loads(args.pijul.read_text())
if not pijul['completed'] or pijul['seed'] != stores['seed']:
raise ValueError('Pijul report must be complete and have the same seed')
hg = json.loads(args.mercurial.read_text())
chg = json.loads(args.chg.read_text())
if not chg['completed'] or chg['seed'] != stores['seed']:
raise ValueError('chg report must be complete and have the same seed')
nab = json.loads(args.nab.read_text())
if not nab['completed'] or nab['seed'] != stores['seed']:
raise ValueError('NAB report must be complete and have the same seed')
if not hg['completed'] or hg['seed'] != stores['seed']:
raise ValueError('Mercurial report must be complete and have the same seed')
if not git['completed'] or git['seed'] != stores['seed']:
raise ValueError('packed Git report must be complete and have the same seed')
if not stores['completed'] or not deltas['completed'] or stores['seed'] != deltas['seed']:
raise ValueError('reports must be complete and have the same seed')
if any((args.wiki, args.wiki_pijul, args.wiki_manifest)) and not all((args.wiki, args.wiki_pijul, args.wiki_manifest)):
raise ValueError('all Wikipedia inputs must be supplied together')
wiki = json.loads(args.wiki.read_text()) if args.wiki else None
wiki_pijul = json.loads(args.wiki_pijul.read_text()) if args.wiki else None
if wiki and (not wiki['completed'] or not wiki_pijul['completed'] or wiki['manifest_sha256'] != wiki_pijul['manifest_sha256']):
raise ValueError('Wikipedia reports must be complete and use the same manifest')
if wiki and hashlib.sha256(args.wiki_manifest.read_bytes()).hexdigest() != wiki['manifest_sha256']:
raise ValueError('Wikipedia manifest hash mismatch')
if any((args.linux, args.linux_pijul, args.linux_manifest)) and not all((args.linux, args.linux_pijul, args.linux_manifest)):
raise ValueError('all Linux inputs must be supplied together')
linux = json.loads(args.linux.read_text()) if args.linux else None
linux_pijul = json.loads(args.linux_pijul.read_text()) if args.linux else None
if linux and (not linux['completed'] or not linux_pijul['completed'] or linux['manifest_sha256'] != linux_pijul['manifest_sha256']):
raise ValueError('Linux reports must be complete and use the same manifest')
if linux and hashlib.sha256(args.linux_manifest.read_bytes()).hexdigest() != linux['manifest_sha256']:
raise ValueError('Linux manifest hash mismatch')
data = {}
names = [('nab', 'NAB'), ('dnab', 'DNAB'), ('fossil', 'Fossil'), ('git', 'Git'), ('rcs', 'RCS'), ('hg', 'Mercurial'), ('chg', 'Mercurial (chg)'), ('pijul', 'Pijul')]
delta_names = [('xdelta3', 'xdelta3'), ('rdiff', 'librsync / rdiff'), ('bsdiff', 'bsdiff / bspatch'), ('zstd', 'zstd patch-from')]
for key, case in deltas['cases'].items():
matches = [c for c in stores['cases'] if f"{c['workload']}-{c['initial_bytes']}" == key]
if not matches or any(c['fixture_sha256'] != case['fixture_sha256'] for c in matches):
raise ValueError(f'fixture mismatch: {key}')
git_matches = [c for c in git['cases'] if f"{c['workload']}-{c['initial_bytes']}" == key]
if len(git_matches) != len(matches) or any(c['fixture_sha256'] != case['fixture_sha256'] for c in git_matches):
raise ValueError(f'packed Git fixture mismatch: {key}')
hg_matches = [c for c in hg['cases'] if f"{c['workload']}-{c['initial_bytes']}" == key]
if len(hg_matches) != len(matches) or any(c['fixture_sha256'] != case['fixture_sha256'] for c in hg_matches):
raise ValueError(f'Mercurial fixture mismatch: {key}')
chg_matches = [c for c in chg['cases'] if f"{c['workload']}-{c['initial_bytes']}" == key]
if len(chg_matches) != len(matches) or any(c['fixture_sha256'] != case['fixture_sha256'] for c in chg_matches):
raise ValueError(f'chg fixture mismatch: {key}')
pijul_matches = [c for c in pijul['cases'] if f"{c['workload']}-{c['initial_bytes']}" == key]
if len(pijul_matches) != len(matches) or any(c['fixture_sha256'] != case['fixture_sha256'] for c in pijul_matches):
raise ValueError(f'Pijul fixture mismatch: {key}')
nab_matches = [c for c in nab['cases'] if f"{c['workload']}-{c['initial_bytes']}" == key]
if len(nab_matches) != len(matches) or any(c['fixture_sha256'] != case['fixture_sha256'] for c in nab_matches):
raise ValueError(f'NAB fixture mismatch: {key}')
rows = []
for tool, label in names:
entries = [next(x for x in c['tools']['hg' if tool == 'chg' else tool]['checkpoints'] if x['versions'] == 100)
for c in (nab_matches if tool == 'nab' else git_matches if tool == 'git' else chg_matches if tool == 'chg' else hg_matches if tool == 'hg' else pijul_matches if tool == 'pijul' else matches)]
row = {'name': label, 'kind': 'store',
'patch': statistics.median(e['write_last_100']['median_ms'] for e in entries),
'rebuild': statistics.median(e['reads']['latest']['median_ms'] for e in entries),
'size': statistics.median(e['repository_logical_bytes'] for e in entries) / 1024}
rows.append(row)
for tool, label in delta_names:
entries = [t['tools'][tool] for t in case['trials']]
patch = statistics.median(e['encode']['median_ms'] for e in entries)
rebuild = statistics.median(e['decode']['sum_ms'] for e in entries)
size = statistics.median(e['base_plus_deltas_bytes'] for e in entries) / 1024
rows.append({'name': label, 'kind': 'delta', 'patch': patch, 'rebuild': rebuild, 'size': size})
adjusted = rebuild - startup['tools'][tool]['sum_ms']
if adjusted <= 0:
raise ValueError(f'{key}: startup baseline exceeds {label} rebuild time')
rows.append({'name': label + ' (startup-adjusted)', 'kind': 'delta',
'patch': patch, 'rebuild': adjusted, 'size': size})
data[key] = rows
for history_key, history, history_pijul in [('wikipedia-2004-2005', wiki, wiki_pijul), ('linux-maintainers', linux, linux_pijul)]:
if not history:
continue
trials = history['runs']
history_rows = []
for tool, label in names:
if tool == 'pijul':
cases = history_pijul['cases'][:trials]
if len(cases) != trials or any(c['tools']['pijul'].get('verified_indices') != [0, 250, 500, 750, 999] for c in cases):
raise ValueError(f'incomplete {history_key} Pijul trials')
entries = [c['tools']['pijul'] for c in cases]
patch = statistics.median(statistics.median(e['write_ms']) for e in entries)
rebuild = statistics.median(e['checkpoints'][0]['reads']['latest']['median_ms'] for e in entries)
size = statistics.median(e['checkpoints'][0]['repository_logical_bytes'] for e in entries) / 1024
else:
entries = [r for r in history['trials'] if r['tool'] == tool and r['completed']]
if len(entries) != trials or any(e['verified_revisions'] != 1000 for e in entries):
raise ValueError(f'incomplete {history_key} trials: {tool}')
patch = statistics.median(e['write']['median_ms'] for e in entries)
rebuild = statistics.median(e['newest_rebuild']['median_ms'] for e in entries)
size = statistics.median(e['repository_logical_bytes'] for e in entries) / 1024
history_rows.append({'name': label, 'kind': 'store', 'patch': patch, 'rebuild': rebuild, 'size': size})
for tool, label in delta_names:
entries = [r for r in history['trials'] if r['tool'] == tool and r['completed']]
if len(entries) != trials or any(e['verified_revisions'] != 1000 for e in entries):
raise ValueError(f'incomplete {history_key} trials: {tool}')
patch = statistics.median(e['encode']['median_ms'] for e in entries)
rebuild = statistics.median(e['newest_rebuild_ms'] for e in entries)
size = statistics.median(e['repository_logical_bytes'] for e in entries) / 1024
history_rows.append({'name': label, 'kind': 'delta', 'patch': patch, 'rebuild': rebuild, 'size': size})
adjusted = statistics.median(e['newest_rebuild_ms'] - e['decoder_startup']['sum_ms'] for e in entries)
if adjusted <= 0:
raise ValueError(f'{history_key}: startup estimate exceeds {label} measured rebuild')
history_rows.append({'name': label + ' (startup-adjusted)', 'kind': 'delta',
'patch': patch, 'rebuild': adjusted, 'size': size})
data[history_key] = history_rows
(SITE / 'data.js').write_text('window.BENCHMARK_DATA = ' + json.dumps(data, indent=2) + ';\n')
sources = [(args.stores, 'store-results.json'), (args.deltas, 'delta-results.json'),
(args.packed_git, 'packed-git-results.json'), (args.mercurial, 'mercurial-results.json'),
(args.nab, 'nab-results.json'), (args.chg, 'chg-results.json'),
(args.pack_results, 'nab-pack-recover-results.json'), (args.pijul, 'pijul-results.json'),
(args.startup, 'delta-startup-results.json')]
if wiki:
sources += [(args.wiki, 'wikipedia-results.json'), (args.wiki_pijul, 'wikipedia-pijul-results.json'),
(args.wiki_manifest, 'wikipedia-manifest.json')]
if linux:
sources += [(args.linux, 'linux-results.json'), (args.linux_pijul, 'linux-pijul-results.json'),
(args.linux_manifest, 'linux-manifest.json')]
for source, dest in sources:
shutil.copyfile(source, SITE / dest)
for name in ['compare_stores.py', 'compare_deltas.py', 'benchmark_fixtures.py', 'measure_delta_startup.py', 'bench_pack_recover.py', 'benchmark_pijul.py', 'benchmark_wikipedia.py', 'benchmark_linux_maintainers.py', 'benchmark_pijul_wikipedia.py', 'render_splicebench.py']:
code = html.escape((ROOT / name).read_text())
page = ('<!doctype html><html lang="en"><head><meta charset="utf-8">'
'<meta name="viewport" content="width=device-width,initial-scale=1">'
f'<title>{html.escape(name)}</title><style>body{{background:#f7f2e5;color:#292b25;'
'font:14px system-ui,sans-serif;margin:32px}a{color:#375f4a}'
'pre{white-space:pre;overflow:auto;font:13px/1.5 ui-monospace,monospace}'
'</style></head><body><a href="index.html">← Back to benchmark</a>'
f'<h1>{html.escape(name)}</h1><pre>{code}</pre></body></html>\n')
(SITE / f'{name}.html').write_text(page)
print(f'Rendered {len(data)} cases from seed {stores["seed"]} into {SITE}')
if __name__ == '__main__':
main()