| 1 | #!/usr/bin/env node |
| 2 | // Benchmark committed repository copies. Source checkouts are never changed. |
| 3 | import { execFile } from "node:child_process" ; |
| 4 | import { appendFile, mkdtemp, mkdir, readdir, rm, writeFile, rename } from "node:fs/promises" ; |
| 5 | import { tmpdir } from "node:os" ; |
| 6 | import { resolve, join, basename } from "node:path" ; |
| 7 | import { promisify } from "node:util" ; |
| 8 | import { randomUUID } from "node:crypto" ; |
| 9 | import WebSocket from "ws" ; |
| 10 | import { finalizeEvent, generateSecretKey, getPublicKey } from "nostr-tools/pure" ; |
| 11 | import { npubEncode } from "nostr-tools/nip19" ; |
| 12 | import { startGitSigningProxy } from "./git-signing-proxy.mjs" ; |
| 13 | |
| 14 | const exec = promisify(execFile); |
| 15 | const args = process.argv.slice(2); |
| 16 | const option = (key, fallback) => args.includes(key) ? args[args.indexOf(key) + 1] : fallback; |
| 17 | const relay = option("--relay" , "ws://127.0.0.1:17447" ); |
| 18 | const reposDir = option("--repos-dir" ); |
| 19 | const outDir = resolve(option("--out" , "artifacts/git-performance/run" )); |
| 20 | const baseline = option("--baseline" , "" ).replace(/\/$/, "" ); |
| 21 | const only = option("--only" , "" ).split("," ).filter(Boolean); |
| 22 | const privateRepos = args.includes("--private" ); |
| 23 | const privateRunID = randomUUID().slice(0, 8); |
| 24 | const privateOwnerSecretHex = option("--private-owner-secret" , process.env.TINY_BENCHMARK_PRIVATE_OWNER_SECRET || "fc1d06a0fd5e622dcf448d0b3c2fccc891a5ecf74546a7675460d92441fecfdd" ); |
| 25 | if (privateRepos && !/^[0-9a-fA-F]{64}$/.test(privateOwnerSecretHex)) throw Error("--private-owner-secret must be 64 hex characters" ); |
| 26 | const privateOwnerSecret = privateRepos ? Uint8Array.from(Buffer.from(privateOwnerSecretHex, "hex" )) : null; |
| 27 | const repeats = Math.max(1, Number(option("--repeats" , "3" ))); |
| 28 | if (!reposDir) throw Error("--repos-dir is required" ); |
| 29 | const httpURL = relay.replace(/^ws/, "http" ).replace(/\/$/, "" ); |
| 30 | const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); |
| 31 | const report = {schema: 3, started: new Date().toISOString(), relay, baseline, repeats, privateRepos, results: [], failures: []}; |
| 32 | await mkdir(outDir, {recursive: true}); |
| 33 | const progressPath = join(outDir, "progress.jsonl" ); |
| 34 | await writeFile(progressPath, "" ); |
| 35 | const progress = async row => { |
| 36 | const value = {at: new Date().toISOString(), ...row}; |
| 37 | console.log(JSON.stringify(value)); |
| 38 | await appendFile(progressPath, JSON.stringify(value) + "\n" ); |
| 39 | }; |
| 40 | const save = async () => { |
| 41 | const path = join(outDir, "git-performance.json" ); |
| 42 | await writeFile(path + ".tmp" , JSON.stringify(report, null, 2) + "\n" ); |
| 43 | await rename(path + ".tmp" , path); |
| 44 | }; |
| 45 | const git = async (cwd, ...args) => (await exec("git" , ["-C" , cwd, ...args], { |
| 46 | encoding: "utf8" , maxBuffer: 16 * 1024 * 1024, |
| 47 | env: {...process.env, GIT_TERMINAL_PROMPT: "0" , GIT_AUTHOR_NAME: "tiny benchmark" , GIT_AUTHOR_EMAIL: "benchmark@example.invalid" , GIT_COMMITTER_NAME: "tiny benchmark" , GIT_COMMITTER_EMAIL: "benchmark@example.invalid" }, |
| 48 | })).stdout.trim(); |
| 49 | |
| 50 | class Client { |
| 51 | constructor(url) { |
| 52 | this.url = url; |
| 53 | this.challenge = new Promise(resolve => { this.setChallenge = resolve; }); |
| 54 | this.ws = new WebSocket(url); |
| 55 | this.waiters = new Map(); |
| 56 | this.connected = new Promise((resolve, reject) => { |
| 57 | this.ws.once("open" , resolve); |
| 58 | this.ws.once("error" , reject); |
| 59 | }); |
| 60 | this.ws.on("message" , raw => { |
| 61 | let message; |
| 62 | try { message = JSON.parse(raw); } catch { return; } |
| 63 | if (message[0] === "AUTH" ) { this.setChallenge(message[1]); return; } |
| 64 | const waiter = this.waiters.get(message[1]); |
| 65 | if (!waiter) return; |
| 66 | if (message[0] === "EVENT" ) waiter.events.push(message[2]); |
| 67 | else if (message[0] === "EOSE" ) waiter.finish(null, waiter.events); |
| 68 | else if (message[0] === "OK" ) waiter.finish(message[2] ? null : Error(message[3]), message); |
| 69 | else if (message[0] === "CLOSED" ) waiter.finish(Error(message[2])); |
| 70 | }); |
| 71 | this.ws.on("error" , error => { for (const waiter of [...this.waiters.values()]) waiter.finish(error); }); |
| 72 | this.ws.on("close" , () => { for (const waiter of [...this.waiters.values()]) waiter.finish(Error("websocket closed" )); }); |
| 73 | } |
| 74 | async request(message, key) { |
| 75 | await this.connected; |
| 76 | if (this.ws.readyState !== WebSocket.OPEN) throw Error("websocket is not open" ); |
| 77 | return new Promise((resolve, reject) => { |
| 78 | const timer = setTimeout(() => finish(Error("Nostr request timed out after 30 seconds" )), 30000); |
| 79 | const finish = (error, value) => { |
| 80 | clearTimeout(timer); |
| 81 | this.waiters.delete(key); |
| 82 | if (message[0] === "REQ" && this.ws.readyState === WebSocket.OPEN) this.ws.send(JSON.stringify(["CLOSE" , key])); |
| 83 | if (error) reject(error); else resolve(value); |
| 84 | }; |
| 85 | this.waiters.set(key, {events: [], finish}); |
| 86 | this.ws.send(JSON.stringify(message)); |
| 87 | }); |
| 88 | } |
| 89 | publish(event) { return this.request(["EVENT" , event], event.id); } |
| 90 | async authenticate(secret) { |
| 91 | const challenge = await this.challenge; |
| 92 | const proof = finalizeEvent({kind: 22242, created_at: Math.floor(Date.now()/1000), content: "" , |
| 93 | tags: [["relay" , this.url], ["challenge" , challenge]]}, secret); |
| 94 | return this.request(["AUTH" , proof], proof.id); |
| 95 | } |
| 96 | query(filter) { |
| 97 | const id = Math.random().toString(36).slice(2); |
| 98 | return this.request(["REQ" , id, filter], id); |
| 99 | } |
| 100 | close() { this.ws.close(); } |
| 101 | } |
| 102 | |
| 103 | const control = new Client(relay); |
| 104 | const probes = new Client(relay); |
| 105 | await Promise.all([control.connected, probes.connected]); |
| 106 | const probeSecret = generateSecretKey(); |
| 107 | let sequence = 0; |
| 108 | async function probeSample(label) { |
| 109 | const row = {at: new Date().toISOString(), phase: label}; |
| 110 | const event = finalizeEvent({kind: 1, created_at: Math.floor(Date.now()/1000), tags: [], content: `tiny benchmark ${sequence++}` }, probeSecret); |
| 111 | for (const [operation, run] of [["publish" , () => probes.publish(event)], ["query" , async () => { |
| 112 | const found = await probes.query({ids: [event.id], limit: 1}); |
| 113 | if (found.length !== 1 || found[0].id !== event.id) throw Error("probe event was not returned by query" ); |
| 114 | }]]) { |
| 115 | const start = performance.now(); |
| 116 | try { await run(); row[operation + "Ms" ] = performance.now() - start; } |
| 117 | catch (error) { row[operation + "Error" ] = error.message; row[operation + "Ms" ] = performance.now() - start; } |
| 118 | } |
| 119 | return row; |
| 120 | } |
| 121 | |
| 122 | const refsFor = async directory => (await git(directory, "for-each-ref" , "--format=%(refname) %(objectname)" , "refs/heads" , "refs/tags" )) |
| 123 | .split("\n" ).filter(Boolean).map(line => line.split(" " )); |
| 124 | async function verifyRemote(url, refs, head) { |
| 125 | const output = (await exec("git" , ["ls-remote" , url], {encoding: "utf8" , maxBuffer: 16*1024*1024})).stdout; |
| 126 | const actual = new Map(output.trim().split("\n" ).filter(Boolean).map(line => { const [oid, ref] = line.split(/\s+/); return [ref, oid]; })); |
| 127 | for (const [ref, oid] of refs) if (actual.get(ref) !== oid) throw Error(`remote ref mismatch: ${ref}` ); |
| 128 | const expectedHead = refs.find(([ref]) => ref === head)?.[1]; |
| 129 | if (!expectedHead || actual.get("HEAD" ) !== expectedHead) throw Error(`remote HEAD does not resolve to ${head}` ); |
| 130 | } |
| 131 | |
| 132 | async function benchmarkFixture(fixture, index) { |
| 133 | const name = basename(fixture); |
| 134 | const result = {fixture: name, phases: [], probes: [], status: "running" }; |
| 135 | report.results.push(result); |
| 136 | const temporaries = []; |
| 137 | const temp = async () => { const path = await mkdtemp(join(tmpdir(), "tiny-git-benchmark-" )); temporaries.push(path); return path; }; |
| 138 | let active = "idle" , stopProbes = false, loop, signingProxy; |
| 139 | const phase = async (name, fn) => { |
| 140 | active = name; |
| 141 | const started = new Date().toISOString(), start = performance.now(); |
| 142 | await progress({fixture: result.fixture, phase: name, event: "start" }); |
| 143 | let error; |
| 144 | try { return await fn(); } catch (failure) { error = failure; throw failure; } |
| 145 | finally { |
| 146 | const row = {name, started, finished: new Date().toISOString(), ms: performance.now() - start, ok: !error}; |
| 147 | if (error) row.error = error.message.slice(0, 2000); |
| 148 | result.phases.push(row); |
| 149 | await progress({fixture: result.fixture, phase: name, event: "end" , ms: row.ms, ok: row.ok}); |
| 150 | await save(); |
| 151 | active = "between_phases" ; |
| 152 | } |
| 153 | }; |
| 154 | try { |
| 155 | await progress({fixture: name, event: "fixture_start" }); |
| 156 | for (let i=0; i<50; i++) { result.probes.push(await probeSample("idle" )); await sleep(20); } |
| 157 | loop = (async () => { while (!stopProbes) { result.probes.push(await probeSample(active)); await sleep(50); } })(); |
| 158 | const head = await git(fixture, "symbolic-ref" , "HEAD" ); |
| 159 | const refs = await refsFor(fixture); |
| 160 | result.head = head; |
| 161 | result.refs = refs.length; |
| 162 | const sk = privateRepos ? privateOwnerSecret : generateSecretKey(), pubkey = getPublicKey(sk), id = `${name.replace(/\.git$/, "")}-${index}${privateRepos ? "-" + privateRunID : ""}` ; |
| 163 | const repositoryPath = `/${npubEncode(pubkey)}/${encodeURIComponent(id)}.git` ; |
| 164 | const canonicalURL = httpURL + repositoryPath; |
| 165 | let url = canonicalURL; |
| 166 | result.url = canonicalURL; |
| 167 | if (privateRepos) { |
| 168 | await control.authenticate(sk); |
| 169 | signingProxy = await startGitSigningProxy(httpURL, sk, {grasp08: true}); |
| 170 | url = signingProxy.url + repositoryPath; |
| 171 | result.transport = "local GRASP-08 signing proxy; request body spooled on client" ; |
| 172 | } |
| 173 | const announceAt = Math.floor(Date.now()/1000); |
| 174 | const announcement = finalizeEvent({kind: 30617, created_at: announceAt, content: "" , tags: [["d" , id], ["clone" , canonicalURL], ["relays" , relay], ["maintainers" , pubkey], ...(privateRepos ? [["private" , "true" ]] : [])]}, sk); |
| 175 | const makeState = (refs, created_at) => finalizeEvent({kind: 30618, created_at, content: "" , tags: [["d" , id], ["HEAD" , `ref: ${head}` ], ...refs]}, sk); |
| 176 | await phase("announcement_ack" , () => control.publish(announcement)); |
| 177 | const state = makeState(refs, announceAt+1); |
| 178 | await phase("state_ack" , () => control.publish(state)); |
| 179 | // --shared only uses immutable fixture objects; updates live in this copy. |
| 180 | const source = await temp(); |
| 181 | await git(source, "clone" , "--bare" , "--shared" , fixture, "." ); |
| 182 | const refspecs = ["refs/heads/*:refs/heads/*" , "refs/tags/*:refs/tags/*" ]; |
| 183 | const waitVisible = async event => { |
| 184 | const until = Date.now()+30000; |
| 185 | while (Date.now()<until) { |
| 186 | if ((await control.query({ids:[event.id],limit:1})).length) return; |
| 187 | await sleep(25); |
| 188 | } |
| 189 | throw Error("signed Git state did not become visible after transfer" ); |
| 190 | }; |
| 191 | await phase("initial_push" , () => git(source, "push" , url, ...refspecs)); |
| 192 | await phase("promotion_wait" , () => waitVisible(state)); |
| 193 | const baseURL = baseline ? `${baseline}/${encodeURIComponent(name.endsWith(".git") ? name : name+".git")}` : "" ; |
| 194 | if (baseURL) await phase("baseline_initial_push" , () => git(source, "push" , baseURL, ...refspecs)); |
| 195 | const clones = new Map(); |
| 196 | for (const [prefix, target] of [["" , url], ["baseline_" , baseURL]]) { |
| 197 | if (!target) continue; |
| 198 | for (let i=0; i<repeats; i++) { |
| 199 | const destination = await temp(); |
| 200 | await phase(`${prefix}fresh_clone_${i+1}` , () => git(destination, "clone" , target, "." )); |
| 201 | const actualHead = await git(destination, "rev-parse" , "HEAD" ); |
| 202 | if (actualHead !== refs.find(([ref])=>ref===head)[1]) throw Error(`${prefix}clone checked out the wrong HEAD` ); |
| 203 | if (i===0) clones.set(prefix, destination); |
| 204 | else await rm(destination, {recursive: true, force: true}); |
| 205 | } |
| 206 | await phase(prefix+"verify" , async () => { await verifyRemote(target, refs, head); await git(clones.get(prefix), "fsck" , "--full" ); }); |
| 207 | for (let i=0; i<repeats; i++) await phase(`${prefix}noop_fetch_${i+1}` , () => git(clones.get(prefix), "fetch" , "origin" )); |
| 208 | } |
| 209 | const parent = await git(source, "rev-parse" , head); |
| 210 | const tree = await git(source, "rev-parse" , `${head}^{tree}` ); |
| 211 | const next = await git(source, "commit-tree" , tree, "-p" , parent, "-m" , "tiny performance incremental commit" ); |
| 212 | await git(source, "update-ref" , head, next); |
| 213 | const nextRefs = refs.map(([ref, oid]) => [ref, ref===head ? next : oid]); |
| 214 | const nextState = makeState(nextRefs, Math.max(announceAt+2, Math.floor(Date.now()/1000))); |
| 215 | await phase("incremental_state_ack" , () => control.publish(nextState)); |
| 216 | await phase("incremental_push" , () => git(source, "push" , url, `${head}:${head}` )); |
| 217 | await phase("incremental_promotion_wait" , () => waitVisible(nextState)); |
| 218 | if (baseURL) await phase("baseline_incremental_push" , () => git(source, "push" , baseURL, `${head}:${head}` )); |
| 219 | for (const [prefix, target] of [["" ,url], ["baseline_" ,baseURL]]) { |
| 220 | if (!target) continue; |
| 221 | await phase(prefix+"incremental_fetch" , () => git(clones.get(prefix), "fetch" , "origin" , head)); |
| 222 | if (await git(clones.get(prefix), "rev-parse" , "FETCH_HEAD" ) !== next) throw Error("incremental fetch returned wrong commit" ); |
| 223 | for (const count of [2,4]) { |
| 224 | const destinations = await Promise.all(Array.from({length:count},()=>temp())); |
| 225 | await phase(`${prefix}concurrent_clone_${count}` , async () => { |
| 226 | const started = performance.now(); |
| 227 | const runs = await Promise.allSettled(destinations.map(async destination => { |
| 228 | await git(destination,"clone" ,target,"." ); |
| 229 | return performance.now()-started; |
| 230 | })); |
| 231 | const failure = runs.find(run=>run.status==="rejected" ); |
| 232 | if (failure) throw failure.reason; |
| 233 | result.phases.push({name:`${prefix}concurrent_clone_${count}_individual` , durationsMs:runs.map(run=>run.value), ok:true}); |
| 234 | }); |
| 235 | await Promise.all(destinations.map(path=>rm(path,{recursive:true,force:true}))); |
| 236 | } |
| 237 | } |
| 238 | result.status="passed" ; |
| 239 | } catch (error) { |
| 240 | result.status="failed" ; |
| 241 | result.error=error.message.slice(0,4000); |
| 242 | report.failures.push({fixture:name,error:result.error}); |
| 243 | } finally { |
| 244 | stopProbes=true; |
| 245 | if (loop) await loop; |
| 246 | if (signingProxy) await signingProxy.close(); |
| 247 | await Promise.all(temporaries.map(path=>rm(path,{recursive:true,force:true}))); |
| 248 | await progress({fixture:name,event:"fixture_end" ,status:result.status}); |
| 249 | await save(); |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | try { |
| 254 | const fixtures = (await readdir(reposDir,{withFileTypes:true})).filter(entry=>entry.isDirectory() && (!only.length || only.includes(entry.name.replace(/\.git$/,"" )))) |
| 255 | .map(entry=>join(resolve(reposDir),entry.name)); |
| 256 | fixtures.sort((a,b)=>basename(a).localeCompare(basename(b))); |
| 257 | for (let i=0;i<fixtures.length;i++) await benchmarkFixture(fixtures[i],i); |
| 258 | } finally { |
| 259 | control.close(); probes.close(); |
| 260 | report.finished=new Date().toISOString(); |
| 261 | await save(); |
| 262 | } |
| 263 | console.log(JSON.stringify({results:report.results.map(result=>({fixture:result.fixture,status:result.status})),failures:report.failures.length})); |
| 264 | process.exitCode=report.failures.length?1:0; |