| 1 | #!/usr/bin/env python3 |
| 2 | "" "Prepare immutable bare Git benchmark fixtures and inventory corpus metadata. |
| 3 | |
| 4 | Usage: prepare-git-performance.py --source-directory ~/Developer --out /tmp/tinyrelay-corpus [--repos a,b] |
| 5 | Every Git repository directly under the source directory is selected unless |
| 6 | --repos names a subset. The output directory must not already exist. Each selected repository is cloned |
| 7 | with --bare --no-local; the source working trees are never modified. |
| 8 | "" " |
| 9 | from __future__ import annotations |
| 10 | import argparse, json, os, shutil, subprocess, tempfile |
| 11 | from pathlib import Path |
| 12 | |
| 13 | def is_repository(path: Path) -> bool: |
| 14 | return (path / ".git" ).exists() or (path / "HEAD" ).exists() |
| 15 | |
| 16 | def select(source: Path, names: list[str] | None) -> list[str]: |
| 17 | if names: return names |
| 18 | return sorted(p.name for p in source.iterdir() if p.is_dir() and is_repository(p)) |
| 19 | |
| 20 | def run(*args: str, cwd: Path | None = None, input: str | None = None) -> str: |
| 21 | p = subprocess.run(args, cwd=cwd, input=input, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| 22 | if p.returncode: |
| 23 | raise RuntimeError(f"{' '.join(args)}: {p.stderr.strip()}" ) |
| 24 | return p.stdout.strip() |
| 25 | |
| 26 | def inventory(repo: Path, name: str) -> dict: |
| 27 | head_branch = run("git" , "--git-dir" , str(repo), "symbolic-ref" , "HEAD" ) |
| 28 | head = run("git" , "--git-dir" , str(repo), "rev-parse" , head_branch) |
| 29 | refs_lines = run("git" , "--git-dir" , str(repo), "for-each-ref" , "--format=%(refname) %(objectname)" ).splitlines() |
| 30 | tags = [line for line in refs_lines if line.startswith("refs/tags/" )] |
| 31 | head_commits = int(run("git" , "--git-dir" , str(repo), "rev-list" , "--count" , head_branch)) |
| 32 | all_commits = int(run("git" , "--git-dir" , str(repo), "rev-list" , "--all" , "--count" )) |
| 33 | stats = {} |
| 34 | for line in run("git" , "--git-dir" , str(repo), "count-objects" , "-v" ).splitlines(): |
| 35 | k, _, v = line.partition(" " ) |
| 36 | if k and v.isdigit(): stats[k] = int(v) |
| 37 | object_ids = run("git" , "--git-dir" , str(repo), "rev-list" , "--objects" , "--all" ).splitlines() |
| 38 | ids = [line.split()[0] for line in object_ids] |
| 39 | objects = {} |
| 40 | object_bytes = 0 |
| 41 | checked = run("git" , "--git-dir" , str(repo), "cat-file" , "--batch-all-objects" , "--batch-check=%(objectname) %(objecttype) %(objectsize)" ) |
| 42 | sizes = {} |
| 43 | for line in checked.splitlines(): |
| 44 | fields = line.split() |
| 45 | if len(fields) != 3: continue |
| 46 | oid, kind, size = fields[0], fields[1], int(fields[2]) |
| 47 | objects[kind] = objects.get(kind, 0) + 1 |
| 48 | object_bytes += size |
| 49 | sizes[oid] = (kind, size) |
| 50 | blob_sizes = [] |
| 51 | for line in object_ids: |
| 52 | fields = line.split(maxsplit=1) |
| 53 | if len(fields) == 2 and fields[0] in sizes and sizes[fields[0]][0] == "blob" : |
| 54 | blob_sizes.append((sizes[fields[0]][1], fields[1])) |
| 55 | blob_sizes.sort(reverse=True) |
| 56 | return { |
| 57 | "name" : name, "head" : head, "head_branch" : head_branch, |
| 58 | "head_commits" : head_commits, "all_commits" : all_commits, |
| 59 | "refs" : len(refs_lines), "tags" : len(tags), |
| 60 | "objects" : objects, |
| 61 | "object_bytes" : object_bytes, |
| 62 | "pack_bytes" : stats.get("size-pack" , 0) * 1024, |
| 63 | "largest_blobs" : [{"bytes" : size, "path" : path} for size, path in blob_sizes[:10]], |
| 64 | "blobs_over_1MiB" : sum(size > 1024 * 1024 for size, _ in blob_sizes), |
| 65 | "blobs_over_10MiB" : sum(size > 10 * 1024 * 1024 for size, _ in blob_sizes), |
| 66 | } |
| 67 | |
| 68 | def prepare(source: Path, out: Path, names: list[str] | None = None) -> list[dict]: |
| 69 | if out.exists(): raise FileExistsError(f"refusing existing destination: {out}" ) |
| 70 | selected = select(source, names) |
| 71 | if not selected: raise FileNotFoundError(f"no repositories under {source}" ) |
| 72 | out.mkdir(parents=True) |
| 73 | rows = [] |
| 74 | try: |
| 75 | for name in selected: |
| 76 | src = source / name |
| 77 | if not is_repository(src): raise FileNotFoundError(f"missing source repository: {src}" ) |
| 78 | dest = out / (name + ".git" ) |
| 79 | run("git" , "clone" , "--bare" , "--no-local" , str(src), str(dest)) |
| 80 | rows.append(inventory(dest, name)) |
| 81 | (out / "corpus.json" ).write_text(json.dumps(rows, indent=2) + "\n" , encoding="utf-8" ) |
| 82 | return rows |
| 83 | except Exception: |
| 84 | shutil.rmtree(out, ignore_errors=True) |
| 85 | raise |
| 86 | |
| 87 | def self_test() -> None: |
| 88 | with tempfile.TemporaryDirectory(prefix="prepare-git-performance-" ) as d: |
| 89 | root, src, out = Path(d), Path(d) / "src" , Path(d) / "out" |
| 90 | src.mkdir(); run("git" , "init" , "-b" , "trunk" , str(src)); run("git" , "-C" , str(src), "config" , "user.email" , "test@example.invalid" ); run("git" , "-C" , str(src), "config" , "user.name" , "test" ) |
| 91 | (src / "tracked" ).write_text("tracked\n" ); run("git" , "-C" , str(src), "add" , "tracked" ); run("git" , "-C" , str(src), "commit" , "-m" , "one" ); run("git" , "-C" , str(src), "branch" , "feature" ); run("git" , "-C" , str(src), "tag" , "v1" ) |
| 92 | (src / "uncommitted" ).write_text("must not be copied\n" ) |
| 93 | # Exercise prepare() itself without copying a real corpus: three |
| 94 | # fixture names all point at the same tiny synthetic repository. |
| 95 | names = ("alpha" , "beta" , "gamma" ) |
| 96 | for name in names: os.symlink(src, root / name, target_is_directory=True) |
| 97 | rows = prepare(root, out, list(names)) |
| 98 | assert len(rows) == len(names) |
| 99 | row = rows[0] |
| 100 | assert row["head_branch" ] == "refs/heads/trunk" and row["head" ] == run("git" , "-C" , str(src), "rev-parse" , "HEAD" ) |
| 101 | assert row["tags" ] == 1 and row["all_commits" ] == 1 |
| 102 | assert "uncommitted" not in run("git" , "--git-dir" , str(out / "alpha.git" ), "ls-tree" , "-r" , "--name-only" , "HEAD" ) |
| 103 | |
| 104 | if __name__ == "__main__" : |
| 105 | parser = argparse.ArgumentParser(description=__doc__) |
| 106 | parser.add_argument("--source-directory" , type=Path, default=Path.home() / "Developer" ) |
| 107 | parser.add_argument("--out" , type=Path) |
| 108 | parser.add_argument("--repos" , help="comma-separated repository directory names to select" ) |
| 109 | parser.add_argument("--self-test" , action="store_true" ) |
| 110 | args = parser.parse_args() |
| 111 | if args.self_test: self_test() |
| 112 | else: |
| 113 | if args.out is None: parser.error("--out is required unless --self-test is used" ) |
| 114 | rows = prepare(args.source_directory.expanduser(), args.out.expanduser(), args.repos.split("," ) if args.repos else None) |
| 115 | print(json.dumps(rows, indent=2)) |