Unify dataset/tooling scripts into a single dwarf Typer CLI

Replace the five separately-hyphenated uv entry points (steps-to-parquet,
steps-to-parquet-parallel, migrate-geant-steps, bump-dataset-version,
create-root-files) plus the unregistered hparam_scan.py with one `dwarf`
command exposing convert/migrate/bump-gen/bump-schema/status/
update-manifest/create-manifest/make-root/hparam-scan as subcommands.

Each scripts/*.py module now only holds argparse-free business logic;
scripts/dwarf.py wires it up with Typer, matching giant/cli.py's style.
`dwarf convert` merges the old serial/parallel conversion scripts behind
a --jobs flag (default 1: sequential with plain -o; >1: dataset-layout
fan-out via subprocess).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 09:52:37 +02:00
parent 7b37b284f8
commit d5853d5a75
13 changed files with 662 additions and 505 deletions
+14 -25
View File
@@ -1,4 +1,3 @@
#!/usr/bin/env python3
"""Hyperparameter scan over dropout x n_blocks x hidden_dim.
Runs `giant train` sequentially (this machine has a single GPU) for every
@@ -6,13 +5,9 @@ combination, plus one extra run at the default architecture with a higher
learning rate. Runs are shuffled so the parameter space gets coarse coverage
early rather than exhausting one corner of the grid first.
Usage:
uv run python scripts/hparam_scan.py
uv run python scripts/hparam_scan.py --dry-run
uv run python scripts/hparam_scan.py --seed 1 --data /path/to/parquet
See `uv run dwarf hparam-scan --help` for the CLI.
"""
import argparse
import csv
import itertools
import os
@@ -87,31 +82,29 @@ def append_summary(summary_path: Path, row: dict) -> None:
writer.writerow(row)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--data", default=DATA_DEFAULT)
parser.add_argument("--scan-dir", default=SCAN_DIR_DEFAULT)
parser.add_argument("--seed", type=int, default=0)
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
def run_hparam_scan(
data: str = DATA_DEFAULT,
scan_dir: str = SCAN_DIR_DEFAULT,
seed: int = 0,
dry_run: bool = False,
) -> None:
runs = build_runs(seed)
scan_dir_path = Path(scan_dir)
runs = build_runs(args.seed)
scan_dir = Path(args.scan_dir)
if args.dry_run:
if dry_run:
for i, run in enumerate(runs, 1):
print(f"[{i}/{len(runs)}] {run_name(run)}")
return
scan_dir.mkdir(parents=True, exist_ok=True)
summary_path = scan_dir / "scan_summary.csv"
scan_dir_path.mkdir(parents=True, exist_ok=True)
summary_path = scan_dir_path / "scan_summary.csv"
env = os.environ.copy()
env["TQDM_DISABLE"] = "1"
for i, run in enumerate(runs, 1):
name = run_name(run)
out_dir = scan_dir / name
out_dir = scan_dir_path / name
metrics_path = out_dir / "metrics.csv"
last_ckpt = out_dir / "last.pt"
@@ -124,7 +117,7 @@ def main() -> None:
cmd = [
"giant",
"train",
args.data,
data,
"--mode",
"flow",
"--epochs",
@@ -186,7 +179,3 @@ def main() -> None:
print(
f"[{i}/{len(runs)}] {name} — no metrics.csv produced, check train.log"
)
if __name__ == "__main__":
main()