Unify dataset/tooling scripts into a single dwarf Typer CLI
Replace the five separately-hyphenated uv entry points (steps-to-parquet, steps-to-parquet-parallel, migrate-geant-steps, bump-dataset-version, create-root-files) plus the unregistered hparam_scan.py with one `dwarf` command exposing convert/migrate/bump-gen/bump-schema/status/ update-manifest/create-manifest/make-root/hparam-scan as subcommands. Each scripts/*.py module now only holds argparse-free business logic; scripts/dwarf.py wires it up with Typer, matching giant/cli.py's style. `dwarf convert` merges the old serial/parallel conversion scripts behind a --jobs flag (default 1: sequential with plain -o; >1: dataset-layout fan-out via subprocess). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+14
-25
@@ -1,4 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Hyperparameter scan over dropout x n_blocks x hidden_dim.
|
||||
|
||||
Runs `giant train` sequentially (this machine has a single GPU) for every
|
||||
@@ -6,13 +5,9 @@ combination, plus one extra run at the default architecture with a higher
|
||||
learning rate. Runs are shuffled so the parameter space gets coarse coverage
|
||||
early rather than exhausting one corner of the grid first.
|
||||
|
||||
Usage:
|
||||
uv run python scripts/hparam_scan.py
|
||||
uv run python scripts/hparam_scan.py --dry-run
|
||||
uv run python scripts/hparam_scan.py --seed 1 --data /path/to/parquet
|
||||
See `uv run dwarf hparam-scan --help` for the CLI.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import itertools
|
||||
import os
|
||||
@@ -87,31 +82,29 @@ def append_summary(summary_path: Path, row: dict) -> None:
|
||||
writer.writerow(row)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--data", default=DATA_DEFAULT)
|
||||
parser.add_argument("--scan-dir", default=SCAN_DIR_DEFAULT)
|
||||
parser.add_argument("--seed", type=int, default=0)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
def run_hparam_scan(
|
||||
data: str = DATA_DEFAULT,
|
||||
scan_dir: str = SCAN_DIR_DEFAULT,
|
||||
seed: int = 0,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
runs = build_runs(seed)
|
||||
scan_dir_path = Path(scan_dir)
|
||||
|
||||
runs = build_runs(args.seed)
|
||||
scan_dir = Path(args.scan_dir)
|
||||
|
||||
if args.dry_run:
|
||||
if dry_run:
|
||||
for i, run in enumerate(runs, 1):
|
||||
print(f"[{i}/{len(runs)}] {run_name(run)}")
|
||||
return
|
||||
|
||||
scan_dir.mkdir(parents=True, exist_ok=True)
|
||||
summary_path = scan_dir / "scan_summary.csv"
|
||||
scan_dir_path.mkdir(parents=True, exist_ok=True)
|
||||
summary_path = scan_dir_path / "scan_summary.csv"
|
||||
|
||||
env = os.environ.copy()
|
||||
env["TQDM_DISABLE"] = "1"
|
||||
|
||||
for i, run in enumerate(runs, 1):
|
||||
name = run_name(run)
|
||||
out_dir = scan_dir / name
|
||||
out_dir = scan_dir_path / name
|
||||
metrics_path = out_dir / "metrics.csv"
|
||||
last_ckpt = out_dir / "last.pt"
|
||||
|
||||
@@ -124,7 +117,7 @@ def main() -> None:
|
||||
cmd = [
|
||||
"giant",
|
||||
"train",
|
||||
args.data,
|
||||
data,
|
||||
"--mode",
|
||||
"flow",
|
||||
"--epochs",
|
||||
@@ -186,7 +179,3 @@ def main() -> None:
|
||||
print(
|
||||
f"[{i}/{len(runs)}] {name} — no metrics.csv produced, check train.log"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user