Unify dataset/tooling scripts into a single dwarf Typer CLI

Replace the five separately-hyphenated uv entry points (steps-to-parquet,
steps-to-parquet-parallel, migrate-geant-steps, bump-dataset-version,
create-root-files) plus the unregistered hparam_scan.py with one `dwarf`
command exposing convert/migrate/bump-gen/bump-schema/status/
update-manifest/create-manifest/make-root/hparam-scan as subcommands.

Each scripts/*.py module now only holds argparse-free business logic;
scripts/dwarf.py wires it up with Typer, matching giant/cli.py's style.
`dwarf convert` merges the old serial/parallel conversion scripts behind
a --jobs flag (default 1: sequential with plain -o; >1: dataset-layout
fan-out via subprocess).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 09:52:37 +02:00
parent 7b37b284f8
commit d5853d5a75
13 changed files with 662 additions and 505 deletions
+14 -39
View File
@@ -1,4 +1,3 @@
#!/usr/bin/env python3
"""One-time migration of /ceph/lbogner/geant_steps into the versioned layout:
raw/<kind>/<gen>/<detector>/shard-NNN.root
@@ -13,12 +12,13 @@ up pool membership from POOL_ASSIGNMENT — decoupling "where it sits today" fro
"which pool it belongs to".
Defaults to a dry run (prints the planned moves and manifest contents). Pass
--execute to actually move files and write manifests. Pass --copy as well to
copy instead of move, leaving the original files in place — e.g. if another
process is still reading them from their current location.
execute=True to actually move files and write manifests, and copy=True to copy
instead of move, leaving the original files in place — e.g. if another process
is still reading them from their current location.
See `uv run dwarf migrate --help` for the CLI.
"""
import argparse
import os
import re
import shutil
@@ -181,39 +181,18 @@ def plan_manifests(src_root: Path) -> dict[Path, list[str]]:
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"root",
nargs="?",
default="/ceph/lbogner/geant_steps",
help="Dataset root to migrate in place (default: /ceph/lbogner/geant_steps)",
)
parser.add_argument(
"--execute",
action="store_true",
help="Actually move files and write manifests (default: dry run / print plan only)",
)
parser.add_argument(
"--copy",
action="store_true",
help="Copy instead of move, leaving the originals in place "
"(e.g. if another process is still reading them). Implies the legacy "
"train/ etc. directories are left as-is too, since they won't be empty.",
)
args = parser.parse_args()
src_root = Path(args.root)
def run_migration(root: str, execute: bool, copy: bool) -> None:
src_root = Path(root)
if not src_root.is_dir():
parser.error(f"{src_root} is not a directory")
raise SystemExit(f"error: {src_root} is not a directory")
moves, unrecognized = plan_moves(src_root)
manifests = plan_manifests(src_root)
verb = "COPY" if args.copy else "MOVE"
print(f"=== {'EXECUTING' if args.execute else 'DRY RUN'} ({verb}): {src_root} ===\n")
verb = "COPY" if copy else "MOVE"
print(f"=== {'EXECUTING' if execute else 'DRY RUN'} ({verb}): {src_root} ===\n")
print(f"-- {len(moves)} file {'copy' if args.copy else 'move'}(s) --")
print(f"-- {len(moves)} file {'copy' if copy else 'move'}(s) --")
for src, dst in moves:
print(f" {src.relative_to(src_root)} -> {dst.relative_to(src_root)}")
@@ -228,11 +207,11 @@ def main() -> None:
for path in unrecognized:
print(f" {path.relative_to(src_root)}")
if not args.execute:
if not execute:
print("\nDry run only — pass --execute to apply.")
return
transfer = shutil.copy2 if args.copy else shutil.move
transfer = shutil.copy2 if copy else shutil.move
for src, dst in moves:
if dst.exists():
raise FileExistsError(f"refusing to overwrite existing file: {dst}")
@@ -257,7 +236,7 @@ def main() -> None:
# empty since their contents were classified by filename, not location —
# remove them, but only if a move actually emptied them. In --copy mode the
# originals are still there by design, so leave these alone entirely.
if not args.copy:
if not copy:
for stale_dir in ("train", "sampling_train/small", "sampling_train"):
d = src_root / stale_dir
if d.is_dir() and not any(d.iterdir()):
@@ -265,7 +244,3 @@ def main() -> None:
print(f"removed now-empty directory: {d.relative_to(src_root)}")
print("\nDone.")
if __name__ == "__main__":
main()