Unify dataset/tooling scripts into a single dwarf Typer CLI
Replace the five separately-hyphenated uv entry points (steps-to-parquet, steps-to-parquet-parallel, migrate-geant-steps, bump-dataset-version, create-root-files) plus the unregistered hparam_scan.py with one `dwarf` command exposing convert/migrate/bump-gen/bump-schema/status/ update-manifest/create-manifest/make-root/hparam-scan as subcommands. Each scripts/*.py module now only holds argparse-free business logic; scripts/dwarf.py wires it up with Typer, matching giant/cli.py's style. `dwarf convert` merges the old serial/parallel conversion scripts behind a --jobs flag (default 1: sequential with plain -o; >1: dataset-layout fan-out via subprocess). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-time migration of /ceph/lbogner/geant_steps into the versioned layout:
|
||||
|
||||
raw/<kind>/<gen>/<detector>/shard-NNN.root
|
||||
@@ -13,12 +12,13 @@ up pool membership from POOL_ASSIGNMENT — decoupling "where it sits today" fro
|
||||
"which pool it belongs to".
|
||||
|
||||
Defaults to a dry run (prints the planned moves and manifest contents). Pass
|
||||
--execute to actually move files and write manifests. Pass --copy as well to
|
||||
copy instead of move, leaving the original files in place — e.g. if another
|
||||
process is still reading them from their current location.
|
||||
execute=True to actually move files and write manifests, and copy=True to copy
|
||||
instead of move, leaving the original files in place — e.g. if another process
|
||||
is still reading them from their current location.
|
||||
|
||||
See `uv run dwarf migrate --help` for the CLI.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
@@ -181,39 +181,18 @@ def plan_manifests(src_root: Path) -> dict[Path, list[str]]:
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"root",
|
||||
nargs="?",
|
||||
default="/ceph/lbogner/geant_steps",
|
||||
help="Dataset root to migrate in place (default: /ceph/lbogner/geant_steps)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--execute",
|
||||
action="store_true",
|
||||
help="Actually move files and write manifests (default: dry run / print plan only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--copy",
|
||||
action="store_true",
|
||||
help="Copy instead of move, leaving the originals in place "
|
||||
"(e.g. if another process is still reading them). Implies the legacy "
|
||||
"train/ etc. directories are left as-is too, since they won't be empty.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
src_root = Path(args.root)
|
||||
def run_migration(root: str, execute: bool, copy: bool) -> None:
|
||||
src_root = Path(root)
|
||||
if not src_root.is_dir():
|
||||
parser.error(f"{src_root} is not a directory")
|
||||
raise SystemExit(f"error: {src_root} is not a directory")
|
||||
|
||||
moves, unrecognized = plan_moves(src_root)
|
||||
manifests = plan_manifests(src_root)
|
||||
|
||||
verb = "COPY" if args.copy else "MOVE"
|
||||
print(f"=== {'EXECUTING' if args.execute else 'DRY RUN'} ({verb}): {src_root} ===\n")
|
||||
verb = "COPY" if copy else "MOVE"
|
||||
print(f"=== {'EXECUTING' if execute else 'DRY RUN'} ({verb}): {src_root} ===\n")
|
||||
|
||||
print(f"-- {len(moves)} file {'copy' if args.copy else 'move'}(s) --")
|
||||
print(f"-- {len(moves)} file {'copy' if copy else 'move'}(s) --")
|
||||
for src, dst in moves:
|
||||
print(f" {src.relative_to(src_root)} -> {dst.relative_to(src_root)}")
|
||||
|
||||
@@ -228,11 +207,11 @@ def main() -> None:
|
||||
for path in unrecognized:
|
||||
print(f" {path.relative_to(src_root)}")
|
||||
|
||||
if not args.execute:
|
||||
if not execute:
|
||||
print("\nDry run only — pass --execute to apply.")
|
||||
return
|
||||
|
||||
transfer = shutil.copy2 if args.copy else shutil.move
|
||||
transfer = shutil.copy2 if copy else shutil.move
|
||||
for src, dst in moves:
|
||||
if dst.exists():
|
||||
raise FileExistsError(f"refusing to overwrite existing file: {dst}")
|
||||
@@ -257,7 +236,7 @@ def main() -> None:
|
||||
# empty since their contents were classified by filename, not location —
|
||||
# remove them, but only if a move actually emptied them. In --copy mode the
|
||||
# originals are still there by design, so leave these alone entirely.
|
||||
if not args.copy:
|
||||
if not copy:
|
||||
for stale_dir in ("train", "sampling_train/small", "sampling_train"):
|
||||
d = src_root / stale_dir
|
||||
if d.is_dir() and not any(d.iterdir()):
|
||||
@@ -265,7 +244,3 @@ def main() -> None:
|
||||
print(f"removed now-empty directory: {d.relative_to(src_root)}")
|
||||
|
||||
print("\nDone.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user