Unify dataset/tooling scripts into a single dwarf Typer CLI

Replace the five separately-hyphenated uv entry points (steps-to-parquet,
steps-to-parquet-parallel, migrate-geant-steps, bump-dataset-version,
create-root-files) plus the unregistered hparam_scan.py with one `dwarf`
command exposing convert/migrate/bump-gen/bump-schema/status/
update-manifest/create-manifest/make-root/hparam-scan as subcommands.

Each scripts/*.py module now only holds argparse-free business logic;
scripts/dwarf.py wires it up with Typer, matching giant/cli.py's style.
`dwarf convert` merges the old serial/parallel conversion scripts behind
a --jobs flag (default 1: sequential with plain -o; >1: dataset-layout
fan-out via subprocess).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 09:52:37 +02:00
parent 7b37b284f8
commit d5853d5a75
13 changed files with 662 additions and 505 deletions
+34 -77
View File
@@ -1,4 +1,3 @@
#!/usr/bin/env python3
"""Generate new ROOT shards by running a minicalosim executable (e.g.
run_pbwo4, run_sampling) and filing the output into the dataset's raw/ tree:
@@ -15,19 +14,11 @@ appear there, and moves it to the next free shard index for that detector
(existing shards are never overwritten).
--gen must already exist under raw/<kind>/ — create one first with
bump_dataset_version.py bump-gen.
`dwarf bump-gen`.
Usage:
create_root_files.py --executable build/run_pbwo4 --detector pbwo4 \\
--gen gen1 --num-files 4 --events-per-file 10000 --execute
create_root_files.py --executable build/run_sampling \\
--detector sampling_pb_scint:pb_scint \\
--detector sampling_fe_scint:fe_scint \\
--gen gen1 --num-files 4 --events-per-file 10000 --jobs 8 --execute
See `uv run dwarf make-root --help` for the CLI.
"""
import argparse
import os
import re
import shutil
@@ -192,78 +183,48 @@ def run_all(
return results
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--executable", required=True, type=Path, help="Built minicalosim run_* executable"
)
parser.add_argument(
"--detector",
action="append",
required=True,
metavar="NAME[:CONFIG]",
help="Dataset detector label, optionally with ':CONFIG' to pass as the "
"executable's config-name argument (e.g. sampling_pb_scint:pb_scint). "
"Omit ':CONFIG' for executables that take no config selector (e.g. run_pbwo4). "
"Repeatable.",
)
parser.add_argument(
"--num-files", type=int, required=True, help="New shards to create per detector"
)
parser.add_argument(
"--events-per-file", type=int, required=True, help="nEvents passed to the executable"
)
parser.add_argument("--kind", default="steps", help="steps | hits | ... (default: steps)")
parser.add_argument("--gen", required=True, help="Existing gen tag under raw/<kind>/, e.g. gen1")
parser.add_argument(
"--dataset-root",
default="/ceph/lbogner/geant_steps",
help="Dataset root (default: /ceph/lbogner/geant_steps)",
)
parser.add_argument(
"-j", "--jobs", type=int, default=4, help="Parallel simulation runs (default: 4)"
)
parser.add_argument(
"--execute", action="store_true", help="Actually run jobs (default: dry run / print plan)"
)
return parser
def run_make_root(
executable: Path,
detector: list[str],
num_files: int,
events_per_file: int,
kind: str,
gen: str,
dataset_root: str,
jobs: int,
execute: bool,
) -> None:
if jobs < 1:
raise SystemExit("error: --jobs must be >= 1")
if num_files < 1:
raise SystemExit("error: --num-files must be >= 1")
if events_per_file < 1:
raise SystemExit("error: --events-per-file must be >= 1")
if not executable.is_file() or not os.access(executable, os.X_OK):
raise SystemExit(f"error: {executable} is not an executable file")
def main() -> None:
parser = build_parser()
args = parser.parse_args()
if args.jobs < 1:
parser.error("--jobs must be >= 1")
if args.num_files < 1:
parser.error("--num-files must be >= 1")
if args.events_per_file < 1:
parser.error("--events-per-file must be >= 1")
if not args.executable.is_file() or not os.access(args.executable, os.X_OK):
parser.error(f"{args.executable} is not an executable file")
dataset_root = Path(args.dataset_root)
dataset_root_path = Path(dataset_root)
try:
jobs = plan_jobs(args.detector, args.num_files, dataset_root, args.kind, args.gen)
planned_jobs = plan_jobs(detector, num_files, dataset_root_path, kind, gen)
except PlanError as exc:
parser.error(str(exc))
raise SystemExit(f"error: {exc}")
print(f"=== {'EXECUTING' if args.execute else 'DRY RUN'} ===")
print(f"executable: {args.executable}")
for job in jobs:
cmd = [str(args.executable)] + ([job.config] if job.config else []) + [str(args.events_per_file)]
dest = dataset_root / "raw" / args.kind / args.gen / job.detector / f"shard-{job.shard_index:03d}.root"
print(f"=== {'EXECUTING' if execute else 'DRY RUN'} ===")
print(f"executable: {executable}")
for job in planned_jobs:
cmd = [str(executable)] + ([job.config] if job.config else []) + [str(events_per_file)]
dest = dataset_root_path / "raw" / kind / gen / job.detector / f"shard-{job.shard_index:03d}.root"
print(f" {' '.join(cmd)} -> {dest}")
if not args.execute:
if not execute:
print("\nDry run only — pass --execute to apply.")
return
tmp_root = dataset_root / ".sim-tmp"
tmp_root = dataset_root_path / ".sim-tmp"
tmp_root.mkdir(parents=True, exist_ok=True)
results = run_all(
jobs, args.executable, args.events_per_file, dataset_root, args.kind, args.gen,
max_workers=args.jobs, tmp_root=tmp_root,
planned_jobs, executable, events_per_file, dataset_root_path, kind, gen,
max_workers=jobs, tmp_root=tmp_root,
)
if tmp_root.is_dir() and not any(tmp_root.iterdir()):
tmp_root.rmdir()
@@ -273,10 +234,6 @@ def main() -> None:
print(f"\n{len(failures)} of {len(results)} job(s) failed:", file=sys.stderr)
for r in failures:
print(f" {r.job.detector} shard-{r.job.shard_index:03d}: {r.message}", file=sys.stderr)
sys.exit(1)
raise SystemExit(1)
print(f"\nAll {len(results)} job(s) completed.")
if __name__ == "__main__":
main()