Bump ruff line-length to 120 and reformat
CI / Lint (ruff check) (push) Successful in 31s
CI / Format (ruff format) (push) Successful in 32s
CI / Sync project version with tag (push) Has been skipped
CI / Lint (ruff check) (pull_request) Successful in 26s
CI / Type check (ty) (push) Successful in 29s
CI / Format (ruff format) (pull_request) Successful in 33s
CI / Sync project version with tag (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 35s
CI / Tests (pull_request) Successful in 3m47s
CI / Tests (push) Successful in 3m55s
CI / Lint (ruff check) (push) Successful in 31s
CI / Format (ruff format) (push) Successful in 32s
CI / Sync project version with tag (push) Has been skipped
CI / Lint (ruff check) (pull_request) Successful in 26s
CI / Type check (ty) (push) Successful in 29s
CI / Format (ruff format) (pull_request) Successful in 33s
CI / Sync project version with tag (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 35s
CI / Tests (pull_request) Successful in 3m47s
CI / Tests (push) Successful in 3m55s
Rejoins lines that only wrapped because they exceeded the old 88-char limit; ruff check and the full test suite (725 passed) are unaffected.
This commit is contained in:
@@ -82,9 +82,7 @@ def _max_index(parent: Path, pattern: re.Pattern) -> int:
|
||||
|
||||
def _git_user_name() -> str | None:
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["git", "config", "user.name"], capture_output=True, text=True, timeout=2
|
||||
)
|
||||
out = subprocess.run(["git", "config", "user.name"], capture_output=True, text=True, timeout=2)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
# OSError (e.g. git not on PATH) and subprocess.SubprocessError
|
||||
# (e.g. TimeoutExpired) are unrelated hierarchies — TimeoutExpired
|
||||
@@ -142,9 +140,7 @@ def plan_bump_schema(
|
||||
raw_gen_dir = root / "raw" / kind / gen_tag
|
||||
processed_gen_dir = root / "processed" / kind / gen_tag
|
||||
if not raw_gen_dir.is_dir() and not processed_gen_dir.is_dir():
|
||||
raise SystemExit(
|
||||
f"error: {gen_tag} doesn't exist yet for kind={kind} — run bump-gen first"
|
||||
)
|
||||
raise SystemExit(f"error: {gen_tag} doesn't exist yet for kind={kind} — run bump-gen first")
|
||||
if target is not None:
|
||||
if not SCHEMA_RE.match(target):
|
||||
raise SystemExit(f"error: --to must look like 'schemaN', got {target!r}")
|
||||
@@ -154,9 +150,7 @@ def plan_bump_schema(
|
||||
schema_tag = f"schema{next_schema}"
|
||||
new_dirs = [processed_gen_dir / schema_tag]
|
||||
by_suffix = f" ({by})" if by else ""
|
||||
log_line = (
|
||||
f"- `{gen_tag}`/`{schema_tag}` (kind={kind}) — {date} — {reason}{by_suffix}"
|
||||
)
|
||||
log_line = f"- `{gen_tag}`/`{schema_tag}` (kind={kind}) — {date} — {reason}{by_suffix}"
|
||||
return new_dirs, log_line
|
||||
|
||||
|
||||
@@ -212,9 +206,7 @@ def _manifest_referenced_files(pools_root: Path) -> set[Path]:
|
||||
return referenced
|
||||
|
||||
|
||||
def _referenced_root_count(
|
||||
raw_gen_dir: Path, processed_gen_dir: Path
|
||||
) -> tuple[int, int]:
|
||||
def _referenced_root_count(raw_gen_dir: Path, processed_gen_dir: Path) -> tuple[int, int]:
|
||||
"""(total .root files, count with a same-named .parquet under any schema) for one gen."""
|
||||
if not raw_gen_dir.is_dir():
|
||||
return 0, 0
|
||||
@@ -243,9 +235,7 @@ def _referenced_root_count(
|
||||
return total, referenced
|
||||
|
||||
|
||||
def _referenced_parquet_count(
|
||||
schema_dir: Path, manifest_referenced: set[Path]
|
||||
) -> tuple[int, int]:
|
||||
def _referenced_parquet_count(schema_dir: Path, manifest_referenced: set[Path]) -> tuple[int, int]:
|
||||
"""(total .parquet files, count listed in at least one manifest) for one schema dir."""
|
||||
if not schema_dir.is_dir():
|
||||
return 0, 0
|
||||
@@ -342,11 +332,7 @@ def print_status(root: Path) -> None:
|
||||
grand_files = 0
|
||||
for kind_dir in sorted(p for p in raw_root.iterdir() if p.is_dir()):
|
||||
kind = kind_dir.name
|
||||
gens = sorted(
|
||||
int(m.group(1))
|
||||
for m in (GEN_RE.match(p.name) for p in kind_dir.iterdir() if p.is_dir())
|
||||
if m
|
||||
)
|
||||
gens = sorted(int(m.group(1)) for m in (GEN_RE.match(p.name) for p in kind_dir.iterdir() if p.is_dir()) if m)
|
||||
print(_colorize(f"{kind}/", "kind"))
|
||||
kind_total = 0
|
||||
kind_files = 0
|
||||
@@ -359,25 +345,18 @@ def print_status(root: Path) -> None:
|
||||
schemas = sorted(
|
||||
int(m.group(1))
|
||||
for m in (
|
||||
SCHEMA_RE.match(p.name)
|
||||
for p in (schema_dir.iterdir() if schema_dir.is_dir() else [])
|
||||
if p.is_dir()
|
||||
SCHEMA_RE.match(p.name) for p in (schema_dir.iterdir() if schema_dir.is_dir() else []) if p.is_dir()
|
||||
)
|
||||
if m
|
||||
)
|
||||
schema_sizes = {s: _du(schema_dir / f"schema{s}") for s in schemas}
|
||||
schema_counts = {
|
||||
s: _referenced_parquet_count(
|
||||
schema_dir / f"schema{s}", manifest_referenced
|
||||
)
|
||||
for s in schemas
|
||||
s: _referenced_parquet_count(schema_dir / f"schema{s}", manifest_referenced) for s in schemas
|
||||
}
|
||||
processed_size = sum(schema_sizes.values())
|
||||
processed_files = sum(c[0] for c in schema_counts.values())
|
||||
processed_referenced = sum(c[1] for c in schema_counts.values())
|
||||
raw_files, raw_referenced = _referenced_root_count(
|
||||
raw_gen_dir, processed_gen_dir
|
||||
)
|
||||
raw_files, raw_referenced = _referenced_root_count(raw_gen_dir, processed_gen_dir)
|
||||
gen_total = raw_size + processed_size
|
||||
gen_files = raw_files + processed_files
|
||||
kind_total += gen_total
|
||||
@@ -425,9 +404,7 @@ def print_status(root: Path) -> None:
|
||||
print(_reason_line(schema_reason, indent=4))
|
||||
else:
|
||||
print(_colorize(" (none)", "schema"))
|
||||
print(
|
||||
_row(f"{kind} total", kind_total, indent=1, level="gen", count=kind_files)
|
||||
)
|
||||
print(_row(f"{kind} total", kind_total, indent=1, level="gen", count=kind_files))
|
||||
print()
|
||||
grand_total += kind_total
|
||||
grand_files += kind_files
|
||||
@@ -526,13 +503,8 @@ def plan_update_manifest(
|
||||
return result, missing
|
||||
|
||||
|
||||
def apply_update_manifest(
|
||||
manifest_path: Path, lines: list[tuple[str, str | None]]
|
||||
) -> None:
|
||||
out = [
|
||||
replacement if replacement is not None else original
|
||||
for original, replacement in lines
|
||||
]
|
||||
def apply_update_manifest(manifest_path: Path, lines: list[tuple[str, str | None]]) -> None:
|
||||
out = [replacement if replacement is not None else original for original, replacement in lines]
|
||||
manifest_path.write_text("\n".join(out) + "\n")
|
||||
|
||||
|
||||
@@ -552,9 +524,7 @@ def _resolve_manifest_files(manifest_path: Path) -> list[Path]:
|
||||
return files
|
||||
|
||||
|
||||
def plan_create_manifest(
|
||||
output_path: Path, parquet_files: list[Path]
|
||||
) -> tuple[list[str], list[Path], list[Path]]:
|
||||
def plan_create_manifest(output_path: Path, parquet_files: list[Path]) -> tuple[list[str], list[Path], list[Path]]:
|
||||
"""Return (relative_lines, missing_files, resolved_abs_paths)."""
|
||||
manifest_dir = output_path.resolve().parent
|
||||
lines: list[str] = []
|
||||
@@ -569,9 +539,7 @@ def plan_create_manifest(
|
||||
return lines, missing, resolved
|
||||
|
||||
|
||||
def check_holdout_overlap(
|
||||
output_path: Path, resolved_new_files: list[Path]
|
||||
) -> list[tuple[str, Path]]:
|
||||
def check_holdout_overlap(output_path: Path, resolved_new_files: list[Path]) -> list[tuple[str, Path]]:
|
||||
"""Return (other_manifest_name, file) pairs where new files clash with existing manifests.
|
||||
|
||||
The check is triggered when output_path is (or will be) holdout.manifest, or when a
|
||||
@@ -641,9 +609,7 @@ def _run_bump(
|
||||
if gen is None:
|
||||
new_dirs, log_line = plan_bump_gen(root_path, kind, reason, by, date, to)
|
||||
else:
|
||||
new_dirs, log_line = plan_bump_schema(
|
||||
root_path, kind, gen, reason, by, date, to
|
||||
)
|
||||
new_dirs, log_line = plan_bump_schema(root_path, kind, gen, reason, by, date, to)
|
||||
|
||||
print(f"=== {'EXECUTING' if execute else 'DRY RUN'} ===")
|
||||
print("new directories:")
|
||||
|
||||
@@ -84,19 +84,12 @@ def main() -> int:
|
||||
mode = model_config.get("mode", "flow")
|
||||
routed = bool((model_config.get("router") or {}).get("enabled"))
|
||||
print(f"checkpoint: {args.checkpoint}")
|
||||
print(
|
||||
f" mode={mode!r} conditioning={model_config.get('conditioning')!r} "
|
||||
f"routed={routed} ema={args.ema}"
|
||||
)
|
||||
print(f" mode={mode!r} conditioning={model_config.get('conditioning')!r} routed={routed} ema={args.ema}")
|
||||
|
||||
stage1_key = "model_ema" if args.ema and "model_ema" in ckpt else "model"
|
||||
stage2_key = (
|
||||
"sec_decoder_ema" if args.ema and "sec_decoder_ema" in ckpt else "sec_decoder"
|
||||
)
|
||||
stage2_key = "sec_decoder_ema" if args.ema and "sec_decoder_ema" in ckpt else "sec_decoder"
|
||||
if args.ema and stage1_key == "model":
|
||||
print(
|
||||
" warning: --ema requested but no model_ema in checkpoint, using raw weights"
|
||||
)
|
||||
print(" warning: --ema requested but no model_ema in checkpoint, using raw weights")
|
||||
|
||||
# --- old side: the frozen v0.2 snapshot, loaded with the checkpoint's own weights ---
|
||||
old_stage1, old_stage2 = legacy.build_models(model_config)
|
||||
@@ -119,9 +112,7 @@ def main() -> int:
|
||||
print("PASS (construction only, routed checkpoint)")
|
||||
return 0
|
||||
|
||||
remapped1, remapped2 = net.migrate_legacy_state_dict(
|
||||
ckpt[stage1_key], ckpt[stage2_key]
|
||||
)
|
||||
remapped1, remapped2 = net.migrate_legacy_state_dict(ckpt[stage1_key], ckpt[stage2_key])
|
||||
missing1, unexpected1 = new_stage1.load_state_dict(remapped1, strict=True)
|
||||
missing2, unexpected2 = new_stage2.load_state_dict(remapped2, strict=True)
|
||||
if missing1 or unexpected1 or missing2 or unexpected2:
|
||||
@@ -132,9 +123,7 @@ def main() -> int:
|
||||
new_stage1.eval()
|
||||
new_stage2.eval()
|
||||
|
||||
cond_cont, cond_cat, x1, x2, t, z1, z2 = _random_batch(
|
||||
model_config, args.batch, args.seed
|
||||
)
|
||||
cond_cont, cond_cat, x1, x2, t, z1, z2 = _random_batch(model_config, args.batch, args.seed)
|
||||
|
||||
ok = True
|
||||
with torch.no_grad():
|
||||
|
||||
@@ -64,9 +64,7 @@ def parse_detector_spec(spec: str) -> tuple[str, str | None]:
|
||||
if ":" in spec:
|
||||
label, config = spec.split(":", 1)
|
||||
if not label or not config:
|
||||
raise PlanError(
|
||||
f"invalid --detector spec {spec!r}: expected NAME or NAME:CONFIG"
|
||||
)
|
||||
raise PlanError(f"invalid --detector spec {spec!r}: expected NAME or NAME:CONFIG")
|
||||
return label, config
|
||||
return spec, None
|
||||
|
||||
@@ -94,9 +92,7 @@ def plan_jobs(
|
||||
raise PlanError(f"--gen must look like 'genN', got {gen!r}")
|
||||
gen_dir = dataset_root / "raw" / kind / gen
|
||||
if not gen_dir.is_dir():
|
||||
raise PlanError(
|
||||
f"{gen_dir} doesn't exist — run bump_dataset_version.py bump-gen first"
|
||||
)
|
||||
raise PlanError(f"{gen_dir} doesn't exist — run bump_dataset_version.py bump-gen first")
|
||||
|
||||
jobs = []
|
||||
for spec in detector_specs:
|
||||
@@ -124,9 +120,7 @@ def job_seed(kind: str, gen: str, job: SimJob, energy_gev: float | None) -> int:
|
||||
return zlib.crc32(key.encode()) & 0x7FFFFFFF
|
||||
|
||||
|
||||
def build_cmd(
|
||||
executable: Path, job: SimJob, events_per_file: int, energy_gev: float | None
|
||||
) -> list[str]:
|
||||
def build_cmd(executable: Path, job: SimJob, events_per_file: int, energy_gev: float | None) -> list[str]:
|
||||
"""minicalosim executables take positional `[configName] nEvents [energy_GeV]`."""
|
||||
cmd = [str(executable)]
|
||||
if job.config:
|
||||
@@ -147,10 +141,7 @@ def run_job(
|
||||
gen: str,
|
||||
tmp_root: Path,
|
||||
) -> JobResult:
|
||||
workdir = (
|
||||
tmp_root
|
||||
/ f"{kind}-{gen}-{job.detector}-{job.shard_index:03d}-{uuid.uuid4().hex[:8]}"
|
||||
)
|
||||
workdir = tmp_root / f"{kind}-{gen}-{job.detector}-{job.shard_index:03d}-{uuid.uuid4().hex[:8]}"
|
||||
workdir.mkdir(parents=True)
|
||||
|
||||
cmd = build_cmd(executable, job, events_per_file, energy_gev)
|
||||
@@ -174,20 +165,12 @@ def run_job(
|
||||
job,
|
||||
False,
|
||||
None,
|
||||
f"expected exactly one .root output in {workdir}, found {len(produced)}: "
|
||||
f"{[p.name for p in produced]}",
|
||||
f"expected exactly one .root output in {workdir}, found {len(produced)}: {[p.name for p in produced]}",
|
||||
result.stdout,
|
||||
result.stderr,
|
||||
)
|
||||
|
||||
dest = (
|
||||
dataset_root
|
||||
/ "raw"
|
||||
/ kind
|
||||
/ gen
|
||||
/ job.detector
|
||||
/ f"shard-{job.shard_index:03d}.root"
|
||||
)
|
||||
dest = dataset_root / "raw" / kind / gen / job.detector / f"shard-{job.shard_index:03d}.root"
|
||||
if dest.exists():
|
||||
return JobResult(
|
||||
job,
|
||||
@@ -279,14 +262,7 @@ def run_make_root(
|
||||
print(f"executable: {executable}")
|
||||
for job in planned_jobs:
|
||||
cmd = build_cmd(executable, job, events_per_file, energy_gev)
|
||||
dest = (
|
||||
dataset_root_path
|
||||
/ "raw"
|
||||
/ kind
|
||||
/ gen
|
||||
/ job.detector
|
||||
/ f"shard-{job.shard_index:03d}.root"
|
||||
)
|
||||
dest = dataset_root_path / "raw" / kind / gen / job.detector / f"shard-{job.shard_index:03d}.root"
|
||||
seed = job_seed(kind, gen, job, energy_gev)
|
||||
print(f" MINICALOSIM_SEED={seed} {' '.join(cmd)} -> {dest}")
|
||||
|
||||
|
||||
+40
-116
@@ -80,8 +80,7 @@ def convert(
|
||||
typer.Option(
|
||||
"--output",
|
||||
"-o",
|
||||
help="Output Parquet file (default: <input>.parquet). Only valid "
|
||||
"with a single input file and --jobs 1.",
|
||||
help="Output Parquet file (default: <input>.parquet). Only valid with a single input file and --jobs 1.",
|
||||
),
|
||||
] = None,
|
||||
batch_size: Annotated[
|
||||
@@ -91,9 +90,7 @@ def convert(
|
||||
help="Uproot read batch size, e.g. '100 MB' or '500000' (rows)",
|
||||
),
|
||||
] = "100 MB",
|
||||
tree: Annotated[
|
||||
str, typer.Option("--tree", help="Tree name inside the ROOT file")
|
||||
] = "Steps",
|
||||
tree: Annotated[str, typer.Option("--tree", help="Tree name inside the ROOT file")] = "Steps",
|
||||
compression: Annotated[
|
||||
Compression, typer.Option("--compression", help="Parquet compression codec")
|
||||
] = Compression.snappy,
|
||||
@@ -129,15 +126,11 @@ def convert(
|
||||
raise typer.Exit(1)
|
||||
_warn_if_exceeds_shared_quota(jobs, "--jobs")
|
||||
|
||||
compression_value = (
|
||||
"uncompressed" if compression is Compression.none else compression.value
|
||||
)
|
||||
compression_value = "uncompressed" if compression is Compression.none else compression.value
|
||||
|
||||
if jobs == 1:
|
||||
if output is not None and len(root_files) > 1:
|
||||
typer.echo(
|
||||
"error: --output can only be used with a single input file", err=True
|
||||
)
|
||||
typer.echo("error: --output can only be used with a single input file", err=True)
|
||||
raise typer.Exit(1)
|
||||
total_orphaned = 0
|
||||
for root_file in root_files:
|
||||
@@ -150,10 +143,7 @@ def convert(
|
||||
)
|
||||
total_orphaned += n_orphaned
|
||||
if total_orphaned:
|
||||
typer.echo(
|
||||
f"\n{total_orphaned} orphaned child track(s) dropped across "
|
||||
f"{len(root_files)} file(s)."
|
||||
)
|
||||
typer.echo(f"\n{total_orphaned} orphaned child track(s) dropped across {len(root_files)} file(s).")
|
||||
return
|
||||
|
||||
if output is not None:
|
||||
@@ -176,9 +166,7 @@ def convert(
|
||||
|
||||
@app.command()
|
||||
def migrate(
|
||||
root: Annotated[
|
||||
Path, typer.Argument(help="Dataset root to migrate in place")
|
||||
] = _DATASET_ROOT_DEFAULT,
|
||||
root: Annotated[Path, typer.Argument(help="Dataset root to migrate in place")] = _DATASET_ROOT_DEFAULT,
|
||||
execute: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
@@ -190,8 +178,7 @@ def migrate(
|
||||
bool,
|
||||
typer.Option(
|
||||
"--copy",
|
||||
help="Copy instead of move, leaving the originals in place "
|
||||
"(e.g. if another process is still reading them)",
|
||||
help="Copy instead of move, leaving the originals in place (e.g. if another process is still reading them)",
|
||||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
@@ -202,12 +189,8 @@ def migrate(
|
||||
@app.command("bump-gen")
|
||||
def bump_gen(
|
||||
reason: Annotated[str, typer.Option("--reason", help="Why this gen exists")],
|
||||
kind: Annotated[
|
||||
str, typer.Option("--kind", help="steps | hits | ... (default: steps)")
|
||||
] = "steps",
|
||||
by: Annotated[
|
||||
Optional[str], typer.Option("--by", help="Attribution (default: git user.name)")
|
||||
] = None,
|
||||
kind: Annotated[str, typer.Option("--kind", help="steps | hits | ... (default: steps)")] = "steps",
|
||||
by: Annotated[Optional[str], typer.Option("--by", help="Attribution (default: git user.name)")] = None,
|
||||
date: Annotated[
|
||||
Optional[str],
|
||||
typer.Option("--date", help="Override date (default: today, ISO)"),
|
||||
@@ -220,12 +203,8 @@ def bump_gen(
|
||||
help="Target gen tag (default: one past the current highest)",
|
||||
),
|
||||
] = None,
|
||||
execute: Annotated[
|
||||
bool, typer.Option("--execute", help="Apply (default: dry run)")
|
||||
] = False,
|
||||
root: Annotated[
|
||||
Path, typer.Option("--root", help="Dataset root")
|
||||
] = _DATASET_ROOT_DEFAULT,
|
||||
execute: Annotated[bool, typer.Option("--execute", help="Apply (default: dry run)")] = False,
|
||||
root: Annotated[Path, typer.Option("--root", help="Dataset root")] = _DATASET_ROOT_DEFAULT,
|
||||
) -> None:
|
||||
"""Cut a new raw generation."""
|
||||
run_bump_gen(
|
||||
@@ -243,12 +222,8 @@ def bump_gen(
|
||||
def bump_schema(
|
||||
gen: Annotated[str, typer.Option("--gen", help="Existing gen tag, e.g. gen1")],
|
||||
reason: Annotated[str, typer.Option("--reason", help="Why this schema exists")],
|
||||
kind: Annotated[
|
||||
str, typer.Option("--kind", help="steps | hits | ... (default: steps)")
|
||||
] = "steps",
|
||||
by: Annotated[
|
||||
Optional[str], typer.Option("--by", help="Attribution (default: git user.name)")
|
||||
] = None,
|
||||
kind: Annotated[str, typer.Option("--kind", help="steps | hits | ... (default: steps)")] = "steps",
|
||||
by: Annotated[Optional[str], typer.Option("--by", help="Attribution (default: git user.name)")] = None,
|
||||
date: Annotated[
|
||||
Optional[str],
|
||||
typer.Option("--date", help="Override date (default: today, ISO)"),
|
||||
@@ -261,12 +236,8 @@ def bump_schema(
|
||||
help="Target schema tag (default: one past the current highest)",
|
||||
),
|
||||
] = None,
|
||||
execute: Annotated[
|
||||
bool, typer.Option("--execute", help="Apply (default: dry run)")
|
||||
] = False,
|
||||
root: Annotated[
|
||||
Path, typer.Option("--root", help="Dataset root")
|
||||
] = _DATASET_ROOT_DEFAULT,
|
||||
execute: Annotated[bool, typer.Option("--execute", help="Apply (default: dry run)")] = False,
|
||||
root: Annotated[Path, typer.Option("--root", help="Dataset root")] = _DATASET_ROOT_DEFAULT,
|
||||
) -> None:
|
||||
"""Cut a new schema within a gen."""
|
||||
run_bump_schema(
|
||||
@@ -283,9 +254,7 @@ def bump_schema(
|
||||
|
||||
@app.command()
|
||||
def status(
|
||||
root: Annotated[
|
||||
Path, typer.Option("--root", help="Dataset root")
|
||||
] = _DATASET_ROOT_DEFAULT,
|
||||
root: Annotated[Path, typer.Option("--root", help="Dataset root")] = _DATASET_ROOT_DEFAULT,
|
||||
) -> None:
|
||||
"""List existing gens/schemas per kind."""
|
||||
run_status(str(root))
|
||||
@@ -293,9 +262,7 @@ def status(
|
||||
|
||||
@app.command("update-manifest")
|
||||
def update_manifest(
|
||||
manifests: Annotated[
|
||||
list[Path], typer.Argument(help="One or more .manifest files to update")
|
||||
],
|
||||
manifests: Annotated[list[Path], typer.Argument(help="One or more .manifest files to update")],
|
||||
schema: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
@@ -306,9 +273,7 @@ def update_manifest(
|
||||
] = None,
|
||||
gen: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
"--gen", metavar="genN", help="Target gen tag (default: keep existing gen)"
|
||||
),
|
||||
typer.Option("--gen", metavar="genN", help="Target gen tag (default: keep existing gen)"),
|
||||
] = None,
|
||||
execute: Annotated[
|
||||
bool,
|
||||
@@ -316,9 +281,7 @@ def update_manifest(
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Repoint manifest(s) to a new gen and/or schema, verifying all target files exist."""
|
||||
run_update_manifest(
|
||||
[str(m) for m in manifests], schema=schema, execute=execute, gen=gen
|
||||
)
|
||||
run_update_manifest([str(m) for m in manifests], schema=schema, execute=execute, gen=gen)
|
||||
|
||||
|
||||
@app.command("create-manifest")
|
||||
@@ -333,22 +296,15 @@ def create_manifest(
|
||||
typer.Option(
|
||||
"--pool",
|
||||
metavar="DETECTOR",
|
||||
help="Detector name; combined with --type and --root to form "
|
||||
"<root>/pools/<detector>/<type>.manifest",
|
||||
help="Detector name; combined with --type and --root to form <root>/pools/<detector>/<type>.manifest",
|
||||
),
|
||||
] = None,
|
||||
type_: Annotated[
|
||||
Optional[PoolType],
|
||||
typer.Option(
|
||||
"--type", help="Pool type — full, holdout, or dev (required with --pool)"
|
||||
),
|
||||
typer.Option("--type", help="Pool type — full, holdout, or dev (required with --pool)"),
|
||||
] = None,
|
||||
root: Annotated[
|
||||
Path, typer.Option("--root", help="Dataset root (used with --pool)")
|
||||
] = _DATASET_ROOT_DEFAULT,
|
||||
execute: Annotated[
|
||||
bool, typer.Option("--execute", help="Write the manifest (default: dry run)")
|
||||
] = False,
|
||||
root: Annotated[Path, typer.Option("--root", help="Dataset root (used with --pool)")] = _DATASET_ROOT_DEFAULT,
|
||||
execute: Annotated[bool, typer.Option("--execute", help="Write the manifest (default: dry run)")] = False,
|
||||
force: Annotated[
|
||||
bool,
|
||||
typer.Option("--force", help="Overwrite the manifest if it already exists"),
|
||||
@@ -368,9 +324,7 @@ def create_manifest(
|
||||
|
||||
@app.command("make-root")
|
||||
def make_root(
|
||||
executable: Annotated[
|
||||
Path, typer.Option("--executable", help="Built minicalosim run_* executable")
|
||||
],
|
||||
executable: Annotated[Path, typer.Option("--executable", help="Built minicalosim run_* executable")],
|
||||
detector: Annotated[
|
||||
list[str],
|
||||
typer.Option(
|
||||
@@ -382,15 +336,9 @@ def make_root(
|
||||
"Repeatable.",
|
||||
),
|
||||
],
|
||||
num_files: Annotated[
|
||||
int, typer.Option("--num-files", help="New shards to create per detector")
|
||||
],
|
||||
events_per_file: Annotated[
|
||||
int, typer.Option("--events-per-file", help="nEvents passed to the executable")
|
||||
],
|
||||
gen: Annotated[
|
||||
str, typer.Option("--gen", help="Existing gen tag under raw/<kind>/, e.g. gen1")
|
||||
],
|
||||
num_files: Annotated[int, typer.Option("--num-files", help="New shards to create per detector")],
|
||||
events_per_file: Annotated[int, typer.Option("--events-per-file", help="nEvents passed to the executable")],
|
||||
gen: Annotated[str, typer.Option("--gen", help="Existing gen tag under raw/<kind>/, e.g. gen1")],
|
||||
energy_gev: Annotated[
|
||||
float | None,
|
||||
typer.Option(
|
||||
@@ -401,20 +349,12 @@ def make_root(
|
||||
"to name the dataset accordingly.",
|
||||
),
|
||||
] = None,
|
||||
kind: Annotated[
|
||||
str, typer.Option("--kind", help="steps | hits | ... (default: steps)")
|
||||
] = "steps",
|
||||
dataset_root: Annotated[
|
||||
Path, typer.Option("--dataset-root", help="Dataset root")
|
||||
] = _DATASET_ROOT_DEFAULT,
|
||||
jobs: Annotated[
|
||||
int, typer.Option("--jobs", "-j", help="Parallel simulation runs (default: 4)")
|
||||
] = 4,
|
||||
kind: Annotated[str, typer.Option("--kind", help="steps | hits | ... (default: steps)")] = "steps",
|
||||
dataset_root: Annotated[Path, typer.Option("--dataset-root", help="Dataset root")] = _DATASET_ROOT_DEFAULT,
|
||||
jobs: Annotated[int, typer.Option("--jobs", "-j", help="Parallel simulation runs (default: 4)")] = 4,
|
||||
execute: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--execute", help="Actually run jobs (default: dry run / print plan)"
|
||||
),
|
||||
typer.Option("--execute", help="Actually run jobs (default: dry run / print plan)"),
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Generate new ROOT shards via a minicalosim executable."""
|
||||
@@ -441,9 +381,7 @@ class OracleMethod(str, Enum):
|
||||
|
||||
@app.command("build-geometry-oracle")
|
||||
def build_geometry_oracle(
|
||||
data: Annotated[
|
||||
Path, typer.Argument(help="Steps parquet file or directory of steps files")
|
||||
],
|
||||
data: Annotated[Path, typer.Argument(help="Steps parquet file or directory of steps files")],
|
||||
out: Annotated[Path, typer.Option("--out", "-o", help="Output oracle .pkl path")],
|
||||
method: Annotated[
|
||||
OracleMethod,
|
||||
@@ -456,9 +394,7 @@ def build_geometry_oracle(
|
||||
),
|
||||
),
|
||||
] = OracleMethod.slab,
|
||||
k: Annotated[
|
||||
int, typer.Option("--k", help="Neighbours for the knn classifier")
|
||||
] = 1,
|
||||
k: Annotated[int, typer.Option("--k", help="Neighbours for the knn classifier")] = 1,
|
||||
subsample: Annotated[
|
||||
int,
|
||||
typer.Option("--subsample", help="Max reference points sampled from the data"),
|
||||
@@ -504,9 +440,7 @@ def build_geometry_oracle(
|
||||
def warm_cache(
|
||||
data: Annotated[
|
||||
Path,
|
||||
typer.Argument(
|
||||
help="Parquet file, directory, or .manifest — same as `giant train`'s"
|
||||
),
|
||||
typer.Argument(help="Parquet file, directory, or .manifest — same as `giant train`'s"),
|
||||
],
|
||||
val_fraction: Annotated[
|
||||
float,
|
||||
@@ -518,16 +452,13 @@ def warm_cache(
|
||||
] = 0.1,
|
||||
seed: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
"--seed", "-s", help="Must match the `giant train` run(s) to warm for"
|
||||
),
|
||||
typer.Option("--seed", "-s", help="Must match the `giant train` run(s) to warm for"),
|
||||
] = 0,
|
||||
particle_conditioning: Annotated[
|
||||
Conditioning,
|
||||
typer.Option(
|
||||
"--particle-conditioning",
|
||||
help="Must match the `giant train` run(s)' conditioning.particle.type "
|
||||
"to warm for",
|
||||
help="Must match the `giant train` run(s)' conditioning.particle.type to warm for",
|
||||
),
|
||||
] = Conditioning.physical,
|
||||
material_conditioning: Annotated[
|
||||
@@ -543,21 +474,14 @@ def warm_cache(
|
||||
bool,
|
||||
typer.Option(
|
||||
"--router/--no-router",
|
||||
help="Warm the process vocabulary too (only takes effect with "
|
||||
"--router-type process)",
|
||||
help="Warm the process vocabulary too (only takes effect with --router-type process)",
|
||||
),
|
||||
] = False,
|
||||
router_type: Annotated[
|
||||
str, typer.Option("--router-type", help="Router implementation name")
|
||||
] = "energy",
|
||||
n_experts: Annotated[
|
||||
int, typer.Option("--n-experts", help="Number of routed experts")
|
||||
] = 4,
|
||||
router_type: Annotated[str, typer.Option("--router-type", help="Router implementation name")] = "energy",
|
||||
n_experts: Annotated[int, typer.Option("--n-experts", help="Number of routed experts")] = 4,
|
||||
rebuild: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--rebuild", help="Ignore any existing sidecar and recompute every section"
|
||||
),
|
||||
typer.Option("--rebuild", help="Ignore any existing sidecar and recompute every section"),
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Precompute `giant train`'s setup-stage sidecar for `data` ahead of time.
|
||||
|
||||
@@ -38,9 +38,7 @@ def run_build_geometry_oracle(
|
||||
n_bins=n_bins,
|
||||
)
|
||||
|
||||
print(
|
||||
f"method: {method} reference points: {oracle.metadata['n_reference_points']:,}"
|
||||
)
|
||||
print(f"method: {method} reference points: {oracle.metadata['n_reference_points']:,}")
|
||||
print("classes (material, layer_id):")
|
||||
for material, layer_id in oracle.classes:
|
||||
print(f" {material:<12} layer_id={layer_id}")
|
||||
|
||||
+3
-10
@@ -154,9 +154,7 @@ def run_hparam_scan(
|
||||
wall_time_s = time.monotonic() - start
|
||||
|
||||
if metrics_path.exists():
|
||||
epochs_completed, final_val_loss, best_val_loss = final_metrics(
|
||||
metrics_path
|
||||
)
|
||||
epochs_completed, final_val_loss, best_val_loss = final_metrics(metrics_path)
|
||||
append_summary(
|
||||
summary_path,
|
||||
{
|
||||
@@ -171,11 +169,6 @@ def run_hparam_scan(
|
||||
"wall_time_s": round(wall_time_s, 1),
|
||||
},
|
||||
)
|
||||
print(
|
||||
f"[{i}/{len(runs)}] {name} — val_loss {final_val_loss:.4f} "
|
||||
f"({wall_time_s:.1f}s)"
|
||||
)
|
||||
print(f"[{i}/{len(runs)}] {name} — val_loss {final_val_loss:.4f} ({wall_time_s:.1f}s)")
|
||||
else:
|
||||
print(
|
||||
f"[{i}/{len(runs)}] {name} — no metrics.csv produced, check train.log"
|
||||
)
|
||||
print(f"[{i}/{len(runs)}] {name} — no metrics.csv produced, check train.log")
|
||||
|
||||
@@ -50,12 +50,8 @@ PREDICTED_RE = re.compile(
|
||||
r"^(?P<detector>[a-z0-9]+(?:_[a-z0-9]+)*)_10k_(?P<shard>\d+)"
|
||||
r"_predicted(?P<local>_local)?\.parquet$"
|
||||
)
|
||||
SHARD_RE = re.compile(
|
||||
r"^(?P<detector>[a-z0-9]+(?:_[a-z0-9]+)*)_10k_(?P<shard>\d+)\.(?P<ext>root|parquet)$"
|
||||
)
|
||||
LEGACY_PREDICTED_RE = re.compile(
|
||||
r"^pbwo4_10000events_hits_predicted(?P<local>_local)?\.parquet$"
|
||||
)
|
||||
SHARD_RE = re.compile(r"^(?P<detector>[a-z0-9]+(?:_[a-z0-9]+)*)_10k_(?P<shard>\d+)\.(?P<ext>root|parquet)$")
|
||||
LEGACY_PREDICTED_RE = re.compile(r"^pbwo4_10000events_hits_predicted(?P<local>_local)?\.parquet$")
|
||||
LEGACY_RE = re.compile(r"^pbwo4_10000events_hits\.(?P<ext>root|parquet)$")
|
||||
|
||||
|
||||
@@ -110,24 +106,9 @@ def plan_moves(src_root: Path) -> tuple[list[tuple[Path, Path]], list[Path]]:
|
||||
if m:
|
||||
detector, shard, ext = m["detector"], int(m["shard"]), m["ext"]
|
||||
if ext == "root":
|
||||
dst = (
|
||||
src_root
|
||||
/ "raw"
|
||||
/ "steps"
|
||||
/ GEN
|
||||
/ detector
|
||||
/ f"shard-{shard:03d}.root"
|
||||
)
|
||||
dst = src_root / "raw" / "steps" / GEN / detector / f"shard-{shard:03d}.root"
|
||||
else:
|
||||
dst = (
|
||||
src_root
|
||||
/ "processed"
|
||||
/ "steps"
|
||||
/ GEN
|
||||
/ SCHEMA
|
||||
/ detector
|
||||
/ f"shard-{shard:03d}.parquet"
|
||||
)
|
||||
dst = src_root / "processed" / "steps" / GEN / SCHEMA / detector / f"shard-{shard:03d}.parquet"
|
||||
moves.append((path, dst))
|
||||
continue
|
||||
|
||||
@@ -135,19 +116,9 @@ def plan_moves(src_root: Path) -> tuple[list[tuple[Path, Path]], list[Path]]:
|
||||
if m:
|
||||
ext = m["ext"]
|
||||
if ext == "root":
|
||||
dst = (
|
||||
src_root / "raw" / "hits" / LEGACY_GEN / "pbwo4" / "shard-000.root"
|
||||
)
|
||||
dst = src_root / "raw" / "hits" / LEGACY_GEN / "pbwo4" / "shard-000.root"
|
||||
else:
|
||||
dst = (
|
||||
src_root
|
||||
/ "processed"
|
||||
/ "hits"
|
||||
/ LEGACY_GEN
|
||||
/ LEGACY_SCHEMA
|
||||
/ "pbwo4"
|
||||
/ "shard-000.parquet"
|
||||
)
|
||||
dst = src_root / "processed" / "hits" / LEGACY_GEN / LEGACY_SCHEMA / "pbwo4" / "shard-000.parquet"
|
||||
moves.append((path, dst))
|
||||
continue
|
||||
|
||||
@@ -165,20 +136,9 @@ def plan_manifests(src_root: Path) -> dict[Path, list[str]]:
|
||||
for pool, shards in rules.items():
|
||||
manifest_path = manifest_dir / f"{pool}{MANIFEST_SUFFIX}"
|
||||
for shard in shards:
|
||||
dst = (
|
||||
src_root
|
||||
/ "processed"
|
||||
/ "steps"
|
||||
/ GEN
|
||||
/ SCHEMA
|
||||
/ detector
|
||||
/ f"shard-{shard:03d}.parquet"
|
||||
)
|
||||
dst = src_root / "processed" / "steps" / GEN / SCHEMA / detector / f"shard-{shard:03d}.parquet"
|
||||
manifests[manifest_path].append((shard, dst))
|
||||
return {
|
||||
k: [os.path.relpath(dst, start=k.parent) for _, dst in sorted(v)]
|
||||
for k, v in manifests.items()
|
||||
}
|
||||
return {k: [os.path.relpath(dst, start=k.parent) for _, dst in sorted(v)] for k, v in manifests.items()}
|
||||
|
||||
|
||||
def run_migration(root: str, execute: bool, copy: bool) -> None:
|
||||
|
||||
@@ -94,9 +94,7 @@ def _make_rollout(n: int, n_events: int, seed: int) -> pl.DataFrame:
|
||||
"post_dx": post_dir[:, 0],
|
||||
"post_dy": post_dir[:, 1],
|
||||
"post_dz": post_dir[:, 2],
|
||||
"edep": np.where(
|
||||
is_synthetic, np.where(reasons == "escaped", 0.0, pre_E), edep
|
||||
),
|
||||
"edep": np.where(is_synthetic, np.where(reasons == "escaped", 0.0, pre_E), edep),
|
||||
"step_length": np.where(is_synthetic, 0.0, step_length),
|
||||
"material": rng.choice(_MATERIALS, size=n),
|
||||
"layer_id": rng.integers(0, 30, size=n),
|
||||
@@ -163,9 +161,7 @@ def _make_reference(n: int, n_events: int, seed: int) -> pl.DataFrame:
|
||||
)
|
||||
|
||||
|
||||
def _time(
|
||||
spec_id: str, rollout: Path, reference: Path, shared: Path, out: Path
|
||||
) -> float:
|
||||
def _time(spec_id: str, rollout: Path, reference: Path, shared: Path, out: Path) -> float:
|
||||
t0 = time.perf_counter()
|
||||
compute_reduced(
|
||||
spec_id,
|
||||
|
||||
@@ -49,9 +49,7 @@ def latest_schema_tag(processed_gen_dir: Path) -> str | None:
|
||||
return best_tag
|
||||
|
||||
|
||||
def resolve_destination(
|
||||
root_file: Path, dataset_root: Path, schema_override: str | None
|
||||
) -> Path:
|
||||
def resolve_destination(root_file: Path, dataset_root: Path, schema_override: str | None) -> Path:
|
||||
"""Map raw/<kind>/<gen>/<detector>/<file>.root (relative to *dataset_root*)
|
||||
to processed/<kind>/<gen>/<schema>/<detector>/<file>.parquet.
|
||||
|
||||
@@ -66,12 +64,7 @@ def resolve_destination(
|
||||
raise DestinationError(f"{root_file} is not under dataset root {dataset_root}")
|
||||
|
||||
parts = rel.parts
|
||||
if (
|
||||
len(parts) != 5
|
||||
or parts[0] != "raw"
|
||||
or not GEN_RE.match(parts[2])
|
||||
or not parts[4].endswith(".root")
|
||||
):
|
||||
if len(parts) != 5 or parts[0] != "raw" or not GEN_RE.match(parts[2]) or not parts[4].endswith(".root"):
|
||||
raise DestinationError(
|
||||
f"{root_file} does not match raw/<kind>/<gen>/<detector>/<file>.root "
|
||||
f"under {dataset_root} (got relative path: {rel})"
|
||||
@@ -216,13 +209,7 @@ def run_parallel_job(
|
||||
print(f" {root_file}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
total_orphaned = sum(
|
||||
int(m.group(1))
|
||||
for _, _, stdout, _ in results
|
||||
for m in _ORPHAN_RE.finditer(stdout)
|
||||
)
|
||||
total_orphaned = sum(int(m.group(1)) for _, _, stdout, _ in results for m in _ORPHAN_RE.finditer(stdout))
|
||||
if total_orphaned:
|
||||
print(
|
||||
f"\n{total_orphaned} orphaned child track(s) dropped across {len(results)} file(s)."
|
||||
)
|
||||
print(f"\n{total_orphaned} orphaned child track(s) dropped across {len(results)} file(s).")
|
||||
print(f"\nAll {len(results)} conversion(s) completed.")
|
||||
|
||||
Reference in New Issue
Block a user