Bump ruff line-length to 120 and reformat
CI / Lint (ruff check) (push) Successful in 31s
CI / Format (ruff format) (push) Successful in 32s
CI / Sync project version with tag (push) Has been skipped
CI / Lint (ruff check) (pull_request) Successful in 26s
CI / Type check (ty) (push) Successful in 29s
CI / Format (ruff format) (pull_request) Successful in 33s
CI / Sync project version with tag (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 35s
CI / Tests (pull_request) Successful in 3m47s
CI / Tests (push) Successful in 3m55s

Rejoins lines that only wrapped because they exceeded the old 88-char
limit; ruff check and the full test suite (725 passed) are unaffected.
This commit is contained in:
2026-08-12 13:33:09 +02:00
parent 9ce7b32324
commit 55332db67a
66 changed files with 757 additions and 2413 deletions
+15 -49
View File
@@ -82,9 +82,7 @@ def _max_index(parent: Path, pattern: re.Pattern) -> int:
def _git_user_name() -> str | None:
try:
out = subprocess.run(
["git", "config", "user.name"], capture_output=True, text=True, timeout=2
)
out = subprocess.run(["git", "config", "user.name"], capture_output=True, text=True, timeout=2)
except (OSError, subprocess.SubprocessError):
# OSError (e.g. git not on PATH) and subprocess.SubprocessError
# (e.g. TimeoutExpired) are unrelated hierarchies — TimeoutExpired
@@ -142,9 +140,7 @@ def plan_bump_schema(
raw_gen_dir = root / "raw" / kind / gen_tag
processed_gen_dir = root / "processed" / kind / gen_tag
if not raw_gen_dir.is_dir() and not processed_gen_dir.is_dir():
raise SystemExit(
f"error: {gen_tag} doesn't exist yet for kind={kind} — run bump-gen first"
)
raise SystemExit(f"error: {gen_tag} doesn't exist yet for kind={kind} — run bump-gen first")
if target is not None:
if not SCHEMA_RE.match(target):
raise SystemExit(f"error: --to must look like 'schemaN', got {target!r}")
@@ -154,9 +150,7 @@ def plan_bump_schema(
schema_tag = f"schema{next_schema}"
new_dirs = [processed_gen_dir / schema_tag]
by_suffix = f" ({by})" if by else ""
log_line = (
f"- `{gen_tag}`/`{schema_tag}` (kind={kind}) — {date}{reason}{by_suffix}"
)
log_line = f"- `{gen_tag}`/`{schema_tag}` (kind={kind}) — {date}{reason}{by_suffix}"
return new_dirs, log_line
@@ -212,9 +206,7 @@ def _manifest_referenced_files(pools_root: Path) -> set[Path]:
return referenced
def _referenced_root_count(
raw_gen_dir: Path, processed_gen_dir: Path
) -> tuple[int, int]:
def _referenced_root_count(raw_gen_dir: Path, processed_gen_dir: Path) -> tuple[int, int]:
"""(total .root files, count with a same-named .parquet under any schema) for one gen."""
if not raw_gen_dir.is_dir():
return 0, 0
@@ -243,9 +235,7 @@ def _referenced_root_count(
return total, referenced
def _referenced_parquet_count(
schema_dir: Path, manifest_referenced: set[Path]
) -> tuple[int, int]:
def _referenced_parquet_count(schema_dir: Path, manifest_referenced: set[Path]) -> tuple[int, int]:
"""(total .parquet files, count listed in at least one manifest) for one schema dir."""
if not schema_dir.is_dir():
return 0, 0
@@ -342,11 +332,7 @@ def print_status(root: Path) -> None:
grand_files = 0
for kind_dir in sorted(p for p in raw_root.iterdir() if p.is_dir()):
kind = kind_dir.name
gens = sorted(
int(m.group(1))
for m in (GEN_RE.match(p.name) for p in kind_dir.iterdir() if p.is_dir())
if m
)
gens = sorted(int(m.group(1)) for m in (GEN_RE.match(p.name) for p in kind_dir.iterdir() if p.is_dir()) if m)
print(_colorize(f"{kind}/", "kind"))
kind_total = 0
kind_files = 0
@@ -359,25 +345,18 @@ def print_status(root: Path) -> None:
schemas = sorted(
int(m.group(1))
for m in (
SCHEMA_RE.match(p.name)
for p in (schema_dir.iterdir() if schema_dir.is_dir() else [])
if p.is_dir()
SCHEMA_RE.match(p.name) for p in (schema_dir.iterdir() if schema_dir.is_dir() else []) if p.is_dir()
)
if m
)
schema_sizes = {s: _du(schema_dir / f"schema{s}") for s in schemas}
schema_counts = {
s: _referenced_parquet_count(
schema_dir / f"schema{s}", manifest_referenced
)
for s in schemas
s: _referenced_parquet_count(schema_dir / f"schema{s}", manifest_referenced) for s in schemas
}
processed_size = sum(schema_sizes.values())
processed_files = sum(c[0] for c in schema_counts.values())
processed_referenced = sum(c[1] for c in schema_counts.values())
raw_files, raw_referenced = _referenced_root_count(
raw_gen_dir, processed_gen_dir
)
raw_files, raw_referenced = _referenced_root_count(raw_gen_dir, processed_gen_dir)
gen_total = raw_size + processed_size
gen_files = raw_files + processed_files
kind_total += gen_total
@@ -425,9 +404,7 @@ def print_status(root: Path) -> None:
print(_reason_line(schema_reason, indent=4))
else:
print(_colorize(" (none)", "schema"))
print(
_row(f"{kind} total", kind_total, indent=1, level="gen", count=kind_files)
)
print(_row(f"{kind} total", kind_total, indent=1, level="gen", count=kind_files))
print()
grand_total += kind_total
grand_files += kind_files
@@ -526,13 +503,8 @@ def plan_update_manifest(
return result, missing
def apply_update_manifest(
manifest_path: Path, lines: list[tuple[str, str | None]]
) -> None:
out = [
replacement if replacement is not None else original
for original, replacement in lines
]
def apply_update_manifest(manifest_path: Path, lines: list[tuple[str, str | None]]) -> None:
out = [replacement if replacement is not None else original for original, replacement in lines]
manifest_path.write_text("\n".join(out) + "\n")
@@ -552,9 +524,7 @@ def _resolve_manifest_files(manifest_path: Path) -> list[Path]:
return files
def plan_create_manifest(
output_path: Path, parquet_files: list[Path]
) -> tuple[list[str], list[Path], list[Path]]:
def plan_create_manifest(output_path: Path, parquet_files: list[Path]) -> tuple[list[str], list[Path], list[Path]]:
"""Return (relative_lines, missing_files, resolved_abs_paths)."""
manifest_dir = output_path.resolve().parent
lines: list[str] = []
@@ -569,9 +539,7 @@ def plan_create_manifest(
return lines, missing, resolved
def check_holdout_overlap(
output_path: Path, resolved_new_files: list[Path]
) -> list[tuple[str, Path]]:
def check_holdout_overlap(output_path: Path, resolved_new_files: list[Path]) -> list[tuple[str, Path]]:
"""Return (other_manifest_name, file) pairs where new files clash with existing manifests.
The check is triggered when output_path is (or will be) holdout.manifest, or when a
@@ -641,9 +609,7 @@ def _run_bump(
if gen is None:
new_dirs, log_line = plan_bump_gen(root_path, kind, reason, by, date, to)
else:
new_dirs, log_line = plan_bump_schema(
root_path, kind, gen, reason, by, date, to
)
new_dirs, log_line = plan_bump_schema(root_path, kind, gen, reason, by, date, to)
print(f"=== {'EXECUTING' if execute else 'DRY RUN'} ===")
print("new directories:")
+5 -16
View File
@@ -84,19 +84,12 @@ def main() -> int:
mode = model_config.get("mode", "flow")
routed = bool((model_config.get("router") or {}).get("enabled"))
print(f"checkpoint: {args.checkpoint}")
print(
f" mode={mode!r} conditioning={model_config.get('conditioning')!r} "
f"routed={routed} ema={args.ema}"
)
print(f" mode={mode!r} conditioning={model_config.get('conditioning')!r} routed={routed} ema={args.ema}")
stage1_key = "model_ema" if args.ema and "model_ema" in ckpt else "model"
stage2_key = (
"sec_decoder_ema" if args.ema and "sec_decoder_ema" in ckpt else "sec_decoder"
)
stage2_key = "sec_decoder_ema" if args.ema and "sec_decoder_ema" in ckpt else "sec_decoder"
if args.ema and stage1_key == "model":
print(
" warning: --ema requested but no model_ema in checkpoint, using raw weights"
)
print(" warning: --ema requested but no model_ema in checkpoint, using raw weights")
# --- old side: the frozen v0.2 snapshot, loaded with the checkpoint's own weights ---
old_stage1, old_stage2 = legacy.build_models(model_config)
@@ -119,9 +112,7 @@ def main() -> int:
print("PASS (construction only, routed checkpoint)")
return 0
remapped1, remapped2 = net.migrate_legacy_state_dict(
ckpt[stage1_key], ckpt[stage2_key]
)
remapped1, remapped2 = net.migrate_legacy_state_dict(ckpt[stage1_key], ckpt[stage2_key])
missing1, unexpected1 = new_stage1.load_state_dict(remapped1, strict=True)
missing2, unexpected2 = new_stage2.load_state_dict(remapped2, strict=True)
if missing1 or unexpected1 or missing2 or unexpected2:
@@ -132,9 +123,7 @@ def main() -> int:
new_stage1.eval()
new_stage2.eval()
cond_cont, cond_cat, x1, x2, t, z1, z2 = _random_batch(
model_config, args.batch, args.seed
)
cond_cont, cond_cat, x1, x2, t, z1, z2 = _random_batch(model_config, args.batch, args.seed)
ok = True
with torch.no_grad():
+7 -31
View File
@@ -64,9 +64,7 @@ def parse_detector_spec(spec: str) -> tuple[str, str | None]:
if ":" in spec:
label, config = spec.split(":", 1)
if not label or not config:
raise PlanError(
f"invalid --detector spec {spec!r}: expected NAME or NAME:CONFIG"
)
raise PlanError(f"invalid --detector spec {spec!r}: expected NAME or NAME:CONFIG")
return label, config
return spec, None
@@ -94,9 +92,7 @@ def plan_jobs(
raise PlanError(f"--gen must look like 'genN', got {gen!r}")
gen_dir = dataset_root / "raw" / kind / gen
if not gen_dir.is_dir():
raise PlanError(
f"{gen_dir} doesn't exist — run bump_dataset_version.py bump-gen first"
)
raise PlanError(f"{gen_dir} doesn't exist — run bump_dataset_version.py bump-gen first")
jobs = []
for spec in detector_specs:
@@ -124,9 +120,7 @@ def job_seed(kind: str, gen: str, job: SimJob, energy_gev: float | None) -> int:
return zlib.crc32(key.encode()) & 0x7FFFFFFF
def build_cmd(
executable: Path, job: SimJob, events_per_file: int, energy_gev: float | None
) -> list[str]:
def build_cmd(executable: Path, job: SimJob, events_per_file: int, energy_gev: float | None) -> list[str]:
"""minicalosim executables take positional `[configName] nEvents [energy_GeV]`."""
cmd = [str(executable)]
if job.config:
@@ -147,10 +141,7 @@ def run_job(
gen: str,
tmp_root: Path,
) -> JobResult:
workdir = (
tmp_root
/ f"{kind}-{gen}-{job.detector}-{job.shard_index:03d}-{uuid.uuid4().hex[:8]}"
)
workdir = tmp_root / f"{kind}-{gen}-{job.detector}-{job.shard_index:03d}-{uuid.uuid4().hex[:8]}"
workdir.mkdir(parents=True)
cmd = build_cmd(executable, job, events_per_file, energy_gev)
@@ -174,20 +165,12 @@ def run_job(
job,
False,
None,
f"expected exactly one .root output in {workdir}, found {len(produced)}: "
f"{[p.name for p in produced]}",
f"expected exactly one .root output in {workdir}, found {len(produced)}: {[p.name for p in produced]}",
result.stdout,
result.stderr,
)
dest = (
dataset_root
/ "raw"
/ kind
/ gen
/ job.detector
/ f"shard-{job.shard_index:03d}.root"
)
dest = dataset_root / "raw" / kind / gen / job.detector / f"shard-{job.shard_index:03d}.root"
if dest.exists():
return JobResult(
job,
@@ -279,14 +262,7 @@ def run_make_root(
print(f"executable: {executable}")
for job in planned_jobs:
cmd = build_cmd(executable, job, events_per_file, energy_gev)
dest = (
dataset_root_path
/ "raw"
/ kind
/ gen
/ job.detector
/ f"shard-{job.shard_index:03d}.root"
)
dest = dataset_root_path / "raw" / kind / gen / job.detector / f"shard-{job.shard_index:03d}.root"
seed = job_seed(kind, gen, job, energy_gev)
print(f" MINICALOSIM_SEED={seed} {' '.join(cmd)} -> {dest}")
+40 -116
View File
@@ -80,8 +80,7 @@ def convert(
typer.Option(
"--output",
"-o",
help="Output Parquet file (default: <input>.parquet). Only valid "
"with a single input file and --jobs 1.",
help="Output Parquet file (default: <input>.parquet). Only valid with a single input file and --jobs 1.",
),
] = None,
batch_size: Annotated[
@@ -91,9 +90,7 @@ def convert(
help="Uproot read batch size, e.g. '100 MB' or '500000' (rows)",
),
] = "100 MB",
tree: Annotated[
str, typer.Option("--tree", help="Tree name inside the ROOT file")
] = "Steps",
tree: Annotated[str, typer.Option("--tree", help="Tree name inside the ROOT file")] = "Steps",
compression: Annotated[
Compression, typer.Option("--compression", help="Parquet compression codec")
] = Compression.snappy,
@@ -129,15 +126,11 @@ def convert(
raise typer.Exit(1)
_warn_if_exceeds_shared_quota(jobs, "--jobs")
compression_value = (
"uncompressed" if compression is Compression.none else compression.value
)
compression_value = "uncompressed" if compression is Compression.none else compression.value
if jobs == 1:
if output is not None and len(root_files) > 1:
typer.echo(
"error: --output can only be used with a single input file", err=True
)
typer.echo("error: --output can only be used with a single input file", err=True)
raise typer.Exit(1)
total_orphaned = 0
for root_file in root_files:
@@ -150,10 +143,7 @@ def convert(
)
total_orphaned += n_orphaned
if total_orphaned:
typer.echo(
f"\n{total_orphaned} orphaned child track(s) dropped across "
f"{len(root_files)} file(s)."
)
typer.echo(f"\n{total_orphaned} orphaned child track(s) dropped across {len(root_files)} file(s).")
return
if output is not None:
@@ -176,9 +166,7 @@ def convert(
@app.command()
def migrate(
root: Annotated[
Path, typer.Argument(help="Dataset root to migrate in place")
] = _DATASET_ROOT_DEFAULT,
root: Annotated[Path, typer.Argument(help="Dataset root to migrate in place")] = _DATASET_ROOT_DEFAULT,
execute: Annotated[
bool,
typer.Option(
@@ -190,8 +178,7 @@ def migrate(
bool,
typer.Option(
"--copy",
help="Copy instead of move, leaving the originals in place "
"(e.g. if another process is still reading them)",
help="Copy instead of move, leaving the originals in place (e.g. if another process is still reading them)",
),
] = False,
) -> None:
@@ -202,12 +189,8 @@ def migrate(
@app.command("bump-gen")
def bump_gen(
reason: Annotated[str, typer.Option("--reason", help="Why this gen exists")],
kind: Annotated[
str, typer.Option("--kind", help="steps | hits | ... (default: steps)")
] = "steps",
by: Annotated[
Optional[str], typer.Option("--by", help="Attribution (default: git user.name)")
] = None,
kind: Annotated[str, typer.Option("--kind", help="steps | hits | ... (default: steps)")] = "steps",
by: Annotated[Optional[str], typer.Option("--by", help="Attribution (default: git user.name)")] = None,
date: Annotated[
Optional[str],
typer.Option("--date", help="Override date (default: today, ISO)"),
@@ -220,12 +203,8 @@ def bump_gen(
help="Target gen tag (default: one past the current highest)",
),
] = None,
execute: Annotated[
bool, typer.Option("--execute", help="Apply (default: dry run)")
] = False,
root: Annotated[
Path, typer.Option("--root", help="Dataset root")
] = _DATASET_ROOT_DEFAULT,
execute: Annotated[bool, typer.Option("--execute", help="Apply (default: dry run)")] = False,
root: Annotated[Path, typer.Option("--root", help="Dataset root")] = _DATASET_ROOT_DEFAULT,
) -> None:
"""Cut a new raw generation."""
run_bump_gen(
@@ -243,12 +222,8 @@ def bump_gen(
def bump_schema(
gen: Annotated[str, typer.Option("--gen", help="Existing gen tag, e.g. gen1")],
reason: Annotated[str, typer.Option("--reason", help="Why this schema exists")],
kind: Annotated[
str, typer.Option("--kind", help="steps | hits | ... (default: steps)")
] = "steps",
by: Annotated[
Optional[str], typer.Option("--by", help="Attribution (default: git user.name)")
] = None,
kind: Annotated[str, typer.Option("--kind", help="steps | hits | ... (default: steps)")] = "steps",
by: Annotated[Optional[str], typer.Option("--by", help="Attribution (default: git user.name)")] = None,
date: Annotated[
Optional[str],
typer.Option("--date", help="Override date (default: today, ISO)"),
@@ -261,12 +236,8 @@ def bump_schema(
help="Target schema tag (default: one past the current highest)",
),
] = None,
execute: Annotated[
bool, typer.Option("--execute", help="Apply (default: dry run)")
] = False,
root: Annotated[
Path, typer.Option("--root", help="Dataset root")
] = _DATASET_ROOT_DEFAULT,
execute: Annotated[bool, typer.Option("--execute", help="Apply (default: dry run)")] = False,
root: Annotated[Path, typer.Option("--root", help="Dataset root")] = _DATASET_ROOT_DEFAULT,
) -> None:
"""Cut a new schema within a gen."""
run_bump_schema(
@@ -283,9 +254,7 @@ def bump_schema(
@app.command()
def status(
root: Annotated[
Path, typer.Option("--root", help="Dataset root")
] = _DATASET_ROOT_DEFAULT,
root: Annotated[Path, typer.Option("--root", help="Dataset root")] = _DATASET_ROOT_DEFAULT,
) -> None:
"""List existing gens/schemas per kind."""
run_status(str(root))
@@ -293,9 +262,7 @@ def status(
@app.command("update-manifest")
def update_manifest(
manifests: Annotated[
list[Path], typer.Argument(help="One or more .manifest files to update")
],
manifests: Annotated[list[Path], typer.Argument(help="One or more .manifest files to update")],
schema: Annotated[
Optional[str],
typer.Option(
@@ -306,9 +273,7 @@ def update_manifest(
] = None,
gen: Annotated[
Optional[str],
typer.Option(
"--gen", metavar="genN", help="Target gen tag (default: keep existing gen)"
),
typer.Option("--gen", metavar="genN", help="Target gen tag (default: keep existing gen)"),
] = None,
execute: Annotated[
bool,
@@ -316,9 +281,7 @@ def update_manifest(
] = False,
) -> None:
"""Repoint manifest(s) to a new gen and/or schema, verifying all target files exist."""
run_update_manifest(
[str(m) for m in manifests], schema=schema, execute=execute, gen=gen
)
run_update_manifest([str(m) for m in manifests], schema=schema, execute=execute, gen=gen)
@app.command("create-manifest")
@@ -333,22 +296,15 @@ def create_manifest(
typer.Option(
"--pool",
metavar="DETECTOR",
help="Detector name; combined with --type and --root to form "
"<root>/pools/<detector>/<type>.manifest",
help="Detector name; combined with --type and --root to form <root>/pools/<detector>/<type>.manifest",
),
] = None,
type_: Annotated[
Optional[PoolType],
typer.Option(
"--type", help="Pool type — full, holdout, or dev (required with --pool)"
),
typer.Option("--type", help="Pool type — full, holdout, or dev (required with --pool)"),
] = None,
root: Annotated[
Path, typer.Option("--root", help="Dataset root (used with --pool)")
] = _DATASET_ROOT_DEFAULT,
execute: Annotated[
bool, typer.Option("--execute", help="Write the manifest (default: dry run)")
] = False,
root: Annotated[Path, typer.Option("--root", help="Dataset root (used with --pool)")] = _DATASET_ROOT_DEFAULT,
execute: Annotated[bool, typer.Option("--execute", help="Write the manifest (default: dry run)")] = False,
force: Annotated[
bool,
typer.Option("--force", help="Overwrite the manifest if it already exists"),
@@ -368,9 +324,7 @@ def create_manifest(
@app.command("make-root")
def make_root(
executable: Annotated[
Path, typer.Option("--executable", help="Built minicalosim run_* executable")
],
executable: Annotated[Path, typer.Option("--executable", help="Built minicalosim run_* executable")],
detector: Annotated[
list[str],
typer.Option(
@@ -382,15 +336,9 @@ def make_root(
"Repeatable.",
),
],
num_files: Annotated[
int, typer.Option("--num-files", help="New shards to create per detector")
],
events_per_file: Annotated[
int, typer.Option("--events-per-file", help="nEvents passed to the executable")
],
gen: Annotated[
str, typer.Option("--gen", help="Existing gen tag under raw/<kind>/, e.g. gen1")
],
num_files: Annotated[int, typer.Option("--num-files", help="New shards to create per detector")],
events_per_file: Annotated[int, typer.Option("--events-per-file", help="nEvents passed to the executable")],
gen: Annotated[str, typer.Option("--gen", help="Existing gen tag under raw/<kind>/, e.g. gen1")],
energy_gev: Annotated[
float | None,
typer.Option(
@@ -401,20 +349,12 @@ def make_root(
"to name the dataset accordingly.",
),
] = None,
kind: Annotated[
str, typer.Option("--kind", help="steps | hits | ... (default: steps)")
] = "steps",
dataset_root: Annotated[
Path, typer.Option("--dataset-root", help="Dataset root")
] = _DATASET_ROOT_DEFAULT,
jobs: Annotated[
int, typer.Option("--jobs", "-j", help="Parallel simulation runs (default: 4)")
] = 4,
kind: Annotated[str, typer.Option("--kind", help="steps | hits | ... (default: steps)")] = "steps",
dataset_root: Annotated[Path, typer.Option("--dataset-root", help="Dataset root")] = _DATASET_ROOT_DEFAULT,
jobs: Annotated[int, typer.Option("--jobs", "-j", help="Parallel simulation runs (default: 4)")] = 4,
execute: Annotated[
bool,
typer.Option(
"--execute", help="Actually run jobs (default: dry run / print plan)"
),
typer.Option("--execute", help="Actually run jobs (default: dry run / print plan)"),
] = False,
) -> None:
"""Generate new ROOT shards via a minicalosim executable."""
@@ -441,9 +381,7 @@ class OracleMethod(str, Enum):
@app.command("build-geometry-oracle")
def build_geometry_oracle(
data: Annotated[
Path, typer.Argument(help="Steps parquet file or directory of steps files")
],
data: Annotated[Path, typer.Argument(help="Steps parquet file or directory of steps files")],
out: Annotated[Path, typer.Option("--out", "-o", help="Output oracle .pkl path")],
method: Annotated[
OracleMethod,
@@ -456,9 +394,7 @@ def build_geometry_oracle(
),
),
] = OracleMethod.slab,
k: Annotated[
int, typer.Option("--k", help="Neighbours for the knn classifier")
] = 1,
k: Annotated[int, typer.Option("--k", help="Neighbours for the knn classifier")] = 1,
subsample: Annotated[
int,
typer.Option("--subsample", help="Max reference points sampled from the data"),
@@ -504,9 +440,7 @@ def build_geometry_oracle(
def warm_cache(
data: Annotated[
Path,
typer.Argument(
help="Parquet file, directory, or .manifest — same as `giant train`'s"
),
typer.Argument(help="Parquet file, directory, or .manifest — same as `giant train`'s"),
],
val_fraction: Annotated[
float,
@@ -518,16 +452,13 @@ def warm_cache(
] = 0.1,
seed: Annotated[
int,
typer.Option(
"--seed", "-s", help="Must match the `giant train` run(s) to warm for"
),
typer.Option("--seed", "-s", help="Must match the `giant train` run(s) to warm for"),
] = 0,
particle_conditioning: Annotated[
Conditioning,
typer.Option(
"--particle-conditioning",
help="Must match the `giant train` run(s)' conditioning.particle.type "
"to warm for",
help="Must match the `giant train` run(s)' conditioning.particle.type to warm for",
),
] = Conditioning.physical,
material_conditioning: Annotated[
@@ -543,21 +474,14 @@ def warm_cache(
bool,
typer.Option(
"--router/--no-router",
help="Warm the process vocabulary too (only takes effect with "
"--router-type process)",
help="Warm the process vocabulary too (only takes effect with --router-type process)",
),
] = False,
router_type: Annotated[
str, typer.Option("--router-type", help="Router implementation name")
] = "energy",
n_experts: Annotated[
int, typer.Option("--n-experts", help="Number of routed experts")
] = 4,
router_type: Annotated[str, typer.Option("--router-type", help="Router implementation name")] = "energy",
n_experts: Annotated[int, typer.Option("--n-experts", help="Number of routed experts")] = 4,
rebuild: Annotated[
bool,
typer.Option(
"--rebuild", help="Ignore any existing sidecar and recompute every section"
),
typer.Option("--rebuild", help="Ignore any existing sidecar and recompute every section"),
] = False,
) -> None:
"""Precompute `giant train`'s setup-stage sidecar for `data` ahead of time.
+1 -3
View File
@@ -38,9 +38,7 @@ def run_build_geometry_oracle(
n_bins=n_bins,
)
print(
f"method: {method} reference points: {oracle.metadata['n_reference_points']:,}"
)
print(f"method: {method} reference points: {oracle.metadata['n_reference_points']:,}")
print("classes (material, layer_id):")
for material, layer_id in oracle.classes:
print(f" {material:<12} layer_id={layer_id}")
+3 -10
View File
@@ -154,9 +154,7 @@ def run_hparam_scan(
wall_time_s = time.monotonic() - start
if metrics_path.exists():
epochs_completed, final_val_loss, best_val_loss = final_metrics(
metrics_path
)
epochs_completed, final_val_loss, best_val_loss = final_metrics(metrics_path)
append_summary(
summary_path,
{
@@ -171,11 +169,6 @@ def run_hparam_scan(
"wall_time_s": round(wall_time_s, 1),
},
)
print(
f"[{i}/{len(runs)}] {name} — val_loss {final_val_loss:.4f} "
f"({wall_time_s:.1f}s)"
)
print(f"[{i}/{len(runs)}] {name} — val_loss {final_val_loss:.4f} ({wall_time_s:.1f}s)")
else:
print(
f"[{i}/{len(runs)}] {name} — no metrics.csv produced, check train.log"
)
print(f"[{i}/{len(runs)}] {name} — no metrics.csv produced, check train.log")
+8 -48
View File
@@ -50,12 +50,8 @@ PREDICTED_RE = re.compile(
r"^(?P<detector>[a-z0-9]+(?:_[a-z0-9]+)*)_10k_(?P<shard>\d+)"
r"_predicted(?P<local>_local)?\.parquet$"
)
SHARD_RE = re.compile(
r"^(?P<detector>[a-z0-9]+(?:_[a-z0-9]+)*)_10k_(?P<shard>\d+)\.(?P<ext>root|parquet)$"
)
LEGACY_PREDICTED_RE = re.compile(
r"^pbwo4_10000events_hits_predicted(?P<local>_local)?\.parquet$"
)
SHARD_RE = re.compile(r"^(?P<detector>[a-z0-9]+(?:_[a-z0-9]+)*)_10k_(?P<shard>\d+)\.(?P<ext>root|parquet)$")
LEGACY_PREDICTED_RE = re.compile(r"^pbwo4_10000events_hits_predicted(?P<local>_local)?\.parquet$")
LEGACY_RE = re.compile(r"^pbwo4_10000events_hits\.(?P<ext>root|parquet)$")
@@ -110,24 +106,9 @@ def plan_moves(src_root: Path) -> tuple[list[tuple[Path, Path]], list[Path]]:
if m:
detector, shard, ext = m["detector"], int(m["shard"]), m["ext"]
if ext == "root":
dst = (
src_root
/ "raw"
/ "steps"
/ GEN
/ detector
/ f"shard-{shard:03d}.root"
)
dst = src_root / "raw" / "steps" / GEN / detector / f"shard-{shard:03d}.root"
else:
dst = (
src_root
/ "processed"
/ "steps"
/ GEN
/ SCHEMA
/ detector
/ f"shard-{shard:03d}.parquet"
)
dst = src_root / "processed" / "steps" / GEN / SCHEMA / detector / f"shard-{shard:03d}.parquet"
moves.append((path, dst))
continue
@@ -135,19 +116,9 @@ def plan_moves(src_root: Path) -> tuple[list[tuple[Path, Path]], list[Path]]:
if m:
ext = m["ext"]
if ext == "root":
dst = (
src_root / "raw" / "hits" / LEGACY_GEN / "pbwo4" / "shard-000.root"
)
dst = src_root / "raw" / "hits" / LEGACY_GEN / "pbwo4" / "shard-000.root"
else:
dst = (
src_root
/ "processed"
/ "hits"
/ LEGACY_GEN
/ LEGACY_SCHEMA
/ "pbwo4"
/ "shard-000.parquet"
)
dst = src_root / "processed" / "hits" / LEGACY_GEN / LEGACY_SCHEMA / "pbwo4" / "shard-000.parquet"
moves.append((path, dst))
continue
@@ -165,20 +136,9 @@ def plan_manifests(src_root: Path) -> dict[Path, list[str]]:
for pool, shards in rules.items():
manifest_path = manifest_dir / f"{pool}{MANIFEST_SUFFIX}"
for shard in shards:
dst = (
src_root
/ "processed"
/ "steps"
/ GEN
/ SCHEMA
/ detector
/ f"shard-{shard:03d}.parquet"
)
dst = src_root / "processed" / "steps" / GEN / SCHEMA / detector / f"shard-{shard:03d}.parquet"
manifests[manifest_path].append((shard, dst))
return {
k: [os.path.relpath(dst, start=k.parent) for _, dst in sorted(v)]
for k, v in manifests.items()
}
return {k: [os.path.relpath(dst, start=k.parent) for _, dst in sorted(v)] for k, v in manifests.items()}
def run_migration(root: str, execute: bool, copy: bool) -> None:
+2 -6
View File
@@ -94,9 +94,7 @@ def _make_rollout(n: int, n_events: int, seed: int) -> pl.DataFrame:
"post_dx": post_dir[:, 0],
"post_dy": post_dir[:, 1],
"post_dz": post_dir[:, 2],
"edep": np.where(
is_synthetic, np.where(reasons == "escaped", 0.0, pre_E), edep
),
"edep": np.where(is_synthetic, np.where(reasons == "escaped", 0.0, pre_E), edep),
"step_length": np.where(is_synthetic, 0.0, step_length),
"material": rng.choice(_MATERIALS, size=n),
"layer_id": rng.integers(0, 30, size=n),
@@ -163,9 +161,7 @@ def _make_reference(n: int, n_events: int, seed: int) -> pl.DataFrame:
)
def _time(
spec_id: str, rollout: Path, reference: Path, shared: Path, out: Path
) -> float:
def _time(spec_id: str, rollout: Path, reference: Path, shared: Path, out: Path) -> float:
t0 = time.perf_counter()
compute_reduced(
spec_id,
+4 -17
View File
@@ -49,9 +49,7 @@ def latest_schema_tag(processed_gen_dir: Path) -> str | None:
return best_tag
def resolve_destination(
root_file: Path, dataset_root: Path, schema_override: str | None
) -> Path:
def resolve_destination(root_file: Path, dataset_root: Path, schema_override: str | None) -> Path:
"""Map raw/<kind>/<gen>/<detector>/<file>.root (relative to *dataset_root*)
to processed/<kind>/<gen>/<schema>/<detector>/<file>.parquet.
@@ -66,12 +64,7 @@ def resolve_destination(
raise DestinationError(f"{root_file} is not under dataset root {dataset_root}")
parts = rel.parts
if (
len(parts) != 5
or parts[0] != "raw"
or not GEN_RE.match(parts[2])
or not parts[4].endswith(".root")
):
if len(parts) != 5 or parts[0] != "raw" or not GEN_RE.match(parts[2]) or not parts[4].endswith(".root"):
raise DestinationError(
f"{root_file} does not match raw/<kind>/<gen>/<detector>/<file>.root "
f"under {dataset_root} (got relative path: {rel})"
@@ -216,13 +209,7 @@ def run_parallel_job(
print(f" {root_file}", file=sys.stderr)
raise SystemExit(1)
total_orphaned = sum(
int(m.group(1))
for _, _, stdout, _ in results
for m in _ORPHAN_RE.finditer(stdout)
)
total_orphaned = sum(int(m.group(1)) for _, _, stdout, _ in results for m in _ORPHAN_RE.finditer(stdout))
if total_orphaned:
print(
f"\n{total_orphaned} orphaned child track(s) dropped across {len(results)} file(s)."
)
print(f"\n{total_orphaned} orphaned child track(s) dropped across {len(results)} file(s).")
print(f"\nAll {len(results)} conversion(s) completed.")