Add data-integrity guards against silent NaN/Inf propagation and races
CI / Lint (ruff check) (push) Successful in 33s
CI / Format (ruff format) (push) Successful in 34s
CI / Sync project version with tag (push) Has been skipped
CI / Type check (ty) (push) Successful in 32s
CI / Tests (push) Successful in 2m12s

- log_transform / _validate_unit_pre_dir now raise on non-finite input
  instead of letting a NaN row silently poison the persisted normalizer
  cache (norm < 1e-6 was always False for NaN, so the existing guard
  never caught it).
- encode_secondaries warns when a row's secondary energies cumulatively
  exceed e_sec, instead of silently saturating the overflowing slot's
  stick-breaking logit via the _EPS floor.
- EVENT_ID_FILE_STRIDE overflow now raises instead of silently colliding
  two files' event ids together (reintroducing train/val leakage).
- make_event_split(val_fraction=0.0) now actually holds out nothing,
  instead of always forcing at least 1 validation event.
- setup_cache.save() is now serialized with a flock, since two
  concurrent writers (a real scenario on this repo's shared
  portal/condor machines) could otherwise race and silently drop one
  writer's freshly-computed cache section.
- Documented (no behavior change) the pre_dir ≈ -ẑ antipodal rotation
  singularity in _rodrigues_axis, which is real but inherent to any
  single-valued local-frame convention.

Each fix has a regression test.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-03 13:47:29 +02:00
parent a4c0443e01
commit 74343d3e48
8 changed files with 247 additions and 15 deletions
+23 -3
View File
@@ -27,6 +27,26 @@ def event_id_offset(file_index: int) -> int:
return file_index * EVENT_ID_FILE_STRIDE
def _offset_event_id(raw_ids: np.ndarray, offset: int) -> np.ndarray:
"""Add this file's `event_id_offset`, after checking the raw ids fit in one stride block.
Without this check, a file whose own raw event_id numbering reaches
`EVENT_ID_FILE_STRIDE` (an unusually large job, or non-contiguous
numbering) would silently collide into the next file's offset block,
merging unrelated events across files — reintroducing exactly the
train/val event leakage this offset scheme exists to prevent.
"""
raw_ids = np.asarray(raw_ids, dtype=np.int64)
if raw_ids.size and int(raw_ids.max()) >= EVENT_ID_FILE_STRIDE:
raise ValueError(
f"event_id {int(raw_ids.max())} >= EVENT_ID_FILE_STRIDE "
f"({EVENT_ID_FILE_STRIDE}) — this file has a larger event_id "
"than the per-file offset scheme can support without colliding "
"with the next file's id block."
)
return raw_ids + offset
def _read_manifest(path: Path) -> list[Path]:
files = []
for line in path.read_text().splitlines():
@@ -98,7 +118,7 @@ def _df_to_dict(df: pd.DataFrame, offset: int = 0) -> dict[str, np.ndarray]:
has_sec_lists = "sec_E_list" in df.columns
d: dict[str, np.ndarray] = {
"event_id": df["event_id"].to_numpy().astype(np.int64) + offset,
"event_id": _offset_event_id(df["event_id"].to_numpy(), offset),
"pdg": df["pdg"].to_numpy(dtype=np.int32),
"pre_pos": df[["pre_x", "pre_y", "pre_z"]].to_numpy(dtype=np.float32),
"pre_E": df["pre_E"].to_numpy(dtype=np.float32),
@@ -142,7 +162,7 @@ def load_steps(path: str | Path, offset: int = 0) -> dict[str, np.ndarray]:
def load_event_ids(path: str | Path, offset: int = 0) -> np.ndarray:
"""Read only the event_id column — cheap scan for split assignment."""
ids = pd.read_parquet(path, columns=["event_id"])["event_id"].to_numpy()
return ids.astype(np.int64) + offset
return _offset_event_id(ids, offset)
def iter_file_chunks(
@@ -173,7 +193,7 @@ _COND_COLS = [
def _cond_df_to_dict(df: pd.DataFrame, offset: int = 0) -> dict[str, np.ndarray]:
return {
"event_id": df["event_id"].to_numpy().astype(np.int64) + offset,
"event_id": _offset_event_id(df["event_id"].to_numpy(), offset),
"pdg": df["pdg"].to_numpy(dtype=np.int32),
"pre_pos": df[["pre_x", "pre_y", "pre_z"]].to_numpy(dtype=np.float32),
"pre_E": df["pre_E"].to_numpy(dtype=np.float32),