Add data-integrity guards against silent NaN/Inf propagation and races
- log_transform / _validate_unit_pre_dir now raise on non-finite input instead of letting a NaN row silently poison the persisted normalizer cache (norm < 1e-6 was always False for NaN, so the existing guard never caught it). - encode_secondaries warns when a row's secondary energies cumulatively exceed e_sec, instead of silently saturating the overflowing slot's stick-breaking logit via the _EPS floor. - EVENT_ID_FILE_STRIDE overflow now raises instead of silently colliding two files' event ids together (reintroducing train/val leakage). - make_event_split(val_fraction=0.0) now actually holds out nothing, instead of always forcing at least 1 validation event. - setup_cache.save() is now serialized with a flock, since two concurrent writers (a real scenario on this repo's shared portal/condor machines) could otherwise race and silently drop one writer's freshly-computed cache section. - Documented (no behavior change) the pre_dir ≈ -ẑ antipodal rotation singularity in _rodrigues_axis, which is real but inherent to any single-valued local-frame convention. Each fix has a regression test. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+23
-3
@@ -27,6 +27,26 @@ def event_id_offset(file_index: int) -> int:
|
||||
return file_index * EVENT_ID_FILE_STRIDE
|
||||
|
||||
|
||||
def _offset_event_id(raw_ids: np.ndarray, offset: int) -> np.ndarray:
|
||||
"""Add this file's `event_id_offset`, after checking the raw ids fit in one stride block.
|
||||
|
||||
Without this check, a file whose own raw event_id numbering reaches
|
||||
`EVENT_ID_FILE_STRIDE` (an unusually large job, or non-contiguous
|
||||
numbering) would silently collide into the next file's offset block,
|
||||
merging unrelated events across files — reintroducing exactly the
|
||||
train/val event leakage this offset scheme exists to prevent.
|
||||
"""
|
||||
raw_ids = np.asarray(raw_ids, dtype=np.int64)
|
||||
if raw_ids.size and int(raw_ids.max()) >= EVENT_ID_FILE_STRIDE:
|
||||
raise ValueError(
|
||||
f"event_id {int(raw_ids.max())} >= EVENT_ID_FILE_STRIDE "
|
||||
f"({EVENT_ID_FILE_STRIDE}) — this file has a larger event_id "
|
||||
"than the per-file offset scheme can support without colliding "
|
||||
"with the next file's id block."
|
||||
)
|
||||
return raw_ids + offset
|
||||
|
||||
|
||||
def _read_manifest(path: Path) -> list[Path]:
|
||||
files = []
|
||||
for line in path.read_text().splitlines():
|
||||
@@ -98,7 +118,7 @@ def _df_to_dict(df: pd.DataFrame, offset: int = 0) -> dict[str, np.ndarray]:
|
||||
has_sec_lists = "sec_E_list" in df.columns
|
||||
|
||||
d: dict[str, np.ndarray] = {
|
||||
"event_id": df["event_id"].to_numpy().astype(np.int64) + offset,
|
||||
"event_id": _offset_event_id(df["event_id"].to_numpy(), offset),
|
||||
"pdg": df["pdg"].to_numpy(dtype=np.int32),
|
||||
"pre_pos": df[["pre_x", "pre_y", "pre_z"]].to_numpy(dtype=np.float32),
|
||||
"pre_E": df["pre_E"].to_numpy(dtype=np.float32),
|
||||
@@ -142,7 +162,7 @@ def load_steps(path: str | Path, offset: int = 0) -> dict[str, np.ndarray]:
|
||||
def load_event_ids(path: str | Path, offset: int = 0) -> np.ndarray:
|
||||
"""Read only the event_id column — cheap scan for split assignment."""
|
||||
ids = pd.read_parquet(path, columns=["event_id"])["event_id"].to_numpy()
|
||||
return ids.astype(np.int64) + offset
|
||||
return _offset_event_id(ids, offset)
|
||||
|
||||
|
||||
def iter_file_chunks(
|
||||
@@ -173,7 +193,7 @@ _COND_COLS = [
|
||||
|
||||
def _cond_df_to_dict(df: pd.DataFrame, offset: int = 0) -> dict[str, np.ndarray]:
|
||||
return {
|
||||
"event_id": df["event_id"].to_numpy().astype(np.int64) + offset,
|
||||
"event_id": _offset_event_id(df["event_id"].to_numpy(), offset),
|
||||
"pdg": df["pdg"].to_numpy(dtype=np.int32),
|
||||
"pre_pos": df[["pre_x", "pre_y", "pre_z"]].to_numpy(dtype=np.float32),
|
||||
"pre_E": df["pre_E"].to_numpy(dtype=np.float32),
|
||||
|
||||
Reference in New Issue
Block a user