Offset event_id per file to avoid cross-file collisions
CI / Format (ruff format) (push) Successful in 25s
CI / Lint (ruff check) (push) Successful in 27s
CI / Sync project version with tag (push) Has been skipped
CI / Type check (ty) (push) Successful in 23s
CI / Lint (ruff check) (pull_request) Successful in 27s
CI / Tests (push) Successful in 1m14s
CI / Format (ruff format) (pull_request) Successful in 27s
CI / Sync project version with tag (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 31s
CI / Tests (pull_request) Successful in 59s

Each input parquet file is one Geant4 job (scripts/steps_to_parquet.py),
and a job's event_id numbering always restarts from 0 — so loading
multiple files together (a directory or .manifest) let same-numbered
events from different files collapse into one during the event index
scan and train/val split, corrupting both. Every per-file event_id now
gets offset by file index * EVENT_ID_FILE_STRIDE (giant/data/loader.py),
threaded through the setup-cache event index, the streaming dataset,
and predict/rollout seeding. Bumps the setup-cache format version so
stale sidecars computed pre-fix are invalidated.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-30 11:36:48 +02:00
parent e288c3fe21
commit b944bba8fb
8 changed files with 268 additions and 24 deletions
+97 -1
View File
@@ -1,5 +1,10 @@
import numpy as np
from giant.data.dataset import make_event_split
import pandas as pd
from giant.constants import COND_DIM, X_DIM
from giant.data import setup_cache
from giant.data.dataset import StreamingStepsDataset, make_event_split
from giant.data.transforms import Normalizer
def test_make_event_split_sizes():
@@ -31,3 +36,94 @@ def test_make_event_split_reproducible():
b_tr, b_val = make_event_split(event_ids, val_fraction=0.1, seed=42)
assert a_tr == b_tr
assert a_val == b_val
# ── StreamingStepsDataset: cross-file event_id offsetting ──────────────────
def _steps_df(event_ids, n_per_event=3, pre_E=100.0):
"""A schema-complete but minimal steps DataFrame — no secondaries, so
`require_secondaries=True` never needs the per-secondary list columns."""
rows = []
for eid in event_ids:
for s in range(n_per_event):
rows.append(
{
"event_id": eid,
"pdg": 11,
"pre_x": 0.0,
"pre_y": 0.0,
"pre_z": 0.0,
"pre_E": pre_E,
"pre_dx": 0.0,
"pre_dy": 0.0,
"pre_dz": 1.0,
"material": "G4_AIR",
"layer_id": s,
"child_track_ids": [],
"e_sec": 0.0,
"step_length": 1.0,
"post_E": pre_E * 0.9,
"edep": pre_E * 0.1,
"post_dx": 0.0,
"post_dy": 0.0,
"post_dz": 1.0,
"post_x": 0.0,
"post_y": 0.0,
"post_z": 1.0,
}
)
return pd.DataFrame(rows)
def _dummy_normalizer(width):
norm = Normalizer()
norm.mean = np.zeros(width, dtype=np.float32)
norm.std = np.ones(width, dtype=np.float32)
return norm
def test_streaming_dataset_offsets_colliding_event_ids_across_files(tmp_path):
"""Two files that each restart event_id from 0 (one Geant4 job per file,
see scripts/steps_to_parquet.py) must not have their same-numbered events
collapsed together: every row from every file must show up in exactly one
of train/val, and the number of distinct events must be the sum across
files, not the union of raw ids."""
n_events, n_per_event = 5, 3
path_a = tmp_path / "a.parquet"
path_b = tmp_path / "b.parquet"
_steps_df(range(n_events), n_per_event=n_per_event).to_parquet(path_a)
_steps_df(range(n_events), n_per_event=n_per_event).to_parquet(path_b)
files = [path_a, path_b]
unique_ids, counts = setup_cache.compute_event_index_from_files(files)
assert len(unique_ids) == 2 * n_events
train_events, val_events = make_event_split(unique_ids, val_fraction=0.4, seed=0)
assert train_events.isdisjoint(val_events)
pdg_map, mat_map = {11: 0}, {"G4_AIR": 0}
cond_norm = _dummy_normalizer(COND_DIM)
tgt_norm = _dummy_normalizer(X_DIM)
def _count_rows(split_events):
ds = StreamingStepsDataset(
files=files,
split_events=split_events,
pdg_map=pdg_map,
mat_map=mat_map,
cond_normalizer=cond_norm,
target_normalizer=tgt_norm,
batch_size=4,
shuffle=False,
conditioning="embedding",
)
return sum(len(batch[0]) for batch in ds)
n_train = _count_rows(train_events)
n_val = _count_rows(val_events)
total_rows = 2 * n_events * n_per_event
assert n_train + n_val == total_rows
assert n_train == int(counts[np.isin(unique_ids, list(train_events))].sum())
assert n_val == int(counts[np.isin(unique_ids, list(val_events))].sum())