Offset event_id per file to avoid cross-file collisions
CI / Format (ruff format) (push) Successful in 25s
CI / Lint (ruff check) (push) Successful in 27s
CI / Sync project version with tag (push) Has been skipped
CI / Type check (ty) (push) Successful in 23s
CI / Lint (ruff check) (pull_request) Successful in 27s
CI / Tests (push) Successful in 1m14s
CI / Format (ruff format) (pull_request) Successful in 27s
CI / Sync project version with tag (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 31s
CI / Tests (pull_request) Successful in 59s
CI / Format (ruff format) (push) Successful in 25s
CI / Lint (ruff check) (push) Successful in 27s
CI / Sync project version with tag (push) Has been skipped
CI / Type check (ty) (push) Successful in 23s
CI / Lint (ruff check) (pull_request) Successful in 27s
CI / Tests (push) Successful in 1m14s
CI / Format (ruff format) (pull_request) Successful in 27s
CI / Sync project version with tag (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 31s
CI / Tests (pull_request) Successful in 59s
Each input parquet file is one Geant4 job (scripts/steps_to_parquet.py), and a job's event_id numbering always restarts from 0 — so loading multiple files together (a directory or .manifest) let same-numbered events from different files collapse into one during the event index scan and train/val split, corrupting both. Every per-file event_id now gets offset by file index * EVENT_ID_FILE_STRIDE (giant/data/loader.py), threaded through the setup-cache event index, the streaming dataset, and predict/rollout seeding. Bumps the setup-cache format version so stale sidecars computed pre-fix are invalidated. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+97
-1
@@ -1,5 +1,10 @@
|
||||
import numpy as np
|
||||
from giant.data.dataset import make_event_split
|
||||
import pandas as pd
|
||||
|
||||
from giant.constants import COND_DIM, X_DIM
|
||||
from giant.data import setup_cache
|
||||
from giant.data.dataset import StreamingStepsDataset, make_event_split
|
||||
from giant.data.transforms import Normalizer
|
||||
|
||||
|
||||
def test_make_event_split_sizes():
|
||||
@@ -31,3 +36,94 @@ def test_make_event_split_reproducible():
|
||||
b_tr, b_val = make_event_split(event_ids, val_fraction=0.1, seed=42)
|
||||
assert a_tr == b_tr
|
||||
assert a_val == b_val
|
||||
|
||||
|
||||
# ── StreamingStepsDataset: cross-file event_id offsetting ──────────────────
|
||||
|
||||
|
||||
def _steps_df(event_ids, n_per_event=3, pre_E=100.0):
|
||||
"""A schema-complete but minimal steps DataFrame — no secondaries, so
|
||||
`require_secondaries=True` never needs the per-secondary list columns."""
|
||||
rows = []
|
||||
for eid in event_ids:
|
||||
for s in range(n_per_event):
|
||||
rows.append(
|
||||
{
|
||||
"event_id": eid,
|
||||
"pdg": 11,
|
||||
"pre_x": 0.0,
|
||||
"pre_y": 0.0,
|
||||
"pre_z": 0.0,
|
||||
"pre_E": pre_E,
|
||||
"pre_dx": 0.0,
|
||||
"pre_dy": 0.0,
|
||||
"pre_dz": 1.0,
|
||||
"material": "G4_AIR",
|
||||
"layer_id": s,
|
||||
"child_track_ids": [],
|
||||
"e_sec": 0.0,
|
||||
"step_length": 1.0,
|
||||
"post_E": pre_E * 0.9,
|
||||
"edep": pre_E * 0.1,
|
||||
"post_dx": 0.0,
|
||||
"post_dy": 0.0,
|
||||
"post_dz": 1.0,
|
||||
"post_x": 0.0,
|
||||
"post_y": 0.0,
|
||||
"post_z": 1.0,
|
||||
}
|
||||
)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _dummy_normalizer(width):
|
||||
norm = Normalizer()
|
||||
norm.mean = np.zeros(width, dtype=np.float32)
|
||||
norm.std = np.ones(width, dtype=np.float32)
|
||||
return norm
|
||||
|
||||
|
||||
def test_streaming_dataset_offsets_colliding_event_ids_across_files(tmp_path):
|
||||
"""Two files that each restart event_id from 0 (one Geant4 job per file,
|
||||
see scripts/steps_to_parquet.py) must not have their same-numbered events
|
||||
collapsed together: every row from every file must show up in exactly one
|
||||
of train/val, and the number of distinct events must be the sum across
|
||||
files, not the union of raw ids."""
|
||||
n_events, n_per_event = 5, 3
|
||||
path_a = tmp_path / "a.parquet"
|
||||
path_b = tmp_path / "b.parquet"
|
||||
_steps_df(range(n_events), n_per_event=n_per_event).to_parquet(path_a)
|
||||
_steps_df(range(n_events), n_per_event=n_per_event).to_parquet(path_b)
|
||||
files = [path_a, path_b]
|
||||
|
||||
unique_ids, counts = setup_cache.compute_event_index_from_files(files)
|
||||
assert len(unique_ids) == 2 * n_events
|
||||
|
||||
train_events, val_events = make_event_split(unique_ids, val_fraction=0.4, seed=0)
|
||||
assert train_events.isdisjoint(val_events)
|
||||
|
||||
pdg_map, mat_map = {11: 0}, {"G4_AIR": 0}
|
||||
cond_norm = _dummy_normalizer(COND_DIM)
|
||||
tgt_norm = _dummy_normalizer(X_DIM)
|
||||
|
||||
def _count_rows(split_events):
|
||||
ds = StreamingStepsDataset(
|
||||
files=files,
|
||||
split_events=split_events,
|
||||
pdg_map=pdg_map,
|
||||
mat_map=mat_map,
|
||||
cond_normalizer=cond_norm,
|
||||
target_normalizer=tgt_norm,
|
||||
batch_size=4,
|
||||
shuffle=False,
|
||||
conditioning="embedding",
|
||||
)
|
||||
return sum(len(batch[0]) for batch in ds)
|
||||
|
||||
n_train = _count_rows(train_events)
|
||||
n_val = _count_rows(val_events)
|
||||
|
||||
total_rows = 2 * n_events * n_per_event
|
||||
assert n_train + n_val == total_rows
|
||||
assert n_train == int(counts[np.isin(unique_ids, list(train_events))].sum())
|
||||
assert n_val == int(counts[np.isin(unique_ids, list(val_events))].sum())
|
||||
|
||||
Reference in New Issue
Block a user