Add streaming data pipeline and giant CLI entry point

- Streaming pipeline: row-group-level parquet reading (PyArrow) so
  large files never fully land in RAM; Welford online algorithm for
  normalizer fitting; StreamingStepsDataset with shuffle buffer and
  multi-worker file striping; event-ID scan and vocab scan via cheap
  single-column reads
- giant/cli.py: typer-based CLI with `giant train` subcommand, mirroring
  scripts/train.py; --shuffle-buffer flag for RAM control
- pyproject.toml: add typer>=0.12 dependency and giant entry point
- train.py: replace len(loader.dataset) with local counters (compatible
  with IterableDataset which has no __len__)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

giant/data/loader.py

@@ -1,11 +1,22 @@
 from pathlib import Path
+from typing import Iterator
 
 import numpy as np
 import pandas as pd
+import pyarrow.parquet as pq
 
 
-def load_steps(path: str | Path) -> dict[str, np.ndarray]:
-    df = pd.read_parquet(path)
+def find_parquet_files(path: str | Path) -> list[Path]:
+    p = Path(path)
+    if p.is_dir():
+        files = sorted(p.glob("*.parquet"))
+        if not files:
+            raise FileNotFoundError(f"no .parquet files found in {p}")
+        return files
+    return [p]
+
+
+def _df_to_dict(df: pd.DataFrame) -> dict[str, np.ndarray]:
     return {
         "event_id": df["event_id"].to_numpy(),
         "pdg": df["pdg"].to_numpy(dtype=np.int32),
@@ -22,6 +33,22 @@ def load_steps(path: str | Path) -> dict[str, np.ndarray]:
     }
 
 
+def load_steps(path: str | Path) -> dict[str, np.ndarray]:
+    return _df_to_dict(pd.read_parquet(path))
+
+
+def load_event_ids(path: str | Path) -> np.ndarray:
+    """Read only the event_id column — cheap scan for split assignment."""
+    return pd.read_parquet(path, columns=["event_id"])["event_id"].to_numpy()
+
+
+def iter_file_chunks(path: str | Path) -> Iterator[dict[str, np.ndarray]]:
+    """Yield one parquet row-group at a time so a large file never fully loads."""
+    pf = pq.ParquetFile(path)
+    for i in range(pf.num_row_groups):
+        yield _df_to_dict(pf.read_row_group(i).to_pandas())
+
+
 def build_index_maps(
     data: dict[str, np.ndarray],
 ) -> tuple[dict[int, int], dict[int, int]]:
@@ -31,3 +58,19 @@ def build_index_maps(
         {v: i for i, v in enumerate(pdg_vals)},
         {v: i for i, v in enumerate(mat_vals)},
     )
+
+
+def build_index_maps_from_files(
+    files: list[Path],
+) -> tuple[dict[int, int], dict[int, int]]:
+    """Scan only pdg and material_id columns across all files (2-column read)."""
+    pdg_vals: set[int] = set()
+    mat_vals: set[int] = set()
+    for path in files:
+        df = pd.read_parquet(path, columns=["pdg", "material_id"])
+        pdg_vals.update(int(v) for v in df["pdg"].unique())
+        mat_vals.update(int(v) for v in df["material_id"].unique())
+    return (
+        {v: i for i, v in enumerate(sorted(pdg_vals))},
+        {v: i for i, v in enumerate(sorted(mat_vals))},
+    )