lars
c984d0a19d
CI / Sync project version with tag (hand-pushed tags only) (pull_request) Has been skipped
CI / Publish package to Gitea package registry (pull_request) Has been skipped
CI / Format (ruff format) (pull_request) Successful in 53s
CI / Type check (ty) (pull_request) Successful in 57s
CI / Lint (ruff check) (pull_request) Successful in 41s
CI / Tests (pull_request) Successful in 8m20s
CI / Release (bump, changelog, badges, tag) on merge to master (pull_request) Has been skipped
uv.lock was stale (ty 0.0.50 -> 0.0.78, ruff 0.15 -> 0.16, polars, numpy, typer, wandb, pytest, and others), all within existing pyproject.toml bounds. ruff 0.16 widened its default rule selection, taking this repo from 0 to 274 lint errors under the same config; --fix handled most of it (import sorting, Optional[X] -> X | None, ...), and the remainder (unused unpacked variables, dict()-as-literal, subprocess.run without explicit check=, a couple of intentional broad excepts/naive datetimes) were fixed or annotated by hand. Also fixes a real type-narrowing gap ty 0.0.78 caught in test_config_consumed_keys.py's `or`-combined isinstance check. torch stays pinned to 2.3.x (deliberate, see CLAUDE.md); pyarrow's <25 ceiling is left as a separate decision. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TMdZFqXXig7i3XkirSUxef
235 lines
8.3 KiB
Python
235 lines
8.3 KiB
Python
from test_pipeline import _make_synthetic_steps
|
|
from typer.testing import CliRunner
|
|
|
|
from giant import cli as giant_cli
|
|
from giant.config import Conditioning
|
|
from giant.data import setup_cache
|
|
from giant.tools import dwarf
|
|
from giant.tools.dwarf import app
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
def test_conditioning_enum_shared_across_both_clis():
|
|
"""giant.cli and giant.tools.dwarf must use the one giant.config.Conditioning
|
|
enum, not independently redefined copies that could silently drift apart
|
|
on valid --conditioning values."""
|
|
assert dwarf.Conditioning is Conditioning
|
|
assert giant_cli.Conditioning is Conditioning
|
|
|
|
|
|
def test_warn_if_exceeds_shared_quota_warns_over_quarter_cpu(monkeypatch, capsys):
|
|
monkeypatch.setattr(dwarf.os, "cpu_count", lambda: 8) # quota = 2
|
|
dwarf._warn_if_exceeds_shared_quota(3, "--jobs")
|
|
assert "warning: --jobs=3 exceeds" in capsys.readouterr().err
|
|
|
|
|
|
def test_warn_if_exceeds_shared_quota_silent_within_quota(monkeypatch, capsys):
|
|
monkeypatch.setattr(dwarf.os, "cpu_count", lambda: 8) # quota = 2
|
|
dwarf._warn_if_exceeds_shared_quota(2, "--jobs")
|
|
assert capsys.readouterr().err == ""
|
|
|
|
|
|
def test_convert_rejects_jobs_below_one(tmp_path):
|
|
root_file = tmp_path / "shard.root"
|
|
root_file.touch()
|
|
result = runner.invoke(app, ["convert", str(root_file), "--jobs", "0"])
|
|
assert result.exit_code != 0
|
|
assert "--jobs must be >= 1" in result.output
|
|
|
|
|
|
def test_convert_rejects_output_with_multiple_files(tmp_path):
|
|
a = tmp_path / "a.root"
|
|
b = tmp_path / "b.root"
|
|
a.touch()
|
|
b.touch()
|
|
result = runner.invoke(app, ["convert", str(a), str(b), "--output", "out.parquet"])
|
|
assert result.exit_code != 0
|
|
assert "--output can only be used with a single input file" in result.output
|
|
|
|
|
|
def test_convert_rejects_output_with_parallel_jobs(tmp_path):
|
|
root_file = tmp_path / "shard.root"
|
|
root_file.touch()
|
|
result = runner.invoke(app, ["convert", str(root_file), "--output", "out.parquet", "--jobs", "2"])
|
|
assert result.exit_code != 0
|
|
assert "--output cannot be combined with --jobs > 1" in result.output
|
|
|
|
|
|
def test_convert_default_jobs_is_one():
|
|
result = runner.invoke(app, ["convert", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "default: 1" in result.output
|
|
|
|
|
|
def test_bump_gen_requires_reason():
|
|
result = runner.invoke(app, ["bump-gen"])
|
|
assert result.exit_code != 0
|
|
assert "reason" in result.output.lower()
|
|
|
|
|
|
def test_create_manifest_requires_exactly_one_of_output_or_pool(tmp_path):
|
|
f = tmp_path / "a.parquet"
|
|
f.touch()
|
|
result = runner.invoke(app, ["create-manifest", str(f)])
|
|
assert result.exit_code != 0
|
|
assert "exactly one of --output or --pool is required" in result.output
|
|
|
|
|
|
def test_create_manifest_requires_type_with_pool(tmp_path):
|
|
f = tmp_path / "a.parquet"
|
|
f.touch()
|
|
result = runner.invoke(app, ["create-manifest", "--pool", "pbwo4", str(f)])
|
|
assert result.exit_code != 0
|
|
assert "--type is required when --pool is given" in result.output
|
|
|
|
|
|
def test_status_reports_missing_root(tmp_path):
|
|
missing = tmp_path / "does-not-exist"
|
|
result = runner.invoke(app, ["status", "--root", str(missing)])
|
|
assert result.exit_code != 0
|
|
assert "is not a directory" in result.output
|
|
|
|
|
|
def test_warm_cache_writes_sidecar(tmp_path):
|
|
data = _make_synthetic_steps(tmp_path / "data.parquet", n_events=20)
|
|
|
|
result = runner.invoke(app, ["warm-cache", str(data)])
|
|
|
|
assert result.exit_code == 0, result.output
|
|
loaded = setup_cache.load(data, [data])
|
|
assert loaded is not None
|
|
assert loaded.vocab is not None
|
|
assert loaded.event_index is not None
|
|
assert "valfrac=0.1_seed=0_pcond=physical_mcond=physical" in loaded.normalizers
|
|
|
|
|
|
def test_warm_cache_second_run_hits_cache(tmp_path):
|
|
data = _make_synthetic_steps(tmp_path / "data.parquet", n_events=20)
|
|
runner.invoke(app, ["warm-cache", str(data)])
|
|
|
|
result = runner.invoke(app, ["warm-cache", str(data)])
|
|
|
|
assert result.exit_code == 0, result.output
|
|
assert "event index: cache hit" in result.output
|
|
assert "vocabulary maps: cache hit" in result.output
|
|
assert "normalizer: cache hit" in result.output
|
|
|
|
|
|
def test_warm_cache_router_process_warms_proc_map(tmp_path):
|
|
data = _make_synthetic_steps(tmp_path / "data.parquet", n_events=20)
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"warm-cache",
|
|
str(data),
|
|
# router.type="process" is incompatible with the default
|
|
# conditioning.particle.type="physical" (validate_config, now
|
|
# enforced by warm-cache too — see gitea #59).
|
|
"--particle-conditioning",
|
|
"embedding",
|
|
"--router",
|
|
"--router-type",
|
|
"process",
|
|
"--n-experts",
|
|
"3",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0, result.output
|
|
loaded = setup_cache.load(data, [data])
|
|
assert loaded is not None
|
|
assert 3 in loaded.proc_maps
|
|
|
|
|
|
def test_warm_cache_rebuild_ignores_existing(tmp_path):
|
|
data = _make_synthetic_steps(tmp_path / "data.parquet", n_events=20)
|
|
files = [data]
|
|
stale = setup_cache.SetupCache.empty(files)
|
|
stale.vocab = ({999999: 0}, {"G4_AIR": 0}) # deliberately wrong
|
|
setup_cache.save(data, files, stale)
|
|
|
|
result = runner.invoke(app, ["warm-cache", str(data), "--rebuild"])
|
|
|
|
assert result.exit_code == 0, result.output
|
|
loaded = setup_cache.load(data, files)
|
|
assert loaded is not None
|
|
assert loaded.vocab is not None
|
|
assert set(loaded.vocab[0].keys()) == {11, 22}
|
|
|
|
|
|
def test_warm_cache_different_val_fraction_is_separate_entry(tmp_path):
|
|
data = _make_synthetic_steps(tmp_path / "data.parquet", n_events=20)
|
|
runner.invoke(app, ["warm-cache", str(data), "--val-fraction", "0.1"])
|
|
|
|
result = runner.invoke(app, ["warm-cache", str(data), "--val-fraction", "0.3"])
|
|
|
|
assert result.exit_code == 0, result.output
|
|
assert "vocabulary maps: cache hit" in result.output
|
|
assert "fitting normalizer (streaming)" in result.output
|
|
loaded = setup_cache.load(data, [data])
|
|
assert loaded is not None
|
|
assert "valfrac=0.1_seed=0_pcond=physical_mcond=physical" in loaded.normalizers
|
|
assert "valfrac=0.3_seed=0_pcond=physical_mcond=physical" in loaded.normalizers
|
|
|
|
|
|
def test_warm_cache_config_warms_particle_type_n_classes(tmp_path):
|
|
"""gitea #59: a config setting stage2_model.particle_type.n_classes away
|
|
from its 0 (= inherit conditioning.particle.emb_dim) default must warm
|
|
the pdg top-N map under that n_classes, not the emb_dim default, so a
|
|
later `giant train --config <same file>` run hits it instead of quietly
|
|
re-scanning every parquet file."""
|
|
data = _make_synthetic_steps(tmp_path / "data.parquet", n_events=20)
|
|
config_path = tmp_path / "config.toml"
|
|
config_path.write_text("[meta]\nconfig_version = 3\n\n[stage2_model.particle_type]\nn_classes = 32\n")
|
|
|
|
runner.invoke(app, ["warm-cache", str(data), "--config", str(config_path)])
|
|
result = runner.invoke(app, ["warm-cache", str(data), "--config", str(config_path)])
|
|
|
|
assert result.exit_code == 0, result.output
|
|
assert "pdg top-N map: cache hit" in result.output
|
|
assert "32 classes" in result.output
|
|
|
|
|
|
def test_warm_cache_config_rejects_val_fraction_flag(tmp_path):
|
|
data = _make_synthetic_steps(tmp_path / "data.parquet", n_events=20)
|
|
config_path = tmp_path / "config.toml"
|
|
config_path.write_text("[meta]\nconfig_version = 3\n")
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
["warm-cache", str(data), "--config", str(config_path), "--val-fraction", "0.2"],
|
|
)
|
|
|
|
assert result.exit_code != 0
|
|
assert "--config" in result.output
|
|
assert "--val-fraction" in result.output
|
|
|
|
|
|
def test_warm_cache_config_rejects_router_flags(tmp_path):
|
|
data = _make_synthetic_steps(tmp_path / "data.parquet", n_events=20)
|
|
config_path = tmp_path / "config.toml"
|
|
config_path.write_text("[meta]\nconfig_version = 3\n")
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"warm-cache",
|
|
str(data),
|
|
"--config",
|
|
str(config_path),
|
|
"--router",
|
|
"--router-type",
|
|
"process",
|
|
"--n-experts",
|
|
"3",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code != 0
|
|
assert "--config" in result.output
|
|
assert "--router/--no-router" in result.output
|
|
assert "--router-type" in result.output
|
|
assert "--n-experts" in result.output
|