Add giant.analysis module for notebook-based model quality diagnostics
Provides stratified marginal comparisons, joint-structure checks (correlation matrices, physically-coupled pairwise plots, direction alignment), and physical-constraint validation (unit-norm directions, non-negative raw targets) for a trained model's generated samples, building on the aggregate marginal/KL check already in giant.validate. Supports two entry points: live sampling against a checkpoint + val data (load_model_bundle/collect_samples), or loading a precomputed `giant predict --coord local` parquet directly (load_predicted_local) without needing the checkpoint at all. Predict output is now tagged with parquet schema metadata so the loader can verify a file's format and reject coord=global or untagged files with a clear error instead of guessing from column names. Also extends the config git-hash mismatch warning (added for --config loading) to checkpoint loading: both `giant predict` and analysis.load_model_bundle now look for a config.toml next to the checkpoint and warn (without failing) if it was generated from a different git commit. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
from giant import config as gconfig
|
||||
|
||||
|
||||
def _write_config(path, git_hash):
|
||||
path.write_text(
|
||||
f"""
|
||||
[train]
|
||||
epochs = 5
|
||||
|
||||
[model]
|
||||
hidden_dim = 64
|
||||
|
||||
[meta]
|
||||
git_hash = "{git_hash}"
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def test_merge_cli_overrides_applies_file_then_cli(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(gconfig, "git_hash", lambda: "abc123")
|
||||
path = tmp_path / "config.toml"
|
||||
_write_config(path, "abc123")
|
||||
|
||||
cfg = gconfig.merge_cli_overrides(
|
||||
gconfig.DEFAULT_CONFIG, path, train_overrides={}, model_overrides={"hidden_dim": 128},
|
||||
)
|
||||
assert cfg["train"]["epochs"] == 5 # from file
|
||||
assert cfg["model"]["hidden_dim"] == 128 # CLI override wins over file
|
||||
|
||||
|
||||
def test_merge_cli_overrides_warns_on_git_hash_mismatch(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.setattr(gconfig, "git_hash", lambda: "current999")
|
||||
path = tmp_path / "config.toml"
|
||||
_write_config(path, "old111")
|
||||
|
||||
cfg = gconfig.merge_cli_overrides(gconfig.DEFAULT_CONFIG, path, {}, {})
|
||||
|
||||
assert cfg["train"]["epochs"] == 5 # does not fail, config still applied
|
||||
captured = capsys.readouterr()
|
||||
assert "warning" in captured.err
|
||||
assert "old111" in captured.err
|
||||
assert "current999" in captured.err
|
||||
|
||||
|
||||
def test_merge_cli_overrides_no_warning_on_matching_git_hash(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.setattr(gconfig, "git_hash", lambda: "same123")
|
||||
path = tmp_path / "config.toml"
|
||||
_write_config(path, "same123")
|
||||
|
||||
gconfig.merge_cli_overrides(gconfig.DEFAULT_CONFIG, path, {}, {})
|
||||
assert capsys.readouterr().err == ""
|
||||
|
||||
|
||||
def test_merge_cli_overrides_no_warning_when_git_hash_unknown(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.setattr(gconfig, "git_hash", lambda: "unknown")
|
||||
path = tmp_path / "config.toml"
|
||||
_write_config(path, "abc123")
|
||||
|
||||
gconfig.merge_cli_overrides(gconfig.DEFAULT_CONFIG, path, {}, {})
|
||||
assert capsys.readouterr().err == ""
|
||||
|
||||
|
||||
def test_merge_cli_overrides_no_warning_when_meta_section_absent(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.setattr(gconfig, "git_hash", lambda: "current999")
|
||||
path = tmp_path / "config.toml"
|
||||
path.write_text("[train]\nepochs = 5\n")
|
||||
|
||||
gconfig.merge_cli_overrides(gconfig.DEFAULT_CONFIG, path, {}, {})
|
||||
assert capsys.readouterr().err == ""
|
||||
|
||||
|
||||
def test_warn_if_checkpoint_config_mismatch_finds_sibling_toml(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.setattr(gconfig, "git_hash", lambda: "current999")
|
||||
ckpt_path = tmp_path / "best.pt"
|
||||
ckpt_path.write_bytes(b"") # contents irrelevant, only its directory is used
|
||||
_write_config(tmp_path / "config.toml", "old111")
|
||||
|
||||
gconfig.warn_if_checkpoint_config_mismatch(ckpt_path)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert "warning" in captured.err
|
||||
assert "old111" in captured.err
|
||||
assert "current999" in captured.err
|
||||
|
||||
|
||||
def test_warn_if_checkpoint_config_mismatch_no_warning_when_toml_absent(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.setattr(gconfig, "git_hash", lambda: "current999")
|
||||
ckpt_path = tmp_path / "best.pt"
|
||||
ckpt_path.write_bytes(b"")
|
||||
|
||||
gconfig.warn_if_checkpoint_config_mismatch(ckpt_path)
|
||||
assert capsys.readouterr().err == ""
|
||||
|
||||
|
||||
def test_warn_if_checkpoint_config_mismatch_no_warning_when_hashes_match(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.setattr(gconfig, "git_hash", lambda: "same123")
|
||||
ckpt_path = tmp_path / "best.pt"
|
||||
ckpt_path.write_bytes(b"")
|
||||
_write_config(tmp_path / "config.toml", "same123")
|
||||
|
||||
gconfig.warn_if_checkpoint_config_mismatch(ckpt_path)
|
||||
assert capsys.readouterr().err == ""
|
||||
Reference in New Issue
Block a user