Add lazy polars I/O and duplicate KL/constraint checks for giant.analysis

load_predicted_local now reads predict parquet via a lazy polars scan with
column projection pushed into the reader, instead of materializing the
whole file as a pandas DataFrame. Also adds marginal_table_pl and
constraint_report_pl, polars-native duplicates that read straight from a
predict parquet path/LazyFrame and stay lazy per (group, dim) pair, so
peak memory is one column slice rather than the whole SampleCollection.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-18 17:06:58 +02:00
co-authored by Claude Sonnet 4.6
parent a867fc4aae
commit 80b198c1a5
4 changed files with 230 additions and 13 deletions
+55
View File
@@ -12,10 +12,12 @@ from giant.analysis import (
RAW_TARGET_NAMES,
SampleCollection,
constraint_report,
constraint_report_pl,
correlation_matrices,
direction_alignment,
load_predicted_local,
marginal_table,
marginal_table_pl,
plot_constraint_violations,
plot_correlation_matrices,
plot_direction_alignment,
@@ -241,3 +243,56 @@ def test_load_predicted_local_rejects_mismatched_schema_version(tmp_path):
)
with pytest.raises(ValueError, match="schema version"):
load_predicted_local(path)
def _predicted_local_path(tmp_path, n=200, seed=0):
path = tmp_path / "predicted_local.parquet"
_write_predicted_local_parquet(
path,
n=n,
rng=np.random.default_rng(seed),
metadata={
PREDICT_COORD_METADATA_KEY: "local",
PREDICT_SCHEMA_VERSION_KEY: PREDICT_SCHEMA_VERSION,
},
)
return path
@pytest.mark.parametrize("group_by", [None, "pdg", "material", "energy"])
def test_marginal_table_pl_matches_numpy_version(tmp_path, group_by):
path = _predicted_local_path(tmp_path)
collection = load_predicted_local(path)
expected = marginal_table(collection, group_by=group_by).sort_values(["group", "dim"])
actual = marginal_table_pl(path, group_by=group_by).sort(["group", "dim"]).to_pandas()
assert list(expected["group"]) == list(actual["group"])
assert list(expected["n"]) == list(actual["n"])
for col in ["real_mean", "gen_mean", "real_std", "gen_std", "kl_real_gen"]:
np.testing.assert_allclose(
expected[col].to_numpy(), actual[col].to_numpy(), atol=1e-4, rtol=1e-4,
)
def test_marginal_table_pl_rejects_missing_metadata(tmp_path):
path = tmp_path / "no_metadata.parquet"
_write_predicted_local_parquet(path, metadata=None)
with pytest.raises(ValueError, match="no '.*' parquet metadata"):
marginal_table_pl(path)
def test_constraint_report_pl_matches_numpy_version(tmp_path):
path = _predicted_local_path(tmp_path)
collection = load_predicted_local(path)
expected = constraint_report(collection)
actual = constraint_report_pl(path).to_pandas()
assert list(expected["check"]) == list(actual["check"])
np.testing.assert_allclose(
expected["violation_rate"].to_numpy(), actual["violation_rate"].to_numpy(), atol=1e-6,
)
np.testing.assert_allclose(
expected["mean_abs_error"].to_numpy(), actual["mean_abs_error"].to_numpy(), atol=1e-4,
)