Add lazy polars I/O and duplicate KL/constraint checks for giant.analysis
load_predicted_local now reads predict parquet via a lazy polars scan with column projection pushed into the reader, instead of materializing the whole file as a pandas DataFrame. Also adds marginal_table_pl and constraint_report_pl, polars-native duplicates that read straight from a predict parquet path/LazyFrame and stay lazy per (group, dim) pair, so peak memory is one column slice rather than the whole SampleCollection. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -12,10 +12,12 @@ from giant.analysis import (
|
||||
RAW_TARGET_NAMES,
|
||||
SampleCollection,
|
||||
constraint_report,
|
||||
constraint_report_pl,
|
||||
correlation_matrices,
|
||||
direction_alignment,
|
||||
load_predicted_local,
|
||||
marginal_table,
|
||||
marginal_table_pl,
|
||||
plot_constraint_violations,
|
||||
plot_correlation_matrices,
|
||||
plot_direction_alignment,
|
||||
@@ -241,3 +243,56 @@ def test_load_predicted_local_rejects_mismatched_schema_version(tmp_path):
|
||||
)
|
||||
with pytest.raises(ValueError, match="schema version"):
|
||||
load_predicted_local(path)
|
||||
|
||||
|
||||
def _predicted_local_path(tmp_path, n=200, seed=0):
|
||||
path = tmp_path / "predicted_local.parquet"
|
||||
_write_predicted_local_parquet(
|
||||
path,
|
||||
n=n,
|
||||
rng=np.random.default_rng(seed),
|
||||
metadata={
|
||||
PREDICT_COORD_METADATA_KEY: "local",
|
||||
PREDICT_SCHEMA_VERSION_KEY: PREDICT_SCHEMA_VERSION,
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group_by", [None, "pdg", "material", "energy"])
|
||||
def test_marginal_table_pl_matches_numpy_version(tmp_path, group_by):
|
||||
path = _predicted_local_path(tmp_path)
|
||||
collection = load_predicted_local(path)
|
||||
|
||||
expected = marginal_table(collection, group_by=group_by).sort_values(["group", "dim"])
|
||||
actual = marginal_table_pl(path, group_by=group_by).sort(["group", "dim"]).to_pandas()
|
||||
|
||||
assert list(expected["group"]) == list(actual["group"])
|
||||
assert list(expected["n"]) == list(actual["n"])
|
||||
for col in ["real_mean", "gen_mean", "real_std", "gen_std", "kl_real_gen"]:
|
||||
np.testing.assert_allclose(
|
||||
expected[col].to_numpy(), actual[col].to_numpy(), atol=1e-4, rtol=1e-4,
|
||||
)
|
||||
|
||||
|
||||
def test_marginal_table_pl_rejects_missing_metadata(tmp_path):
|
||||
path = tmp_path / "no_metadata.parquet"
|
||||
_write_predicted_local_parquet(path, metadata=None)
|
||||
with pytest.raises(ValueError, match="no '.*' parquet metadata"):
|
||||
marginal_table_pl(path)
|
||||
|
||||
|
||||
def test_constraint_report_pl_matches_numpy_version(tmp_path):
|
||||
path = _predicted_local_path(tmp_path)
|
||||
collection = load_predicted_local(path)
|
||||
|
||||
expected = constraint_report(collection)
|
||||
actual = constraint_report_pl(path).to_pandas()
|
||||
|
||||
assert list(expected["check"]) == list(actual["check"])
|
||||
np.testing.assert_allclose(
|
||||
expected["violation_rate"].to_numpy(), actual["violation_rate"].to_numpy(), atol=1e-6,
|
||||
)
|
||||
np.testing.assert_allclose(
|
||||
expected["mean_abs_error"].to_numpy(), actual["mean_abs_error"].to_numpy(), atol=1e-4,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user