diff --git a/giant/analysis/prediction.py b/giant/analysis/prediction.py index 7827cce..c3777d3 100644 --- a/giant/analysis/prediction.py +++ b/giant/analysis/prediction.py @@ -275,7 +275,7 @@ def prediction_secondaries(lf: pl.LazyFrame, prefix: str) -> pl.LazyFrame: lists = [f"{col_prefix}sec_{c}_list" for c in ("E", "pdg", "dx", "dy", "dz")] return ( lf.select("event_id", *lists) - .explode(lists) + .explode(lists, empty_as_null=False) .drop_nulls(lists[0]) .select( "event_id", @@ -307,7 +307,7 @@ def paired_secondaries(lf: pl.LazyFrame) -> pl.LazyFrame: .with_columns(pl.min_horizontal("_n_true", "_n_pred").alias("_n_paired")) .filter(pl.col("_n_paired") > 0) .with_columns(pl.int_ranges(0, pl.col("_n_paired")).alias("_rank")) - .explode("_rank") + .explode("_rank", empty_as_null=False) .select( pl.col("true_sec_pdg_list").list.get(pl.col("_rank")).cast(pl.Int64).alias("true_pdg"), pl.col("sec_pdg_list").list.get(pl.col("_rank")).cast(pl.Int64).alias("pred_pdg"), diff --git a/giant/analysis/sources.py b/giant/analysis/sources.py index 51b033a..9441bda 100644 --- a/giant/analysis/sources.py +++ b/giant/analysis/sources.py @@ -230,7 +230,7 @@ def secondaries(lf: pl.LazyFrame, side: Side) -> pl.LazyFrame: lists = ["sec_E_list", "sec_pdg_list", "sec_dx_list", "sec_dy_list", "sec_dz_list"] return ( lf.select("event_id", *lists) - .explode(lists) + .explode(lists, empty_as_null=False) .drop_nulls("sec_E_list") .select( "event_id", @@ -268,7 +268,7 @@ def secondaries_by_step(lf: pl.LazyFrame, side: Side) -> pl.LazyFrame: return ( lf.select("sec_pdg_list") .with_row_index("_row") - .explode("sec_pdg_list") + .explode("sec_pdg_list", empty_as_null=False) .drop_nulls("sec_pdg_list") .select(pl.struct("_row").alias("step_key"), pl.col("sec_pdg_list").cast(pl.Int64).alias("pdg")) ) diff --git a/giant/data/scan.py b/giant/data/scan.py index b26bd74..cf524b5 100644 --- a/giant/data/scan.py +++ b/giant/data/scan.py @@ -76,7 +76,11 @@ def _pooled_pdg_lazy(path: Path, has_sec_pdg_list: bool) -> pl.LazyFrame: lf = pl.scan_parquet(path, row_index_name="__row") parts = [lf.select(pl.col("pdg").alias("__val"), "__row")] if has_sec_pdg_list: - parts.append(lf.select(pl.col("sec_pdg_list").alias("__val"), "__row").explode("__val").drop_nulls("__val")) + parts.append( + lf.select(pl.col("sec_pdg_list").alias("__val"), "__row") + .explode("__val", empty_as_null=False) + .drop_nulls("__val") + ) combined = pl.concat(parts) return combined.group_by("__val").agg(pl.len().alias("__count"), pl.col("__row").min().alias("__first_row")) diff --git a/giant/tools/steps_to_parquet.py b/giant/tools/steps_to_parquet.py index 880c965..b8230e6 100644 --- a/giant/tools/steps_to_parquet.py +++ b/giant/tools/steps_to_parquet.py @@ -58,7 +58,7 @@ def _add_secondary_attributes(df: pl.DataFrame) -> tuple[pl.DataFrame, int]: exploded = ( df.select(["event_id", "child_track_ids"]) .with_row_index("_step_row") - .explode("child_track_ids") + .explode("child_track_ids", empty_as_null=False) .rename({"child_track_ids": "child_track_id"}) .drop_nulls("child_track_id") ) diff --git a/pyproject.toml b/pyproject.toml index 33f4511..09ac5c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.12" dependencies = [ "numpy>=1.26,<3", - "polars>=1.0,<2", + "polars>=1.43,<2", "pyarrow>=16,<26", "tqdm>=4.60,<5", "typer>=0.12,<1", @@ -43,11 +43,11 @@ wandb = [ convert = [ "uproot>=5.3,<6", "awkward>=2.6,<3", - "polars>=1.0,<2", + "polars>=1.43,<2", ] analysis = [ "matplotlib>=3.8,<4", - "polars>=1.0,<2", + "polars>=1.43,<2", "ipykernel>=7.3.0", # KIT matplotlib theme, published from git.larsbogner.de. Only the local # `giant analyze render` step imports it; compute workers never do. diff --git a/uv.lock b/uv.lock index fd1cfe2..5e901ad 100644 --- a/uv.lock +++ b/uv.lock @@ -892,9 +892,9 @@ requires-dist = [ { name = "pandas", marker = "extra == 'dev'", specifier = ">=2.2,<4" }, { name = "particle", specifier = ">=1.0,<2" }, { name = "plotstyle", marker = "extra == 'analysis'", specifier = ">=1.0.0", index = "https://git.larsbogner.de/api/packages/lars/pypi/simple/" }, - { name = "polars", specifier = ">=1.0,<2" }, - { name = "polars", marker = "extra == 'analysis'", specifier = ">=1.0,<2" }, - { name = "polars", marker = "extra == 'convert'", specifier = ">=1.0,<2" }, + { name = "polars", specifier = ">=1.43,<2" }, + { name = "polars", marker = "extra == 'analysis'", specifier = ">=1.43,<2" }, + { name = "polars", marker = "extra == 'convert'", specifier = ">=1.43,<2" }, { name = "pyarrow", specifier = ">=16,<26" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8,<10" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5,<8" },