From bdebd83c8bdce10258d20c98f3da6c83001e130d Mon Sep 17 00:00:00 2001 From: Lars Bogner Date: Mon, 24 Aug 2026 10:55:15 +0200 Subject: [PATCH] Add giant analyze metrics plots for training progress (gitea #75) MetricsCollector writes one row per epoch to /metrics.csv, but nothing read or plotted it. giant/training/plots.py reads the CSV header dynamically (the column set varies by run: flow/ddpm vs wgan, routed vs not) and renders loss/lr/accuracy/grad-norm/router/wgan-balance/throughput plots with the same plotstyle conventions giant/analysis/render.py uses, skipping any figure whose columns aren't present for a given run. Wired up as `giant analyze metrics `, writing PDFs into the same gitignored analysis_runs/ directory `analyze prep`/`submit` already use (derive_metrics_dir mirrors derive_run_dir) rather than into the training run directory itself. --- giant/cli.py | 19 ++ giant/training/plots.py | 356 +++++++++++++++++++++++++++++++++++ tests/test_training_plots.py | 333 ++++++++++++++++++++++++++++++++ 3 files changed, 708 insertions(+) create mode 100644 giant/training/plots.py create mode 100644 tests/test_training_plots.py diff --git a/giant/cli.py b/giant/cli.py index 9f199b2..c2add9b 100644 --- a/giant/cli.py +++ b/giant/cli.py @@ -1644,6 +1644,25 @@ def analyze_render( typer.echo(f"rendered {len(pdfs)} plots → {Path(run_dir) / 'plots'}") +@analyze_app.command("metrics") +def analyze_metrics( + run_dir: Annotated[Path, typer.Argument(help="Run directory containing metrics.csv (from `giant train`)")], + out_dir: Annotated[ + Optional[Path], + typer.Option( + "--out", + "-o", + help="Override the output directory (default: /analysis_runs/metrics_)", + ), + ] = None, +) -> None: + """Render training-progress plots (loss/lr/accuracy/grad-norm/router/wgan/throughput) from /metrics.csv.""" + from giant.training.plots import render_metrics + + paths = render_metrics(run_dir, out_dir, default_base=Path.cwd() / "analysis_runs") + typer.echo(f"rendered {len(paths)} plots -> {paths[0].parent if paths else '(nothing to render)'}") + + @analyze_app.command("submit") def analyze_submit( rollout_yaml: Annotated[Path, typer.Argument(help="giant rollout YAML sidecar")], diff --git a/giant/training/plots.py b/giant/training/plots.py new file mode 100644 index 0000000..b15ef17 --- /dev/null +++ b/giant/training/plots.py @@ -0,0 +1,356 @@ +"""Training-progress plots from `/metrics.csv` (gitea #75). + +`MetricsCollector` (`giant.training.metrics`) writes one row per epoch with a +column set that varies by run — flow/ddpm vs wgan, routed vs not (see the +`MetricSpec` declarations in `giant.training.trainers`). This module reads +that header dynamically rather than hardcoding a column list, buckets columns +by the fixed naming convention `MetricsCollector` itself documents +(`/train/`, `/val/`, `/router/`, +`/` for point-in-time values, and an unprefixed run-level tail — +see `giant.training.metrics`'s module docstring), and renders one PDF per +applicable figure with the same `plotstyle` conventions +`giant.analysis.render` uses, for visual consistency with the +rollout-vs-reference plots. + +Unlike `giant.analysis`, there is no reduce/chunk/condor split here — the CSV +is tiny and this always runs as one local pass — but the CLI entry point +still lives under `giant analyze` (`analyze metrics`) as the shared home for +plotstyle-rendered diagnostics, and shares its `analysis_runs/` output +convention (see `derive_metrics_dir`) so training-progress plots don't get +written into the training run directory itself. +""" + +from __future__ import annotations + +import csv +import math +from dataclasses import dataclass +from pathlib import Path + +# Stage names are always exactly these two — hardcoded in +# `giant.training.trainers.build_stage_trainers` — so a column belongs to a +# stage iff it's prefixed by one of these, and everything else (bar `epoch`) +# is run-level. This is what makes dynamic header parsing tractable without +# needing to know the per-run metric keys themselves. +_STAGE_NAMES = ("stage1", "stage2") + +_ACC_KEYS = {"nsec_acc", "stop_acc", "type_acc"} +_WGAN_BALANCE_KEYS = {"d_loss", "g_loss", "wasserstein", "gp_loss"} +_ROUTER_KEYS = ("entropy", "util_min", "util_max", "util_std") + + +@dataclass +class MetricsTable: + """`/metrics.csv`, parsed with no hardcoded column list.""" + + epochs: list[int] + columns: dict[str, list[float]] + + @classmethod + def load(cls, path: str | Path) -> "MetricsTable": + with open(path, newline="") as f: + rows = list(csv.DictReader(f)) + epochs = [int(float(r["epoch"])) for r in rows] + fieldnames = rows[0].keys() if rows else [] + columns = {name: [float(r[name]) for r in rows] for name in fieldnames if name != "epoch"} + return cls(epochs=epochs, columns=columns) + + def best_epochs(self) -> list[int]: + is_best = self.columns.get("is_best") + if not is_best: + return [] + return [epoch for epoch, flag in zip(self.epochs, is_best) if flag] + + +# --- column classification -------------------------------------------------- + + +def _stages(columns: dict) -> list[str]: + return [s for s in _STAGE_NAMES if any(name.startswith(f"{s}/") for name in columns)] + + +def _split(columns: dict, stage: str, split: str) -> dict[str, str]: + prefix = f"{stage}/{split}/" + return {name[len(prefix) :]: name for name in columns if name.startswith(prefix)} + + +def _point_in_time(columns: dict, stage: str) -> dict[str, str]: + prefix = f"{stage}/" + out = {} + for name in columns: + if not name.startswith(prefix): + continue + rest = name[len(prefix) :] + head = rest.split("/", 1)[0] + if head not in ("train", "val", "router"): + out[rest] = name + return out + + +def _router(columns: dict, stage: str) -> dict[str, str]: + prefix = f"{stage}/router/" + return {name[len(prefix) :]: name for name in columns if name.startswith(prefix)} + + +def _run_level(columns: dict) -> dict[str, str]: + known_prefixes = tuple(f"{s}/" for s in _STAGE_NAMES) + return {name: name for name in columns if not name.startswith(known_prefixes)} + + +def _loss_keys(train: dict[str, str], val: dict[str, str]) -> list[str]: + keys = {k for k in train if k not in _ACC_KEYS and k not in _WGAN_BALANCE_KEYS and k != "grad_norm"} + keys |= {k for k in val if k not in _ACC_KEYS and k not in _WGAN_BALANCE_KEYS and k != "grad_norm"} + return sorted(keys) + + +# --- output location --------------------------------------------------------- + + +def derive_metrics_dir( + run_dir: str | Path, + out_dir: str | Path | None = None, + default_base: str | Path | None = None, +) -> Path: + """Plots output directory. + + Precedence: an explicit `out_dir` always wins. Otherwise + `default_base / f"metrics_{run_dir.name}"` (the CLI passes the repo's + gitignored `analysis_runs/`, matching `giant.analysis.condor.derive_run_dir`'s + convention) — training-progress plots live alongside rollout-vs-reference + analysis runs, not inside the training run directory itself. + """ + if out_dir is not None: + return Path(out_dir) + base = Path(default_base) if default_base is not None else Path.cwd() / "analysis_runs" + return base / f"metrics_{Path(run_dir).name}" + + +# --- figures ------------------------------------------------------------------ + + +def _mark_best(ax, table: MetricsTable) -> None: + for epoch in table.best_epochs(): + ax.axvline(epoch, color="grey", linestyle="--", linewidth=0.8, alpha=0.7) + + +def _overview_figure(table: MetricsTable): + import plotstyle as ps + + run_level = _run_level(table.columns) + if "val/loss" not in run_level: + return None + fig, ax = ps.new_figure("thesis-single", title="training overview") + ax.plot(table.epochs, table.columns["val/loss"], label="val/loss") + if "val/marginal_kl" in run_level: + kl = table.columns["val/marginal_kl"] + if any(math.isfinite(v) for v in kl): + ax.plot(table.epochs, kl, label="val/marginal_kl") + _mark_best(ax, table) + best = table.best_epochs() + if best: + idx = table.epochs.index(best[-1]) + ax.annotate( + f"best: epoch {best[-1]}\nval/loss={table.columns['val/loss'][idx]:.4g}", + xy=(best[-1], table.columns["val/loss"][idx]), + xytext=(0.98, 0.95), + textcoords="axes fraction", + ha="right", + va="top", + fontsize=8, + ) + ax.set_xlabel("epoch") + ax.set_ylabel("loss") + ps.style_legend(ax, title="series") + return fig + + +def _loss_figure(table: MetricsTable, stage: str): + import plotstyle as ps + + train = _split(table.columns, stage, "train") + val = _split(table.columns, stage, "val") + keys = _loss_keys(train, val) + if not keys: + return None + n = len(keys) + ncols = min(3, n) + nrows = (n + ncols - 1) // ncols + fig, axes = ps.new_figure( + "slide-16x9", + title=f"{stage} loss", + nrows=nrows, + ncols=ncols, + squeeze=False, + ) + flat = axes.ravel() + for ax, key in zip(flat, keys): + if key in train: + ax.plot(table.epochs, table.columns[train[key]], label="train") + if key in val: + ax.plot(table.epochs, table.columns[val[key]], label="val") + ax.set_yscale("log") + ax.set_title(key, fontsize=8) + ax.set_xlabel("epoch") + for j in range(n, len(flat)): + flat[j].set_visible(False) + ps.style_legend(flat[0], title="series") + return fig + + +def _lr_figure(table: MetricsTable): + import plotstyle as ps + + series: dict[str, str] = {} + for stage in _stages(table.columns): + for key, col in _point_in_time(table.columns, stage).items(): + series[f"{stage}/{key}"] = col + if not series: + return None + fig, ax = ps.new_figure("thesis-single", title="learning rate schedule") + for label, col in series.items(): + ax.plot(table.epochs, table.columns[col], label=label) + ax.set_xlabel("epoch") + ax.set_ylabel("learning rate") + ps.style_legend(ax, title="series") + return fig + + +def _accuracy_figure(table: MetricsTable, stage: str): + import plotstyle as ps + + train = _split(table.columns, stage, "train") + val = _split(table.columns, stage, "val") + keys = sorted((set(train) | set(val)) & _ACC_KEYS) + if not keys: + return None + n = len(keys) + fig, axes = ps.new_figure("slide-16x9", title=f"{stage} accuracy", nrows=1, ncols=n, squeeze=False) + flat = axes.ravel() + for ax, key in zip(flat, keys): + if key in train: + ax.plot(table.epochs, table.columns[train[key]], label="train") + if key in val: + ax.plot(table.epochs, table.columns[val[key]], label="val") + ax.set_title(key, fontsize=8) + ax.set_xlabel("epoch") + ax.set_ylim(0, 1) + ps.style_legend(flat[0], title="series") + return fig + + +def _grad_norm_figure(table: MetricsTable): + import plotstyle as ps + + run_level = _run_level(table.columns) + if "grad_norm" not in run_level: + return None + fig, ax = ps.new_figure("thesis-single", title="gradient norm") + ax.plot(table.epochs, table.columns["grad_norm"], label="grad_norm") + for stage in _stages(table.columns): + train = _split(table.columns, stage, "train") + for key in ("grad_norm_d", "grad_norm_g", "grad_norm_type_slice", "grad_norm_cont_slice"): + if key in train: + ax.plot(table.epochs, table.columns[train[key]], label=f"{stage}/{key}") + ax.set_yscale("log") + ax.set_xlabel("epoch") + ax.set_ylabel("grad norm") + ps.style_legend(ax, title="series") + return fig + + +def _router_figure(table: MetricsTable, stage: str): + import plotstyle as ps + + router = _router(table.columns, stage) + if "entropy" not in router: + return None + fig, ax = ps.new_figure("thesis-single", title=f"{stage} router health") + ax.plot(table.epochs, table.columns[router["entropy"]], label="entropy", color="black") + ax.set_xlabel("epoch") + ax.set_ylabel("entropy [bits]") + ax2 = ax.twinx() + for key in ("util_min", "util_max", "util_std"): + if key in router: + ax2.plot(table.epochs, table.columns[router[key]], label=key, linestyle="--") + ax2.set_ylabel("expert utilization") + ax2.set_ylim(0, 1) + lines1, labels1 = ax.get_legend_handles_labels() + lines2, labels2 = ax2.get_legend_handles_labels() + ax.legend(lines1 + lines2, labels1 + labels2, loc="upper right", frameon=False, fontsize=7) + return fig + + +def _wgan_balance_figure(table: MetricsTable, stage: str): + import plotstyle as ps + + train = _split(table.columns, stage, "train") + keys = [k for k in _WGAN_BALANCE_KEYS if k in train] + if not keys: + return None + fig, ax = ps.new_figure("thesis-single", title=f"{stage} WGAN critic/generator balance") + for key in sorted(keys): + ax.plot(table.epochs, table.columns[train[key]], label=key) + ax.set_xlabel("epoch") + ax.set_ylabel("value") + ps.style_legend(ax, title="series") + return fig + + +def _throughput_figure(table: MetricsTable): + import plotstyle as ps + + run_level = _run_level(table.columns) + keys = [k for k in ("samples_per_sec", "gpu_mem_mb", "epoch_time_s") if k in run_level] + if not keys: + return None + fig, axes = ps.new_figure("slide-16x9", title="throughput / resources", nrows=1, ncols=len(keys), squeeze=False) + flat = axes.ravel() + for ax, key in zip(flat, keys): + ax.plot(table.epochs, table.columns[key]) + _mark_best(ax, table) + ax.set_title(key, fontsize=8) + ax.set_xlabel("epoch") + return fig + + +# --- entry point --------------------------------------------------------- + + +def render_metrics( + run_dir: str | Path, + out_dir: str | Path | None = None, + default_base: str | Path | None = None, +) -> list[Path]: + """`/metrics.csv` -> `/.pdf`. + + See `derive_metrics_dir` for how the plots directory is resolved. + """ + import matplotlib.pyplot as plt + import plotstyle as ps + + ps.use() + table = MetricsTable.load(Path(run_dir) / "metrics.csv") + plots_dir = derive_metrics_dir(run_dir, out_dir, default_base) + plots_dir.mkdir(parents=True, exist_ok=True) + + figures = [("overview", _overview_figure(table))] + for stage in _stages(table.columns): + figures.append((f"{stage}_loss", _loss_figure(table, stage))) + figures.append(("lr", _lr_figure(table))) + for stage in _stages(table.columns): + figures.append((f"{stage}_accuracy", _accuracy_figure(table, stage))) + figures.append(("grad_norm", _grad_norm_figure(table))) + for stage in _stages(table.columns): + figures.append((f"{stage}_router", _router_figure(table, stage))) + figures.append((f"{stage}_wgan_balance", _wgan_balance_figure(table, stage))) + figures.append(("throughput", _throughput_figure(table))) + + paths: list[Path] = [] + for name, fig in figures: + if fig is None: + continue + path = plots_dir / name + ps.savefig(fig, str(path), formats=("pdf",)) + plt.close(fig) + paths.append(path.with_suffix(".pdf")) + return paths diff --git a/tests/test_training_plots.py b/tests/test_training_plots.py new file mode 100644 index 0000000..e46d0bd --- /dev/null +++ b/tests/test_training_plots.py @@ -0,0 +1,333 @@ +"""Tests for giant.training.plots (gitea #75) — render smoke tests skipped +where plotstyle/LaTeX is unavailable, plus pure-function column-classification +coverage that needs neither.""" + +from __future__ import annotations + +import csv +from pathlib import Path + +import pytest + +pytest.importorskip("plotstyle") + +from giant.training import plots as plots_mod # noqa: E402 +from giant.training.plots import MetricsTable, derive_metrics_dir, render_metrics # noqa: E402 + +# --- fixtures ---------------------------------------------------------- + + +_RICH_HEADER = [ + "epoch", + "stage1/train/loss", + "stage1/train/loss_gen", + "stage1/train/nsec_acc", + "stage1/train/grad_norm", + "stage1/val/loss", + "stage1/val/loss_gen", + "stage1/val/nsec_acc", + "stage1/lr", + "stage1/router/entropy", + "stage1/router/util_min", + "stage1/router/util_max", + "stage1/router/util_std", + "stage2/train/d_loss", + "stage2/train/g_loss", + "stage2/train/wasserstein", + "stage2/train/gp_loss", + "stage2/train/loss_nsec", + "stage2/train/nsec_acc", + "stage2/train/grad_norm_d", + "stage2/train/grad_norm_g", + "stage2/lr", + "stage2/critic_lr", + "val/loss", + "val/marginal_kl", + "grad_norm", + "gpu_mem_mb", + "samples_per_sec", + "is_best", + "epoch_time_s", +] + +_RICH_ROWS = [ + [ + 1, + 1.0, + 0.8, + 0.5, + 1.2, + 0.9, + 0.7, + 0.6, + 3e-4, + 1.5, + 0.05, + 0.3, + 0.1, + -0.2, + 0.3, + 0.5, + 0.1, + 0.4, + 0.4, + 0.9, + 1.1, + 3e-4, + 1e-4, + 0.85, + 0.4, + 2.1, + 512.0, + 100.0, + 1, + 5.0, + ], + [ + 2, + 0.8, + 0.6, + 0.6, + 1.0, + 0.7, + 0.5, + 0.7, + 2e-4, + 1.6, + 0.06, + 0.28, + 0.09, + -0.1, + 0.25, + 0.4, + 0.09, + 0.3, + 0.5, + 0.8, + 1.0, + 2e-4, + 8e-5, + 0.7, + 0.35, + 1.9, + 520.0, + 105.0, + 0, + 5.1, + ], +] + +_MINIMAL_HEADER = [ + "epoch", + "stage1/train/loss", + "stage1/train/loss_gen", + "stage1/val/loss", + "stage1/val/loss_gen", + "stage1/lr", + "val/loss", + "grad_norm", + "gpu_mem_mb", + "samples_per_sec", + "is_best", + "epoch_time_s", +] + +_MINIMAL_ROWS = [ + [1, 1.0, 0.8, 0.9, 0.7, 3e-4, 0.85, 0.4, 0.0, 100.0, 0, 5.0], + [2, 0.8, 0.6, 0.7, 0.5, 2e-4, 0.7, 0.35, 0.0, 105.0, 1, 5.1], +] + + +def _write_csv(path: Path, header: list[str], rows: list[list]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(header) + writer.writerows(rows) + + +# --- MetricsTable -------------------------------------------------------- + + +def test_metrics_table_load_round_trips(tmp_path: Path): + csv_path = tmp_path / "metrics.csv" + _write_csv(csv_path, _MINIMAL_HEADER, _MINIMAL_ROWS) + table = MetricsTable.load(csv_path) + assert table.epochs == [1, 2] + assert table.columns["stage1/train/loss"] == [1.0, 0.8] + assert "epoch" not in table.columns + assert table.best_epochs() == [2] + + +def test_metrics_table_best_epochs_empty_without_is_best_column(): + table = MetricsTable(epochs=[1, 2], columns={"stage1/train/loss": [1.0, 0.5]}) + assert table.best_epochs() == [] + + +# --- column classification (pure functions, no matplotlib) -------------- + + +def _rich_columns() -> dict[str, list]: + return {name: [0.0] for name in _RICH_HEADER if name != "epoch"} + + +def test_stages_detects_only_stages_present(): + assert plots_mod._stages(_rich_columns()) == ["stage1", "stage2"] + assert plots_mod._stages({"stage2/train/loss": [0.0]}) == ["stage2"] + assert plots_mod._stages({"val/loss": [0.0]}) == [] + + +def test_split_matches_stage_and_split_prefix_only(): + cols = _rich_columns() + train = plots_mod._split(cols, "stage1", "train") + assert train == { + "loss": "stage1/train/loss", + "loss_gen": "stage1/train/loss_gen", + "nsec_acc": "stage1/train/nsec_acc", + "grad_norm": "stage1/train/grad_norm", + } + assert plots_mod._split(cols, "stage2", "val") == {} + + +def test_point_in_time_excludes_train_val_router(): + cols = _rich_columns() + pit = plots_mod._point_in_time(cols, "stage1") + assert pit == {"lr": "stage1/lr"} + pit2 = plots_mod._point_in_time(cols, "stage2") + assert pit2 == {"lr": "stage2/lr", "critic_lr": "stage2/critic_lr"} + + +def test_router_columns(): + cols = _rich_columns() + assert plots_mod._router(cols, "stage1") == { + "entropy": "stage1/router/entropy", + "util_min": "stage1/router/util_min", + "util_max": "stage1/router/util_max", + "util_std": "stage1/router/util_std", + } + assert plots_mod._router(cols, "stage2") == {} + + +def test_run_level_excludes_stage_prefixed_columns_including_val_loss_lookalike(): + cols = _rich_columns() + run_level = plots_mod._run_level(cols) + assert set(run_level) == { + "val/loss", + "val/marginal_kl", + "grad_norm", + "gpu_mem_mb", + "samples_per_sec", + "is_best", + "epoch_time_s", + } + # stage-prefixed "val/loss" lookalike (stage1/val/loss) must not leak in + assert "stage1/val/loss" not in run_level + + +def test_loss_keys_excludes_acc_and_wgan_and_grad_norm(): + train = {"loss": "x", "loss_gen": "x", "nsec_acc": "x", "grad_norm": "x", "d_loss": "x"} + val = {"loss": "x", "loss_gen": "x"} + assert plots_mod._loss_keys(train, val) == ["loss", "loss_gen"] + + +# --- derive_metrics_dir --------------------------------------------------- + + +def test_derive_metrics_dir_explicit_out_dir_wins(): + assert derive_metrics_dir("runs/my-run", out_dir="/somewhere") == Path("/somewhere") + + +def test_derive_metrics_dir_default_base(): + assert derive_metrics_dir("runs/my-run", default_base="/data/analysis_runs") == Path( + "/data/analysis_runs/metrics_my-run" + ) + + +def test_derive_metrics_dir_falls_back_to_cwd_analysis_runs(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + assert derive_metrics_dir("runs/my-run") == tmp_path / "analysis_runs" / "metrics_my-run" + + +# --- render_metrics end to end ------------------------------------------- + + +def _try_render(run_dir: Path, out_dir: Path) -> list[Path]: + try: + return render_metrics(run_dir, out_dir) + except RuntimeError as e: # LaTeX missing at render time + pytest.skip(f"LaTeX rendering unavailable: {e}") + + +def test_render_metrics_rich_run_produces_expected_plots_outside_run_dir(tmp_path: Path): + run_dir = tmp_path / "run" + out_dir = tmp_path / "out" + _write_csv(run_dir / "metrics.csv", _RICH_HEADER, _RICH_ROWS) + + paths = _try_render(run_dir, out_dir) + + names = {p.stem for p in paths} + assert names == { + "overview", + "stage1_loss", + "stage2_loss", + "lr", + "stage1_accuracy", + "stage2_accuracy", + "grad_norm", + "stage1_router", + "stage2_wgan_balance", + "throughput", + } + assert all(p.exists() for p in paths) + assert all(p.is_relative_to(out_dir) for p in paths) + # nothing written into the training run directory itself + assert not any(run_dir.rglob("*.pdf")) + + +def test_render_metrics_minimal_run_omits_router_wgan_accuracy(tmp_path: Path): + run_dir = tmp_path / "run" + out_dir = tmp_path / "out" + _write_csv(run_dir / "metrics.csv", _MINIMAL_HEADER, _MINIMAL_ROWS) + + paths = _try_render(run_dir, out_dir) + + names = {p.stem for p in paths} + assert names == {"overview", "stage1_loss", "lr", "grad_norm", "throughput"} + assert "stage1_accuracy" not in names + assert "stage1_router" not in names + assert "stage1_wgan_balance" not in names + + +def test_render_metrics_default_out_dir_uses_analysis_runs_convention(tmp_path: Path): + run_dir = tmp_path / "runs" / "my-run" + _write_csv(run_dir / "metrics.csv", _MINIMAL_HEADER, _MINIMAL_ROWS) + default_base = tmp_path / "analysis_runs" + + try: + paths = render_metrics(run_dir, default_base=default_base) + except RuntimeError as e: + pytest.skip(f"LaTeX rendering unavailable: {e}") + + assert paths + assert all(p.is_relative_to(default_base / "metrics_my-run") for p in paths) + + +# --- CLI ------------------------------------------------------------------- + + +def test_cli_analyze_metrics_smoke(tmp_path: Path): + from typer.testing import CliRunner + + from giant.cli import app + + run_dir = tmp_path / "run" + out_dir = tmp_path / "out" + _write_csv(run_dir / "metrics.csv", _MINIMAL_HEADER, _MINIMAL_ROWS) + + runner = CliRunner() + result = runner.invoke(app, ["analyze", "metrics", str(run_dir), "--out", str(out_dir)]) + + if result.exit_code != 0 and "LaTeX" in str(result.output): + pytest.skip("LaTeX rendering unavailable") + assert result.exit_code == 0, result.output + assert any(out_dir.glob("*.pdf"))