From bdebd83c8bdce10258d20c98f3da6c83001e130d Mon Sep 17 00:00:00 2001 From: Lars Bogner Date: Mon, 24 Aug 2026 10:55:15 +0200 Subject: [PATCH 1/2] Add giant analyze metrics plots for training progress (gitea #75) MetricsCollector writes one row per epoch to /metrics.csv, but nothing read or plotted it. giant/training/plots.py reads the CSV header dynamically (the column set varies by run: flow/ddpm vs wgan, routed vs not) and renders loss/lr/accuracy/grad-norm/router/wgan-balance/throughput plots with the same plotstyle conventions giant/analysis/render.py uses, skipping any figure whose columns aren't present for a given run. Wired up as `giant analyze metrics `, writing PDFs into the same gitignored analysis_runs/ directory `analyze prep`/`submit` already use (derive_metrics_dir mirrors derive_run_dir) rather than into the training run directory itself. --- giant/cli.py | 19 ++ giant/training/plots.py | 356 +++++++++++++++++++++++++++++++++++ tests/test_training_plots.py | 333 ++++++++++++++++++++++++++++++++ 3 files changed, 708 insertions(+) create mode 100644 giant/training/plots.py create mode 100644 tests/test_training_plots.py diff --git a/giant/cli.py b/giant/cli.py index 9f199b2..c2add9b 100644 --- a/giant/cli.py +++ b/giant/cli.py @@ -1644,6 +1644,25 @@ def analyze_render( typer.echo(f"rendered {len(pdfs)} plots → {Path(run_dir) / 'plots'}") +@analyze_app.command("metrics") +def analyze_metrics( + run_dir: Annotated[Path, typer.Argument(help="Run directory containing metrics.csv (from `giant train`)")], + out_dir: Annotated[ + Optional[Path], + typer.Option( + "--out", + "-o", + help="Override the output directory (default: /analysis_runs/metrics_)", + ), + ] = None, +) -> None: + """Render training-progress plots (loss/lr/accuracy/grad-norm/router/wgan/throughput) from /metrics.csv.""" + from giant.training.plots import render_metrics + + paths = render_metrics(run_dir, out_dir, default_base=Path.cwd() / "analysis_runs") + typer.echo(f"rendered {len(paths)} plots -> {paths[0].parent if paths else '(nothing to render)'}") + + @analyze_app.command("submit") def analyze_submit( rollout_yaml: Annotated[Path, typer.Argument(help="giant rollout YAML sidecar")], diff --git a/giant/training/plots.py b/giant/training/plots.py new file mode 100644 index 0000000..b15ef17 --- /dev/null +++ b/giant/training/plots.py @@ -0,0 +1,356 @@ +"""Training-progress plots from `/metrics.csv` (gitea #75). + +`MetricsCollector` (`giant.training.metrics`) writes one row per epoch with a +column set that varies by run — flow/ddpm vs wgan, routed vs not (see the +`MetricSpec` declarations in `giant.training.trainers`). This module reads +that header dynamically rather than hardcoding a column list, buckets columns +by the fixed naming convention `MetricsCollector` itself documents +(`/train/`, `/val/`, `/router/`, +`/` for point-in-time values, and an unprefixed run-level tail — +see `giant.training.metrics`'s module docstring), and renders one PDF per +applicable figure with the same `plotstyle` conventions +`giant.analysis.render` uses, for visual consistency with the +rollout-vs-reference plots. + +Unlike `giant.analysis`, there is no reduce/chunk/condor split here — the CSV +is tiny and this always runs as one local pass — but the CLI entry point +still lives under `giant analyze` (`analyze metrics`) as the shared home for +plotstyle-rendered diagnostics, and shares its `analysis_runs/` output +convention (see `derive_metrics_dir`) so training-progress plots don't get +written into the training run directory itself. +""" + +from __future__ import annotations + +import csv +import math +from dataclasses import dataclass +from pathlib import Path + +# Stage names are always exactly these two — hardcoded in +# `giant.training.trainers.build_stage_trainers` — so a column belongs to a +# stage iff it's prefixed by one of these, and everything else (bar `epoch`) +# is run-level. This is what makes dynamic header parsing tractable without +# needing to know the per-run metric keys themselves. +_STAGE_NAMES = ("stage1", "stage2") + +_ACC_KEYS = {"nsec_acc", "stop_acc", "type_acc"} +_WGAN_BALANCE_KEYS = {"d_loss", "g_loss", "wasserstein", "gp_loss"} +_ROUTER_KEYS = ("entropy", "util_min", "util_max", "util_std") + + +@dataclass +class MetricsTable: + """`/metrics.csv`, parsed with no hardcoded column list.""" + + epochs: list[int] + columns: dict[str, list[float]] + + @classmethod + def load(cls, path: str | Path) -> "MetricsTable": + with open(path, newline="") as f: + rows = list(csv.DictReader(f)) + epochs = [int(float(r["epoch"])) for r in rows] + fieldnames = rows[0].keys() if rows else [] + columns = {name: [float(r[name]) for r in rows] for name in fieldnames if name != "epoch"} + return cls(epochs=epochs, columns=columns) + + def best_epochs(self) -> list[int]: + is_best = self.columns.get("is_best") + if not is_best: + return [] + return [epoch for epoch, flag in zip(self.epochs, is_best) if flag] + + +# --- column classification -------------------------------------------------- + + +def _stages(columns: dict) -> list[str]: + return [s for s in _STAGE_NAMES if any(name.startswith(f"{s}/") for name in columns)] + + +def _split(columns: dict, stage: str, split: str) -> dict[str, str]: + prefix = f"{stage}/{split}/" + return {name[len(prefix) :]: name for name in columns if name.startswith(prefix)} + + +def _point_in_time(columns: dict, stage: str) -> dict[str, str]: + prefix = f"{stage}/" + out = {} + for name in columns: + if not name.startswith(prefix): + continue + rest = name[len(prefix) :] + head = rest.split("/", 1)[0] + if head not in ("train", "val", "router"): + out[rest] = name + return out + + +def _router(columns: dict, stage: str) -> dict[str, str]: + prefix = f"{stage}/router/" + return {name[len(prefix) :]: name for name in columns if name.startswith(prefix)} + + +def _run_level(columns: dict) -> dict[str, str]: + known_prefixes = tuple(f"{s}/" for s in _STAGE_NAMES) + return {name: name for name in columns if not name.startswith(known_prefixes)} + + +def _loss_keys(train: dict[str, str], val: dict[str, str]) -> list[str]: + keys = {k for k in train if k not in _ACC_KEYS and k not in _WGAN_BALANCE_KEYS and k != "grad_norm"} + keys |= {k for k in val if k not in _ACC_KEYS and k not in _WGAN_BALANCE_KEYS and k != "grad_norm"} + return sorted(keys) + + +# --- output location --------------------------------------------------------- + + +def derive_metrics_dir( + run_dir: str | Path, + out_dir: str | Path | None = None, + default_base: str | Path | None = None, +) -> Path: + """Plots output directory. + + Precedence: an explicit `out_dir` always wins. Otherwise + `default_base / f"metrics_{run_dir.name}"` (the CLI passes the repo's + gitignored `analysis_runs/`, matching `giant.analysis.condor.derive_run_dir`'s + convention) — training-progress plots live alongside rollout-vs-reference + analysis runs, not inside the training run directory itself. + """ + if out_dir is not None: + return Path(out_dir) + base = Path(default_base) if default_base is not None else Path.cwd() / "analysis_runs" + return base / f"metrics_{Path(run_dir).name}" + + +# --- figures ------------------------------------------------------------------ + + +def _mark_best(ax, table: MetricsTable) -> None: + for epoch in table.best_epochs(): + ax.axvline(epoch, color="grey", linestyle="--", linewidth=0.8, alpha=0.7) + + +def _overview_figure(table: MetricsTable): + import plotstyle as ps + + run_level = _run_level(table.columns) + if "val/loss" not in run_level: + return None + fig, ax = ps.new_figure("thesis-single", title="training overview") + ax.plot(table.epochs, table.columns["val/loss"], label="val/loss") + if "val/marginal_kl" in run_level: + kl = table.columns["val/marginal_kl"] + if any(math.isfinite(v) for v in kl): + ax.plot(table.epochs, kl, label="val/marginal_kl") + _mark_best(ax, table) + best = table.best_epochs() + if best: + idx = table.epochs.index(best[-1]) + ax.annotate( + f"best: epoch {best[-1]}\nval/loss={table.columns['val/loss'][idx]:.4g}", + xy=(best[-1], table.columns["val/loss"][idx]), + xytext=(0.98, 0.95), + textcoords="axes fraction", + ha="right", + va="top", + fontsize=8, + ) + ax.set_xlabel("epoch") + ax.set_ylabel("loss") + ps.style_legend(ax, title="series") + return fig + + +def _loss_figure(table: MetricsTable, stage: str): + import plotstyle as ps + + train = _split(table.columns, stage, "train") + val = _split(table.columns, stage, "val") + keys = _loss_keys(train, val) + if not keys: + return None + n = len(keys) + ncols = min(3, n) + nrows = (n + ncols - 1) // ncols + fig, axes = ps.new_figure( + "slide-16x9", + title=f"{stage} loss", + nrows=nrows, + ncols=ncols, + squeeze=False, + ) + flat = axes.ravel() + for ax, key in zip(flat, keys): + if key in train: + ax.plot(table.epochs, table.columns[train[key]], label="train") + if key in val: + ax.plot(table.epochs, table.columns[val[key]], label="val") + ax.set_yscale("log") + ax.set_title(key, fontsize=8) + ax.set_xlabel("epoch") + for j in range(n, len(flat)): + flat[j].set_visible(False) + ps.style_legend(flat[0], title="series") + return fig + + +def _lr_figure(table: MetricsTable): + import plotstyle as ps + + series: dict[str, str] = {} + for stage in _stages(table.columns): + for key, col in _point_in_time(table.columns, stage).items(): + series[f"{stage}/{key}"] = col + if not series: + return None + fig, ax = ps.new_figure("thesis-single", title="learning rate schedule") + for label, col in series.items(): + ax.plot(table.epochs, table.columns[col], label=label) + ax.set_xlabel("epoch") + ax.set_ylabel("learning rate") + ps.style_legend(ax, title="series") + return fig + + +def _accuracy_figure(table: MetricsTable, stage: str): + import plotstyle as ps + + train = _split(table.columns, stage, "train") + val = _split(table.columns, stage, "val") + keys = sorted((set(train) | set(val)) & _ACC_KEYS) + if not keys: + return None + n = len(keys) + fig, axes = ps.new_figure("slide-16x9", title=f"{stage} accuracy", nrows=1, ncols=n, squeeze=False) + flat = axes.ravel() + for ax, key in zip(flat, keys): + if key in train: + ax.plot(table.epochs, table.columns[train[key]], label="train") + if key in val: + ax.plot(table.epochs, table.columns[val[key]], label="val") + ax.set_title(key, fontsize=8) + ax.set_xlabel("epoch") + ax.set_ylim(0, 1) + ps.style_legend(flat[0], title="series") + return fig + + +def _grad_norm_figure(table: MetricsTable): + import plotstyle as ps + + run_level = _run_level(table.columns) + if "grad_norm" not in run_level: + return None + fig, ax = ps.new_figure("thesis-single", title="gradient norm") + ax.plot(table.epochs, table.columns["grad_norm"], label="grad_norm") + for stage in _stages(table.columns): + train = _split(table.columns, stage, "train") + for key in ("grad_norm_d", "grad_norm_g", "grad_norm_type_slice", "grad_norm_cont_slice"): + if key in train: + ax.plot(table.epochs, table.columns[train[key]], label=f"{stage}/{key}") + ax.set_yscale("log") + ax.set_xlabel("epoch") + ax.set_ylabel("grad norm") + ps.style_legend(ax, title="series") + return fig + + +def _router_figure(table: MetricsTable, stage: str): + import plotstyle as ps + + router = _router(table.columns, stage) + if "entropy" not in router: + return None + fig, ax = ps.new_figure("thesis-single", title=f"{stage} router health") + ax.plot(table.epochs, table.columns[router["entropy"]], label="entropy", color="black") + ax.set_xlabel("epoch") + ax.set_ylabel("entropy [bits]") + ax2 = ax.twinx() + for key in ("util_min", "util_max", "util_std"): + if key in router: + ax2.plot(table.epochs, table.columns[router[key]], label=key, linestyle="--") + ax2.set_ylabel("expert utilization") + ax2.set_ylim(0, 1) + lines1, labels1 = ax.get_legend_handles_labels() + lines2, labels2 = ax2.get_legend_handles_labels() + ax.legend(lines1 + lines2, labels1 + labels2, loc="upper right", frameon=False, fontsize=7) + return fig + + +def _wgan_balance_figure(table: MetricsTable, stage: str): + import plotstyle as ps + + train = _split(table.columns, stage, "train") + keys = [k for k in _WGAN_BALANCE_KEYS if k in train] + if not keys: + return None + fig, ax = ps.new_figure("thesis-single", title=f"{stage} WGAN critic/generator balance") + for key in sorted(keys): + ax.plot(table.epochs, table.columns[train[key]], label=key) + ax.set_xlabel("epoch") + ax.set_ylabel("value") + ps.style_legend(ax, title="series") + return fig + + +def _throughput_figure(table: MetricsTable): + import plotstyle as ps + + run_level = _run_level(table.columns) + keys = [k for k in ("samples_per_sec", "gpu_mem_mb", "epoch_time_s") if k in run_level] + if not keys: + return None + fig, axes = ps.new_figure("slide-16x9", title="throughput / resources", nrows=1, ncols=len(keys), squeeze=False) + flat = axes.ravel() + for ax, key in zip(flat, keys): + ax.plot(table.epochs, table.columns[key]) + _mark_best(ax, table) + ax.set_title(key, fontsize=8) + ax.set_xlabel("epoch") + return fig + + +# --- entry point --------------------------------------------------------- + + +def render_metrics( + run_dir: str | Path, + out_dir: str | Path | None = None, + default_base: str | Path | None = None, +) -> list[Path]: + """`/metrics.csv` -> `/.pdf`. + + See `derive_metrics_dir` for how the plots directory is resolved. + """ + import matplotlib.pyplot as plt + import plotstyle as ps + + ps.use() + table = MetricsTable.load(Path(run_dir) / "metrics.csv") + plots_dir = derive_metrics_dir(run_dir, out_dir, default_base) + plots_dir.mkdir(parents=True, exist_ok=True) + + figures = [("overview", _overview_figure(table))] + for stage in _stages(table.columns): + figures.append((f"{stage}_loss", _loss_figure(table, stage))) + figures.append(("lr", _lr_figure(table))) + for stage in _stages(table.columns): + figures.append((f"{stage}_accuracy", _accuracy_figure(table, stage))) + figures.append(("grad_norm", _grad_norm_figure(table))) + for stage in _stages(table.columns): + figures.append((f"{stage}_router", _router_figure(table, stage))) + figures.append((f"{stage}_wgan_balance", _wgan_balance_figure(table, stage))) + figures.append(("throughput", _throughput_figure(table))) + + paths: list[Path] = [] + for name, fig in figures: + if fig is None: + continue + path = plots_dir / name + ps.savefig(fig, str(path), formats=("pdf",)) + plt.close(fig) + paths.append(path.with_suffix(".pdf")) + return paths diff --git a/tests/test_training_plots.py b/tests/test_training_plots.py new file mode 100644 index 0000000..e46d0bd --- /dev/null +++ b/tests/test_training_plots.py @@ -0,0 +1,333 @@ +"""Tests for giant.training.plots (gitea #75) — render smoke tests skipped +where plotstyle/LaTeX is unavailable, plus pure-function column-classification +coverage that needs neither.""" + +from __future__ import annotations + +import csv +from pathlib import Path + +import pytest + +pytest.importorskip("plotstyle") + +from giant.training import plots as plots_mod # noqa: E402 +from giant.training.plots import MetricsTable, derive_metrics_dir, render_metrics # noqa: E402 + +# --- fixtures ---------------------------------------------------------- + + +_RICH_HEADER = [ + "epoch", + "stage1/train/loss", + "stage1/train/loss_gen", + "stage1/train/nsec_acc", + "stage1/train/grad_norm", + "stage1/val/loss", + "stage1/val/loss_gen", + "stage1/val/nsec_acc", + "stage1/lr", + "stage1/router/entropy", + "stage1/router/util_min", + "stage1/router/util_max", + "stage1/router/util_std", + "stage2/train/d_loss", + "stage2/train/g_loss", + "stage2/train/wasserstein", + "stage2/train/gp_loss", + "stage2/train/loss_nsec", + "stage2/train/nsec_acc", + "stage2/train/grad_norm_d", + "stage2/train/grad_norm_g", + "stage2/lr", + "stage2/critic_lr", + "val/loss", + "val/marginal_kl", + "grad_norm", + "gpu_mem_mb", + "samples_per_sec", + "is_best", + "epoch_time_s", +] + +_RICH_ROWS = [ + [ + 1, + 1.0, + 0.8, + 0.5, + 1.2, + 0.9, + 0.7, + 0.6, + 3e-4, + 1.5, + 0.05, + 0.3, + 0.1, + -0.2, + 0.3, + 0.5, + 0.1, + 0.4, + 0.4, + 0.9, + 1.1, + 3e-4, + 1e-4, + 0.85, + 0.4, + 2.1, + 512.0, + 100.0, + 1, + 5.0, + ], + [ + 2, + 0.8, + 0.6, + 0.6, + 1.0, + 0.7, + 0.5, + 0.7, + 2e-4, + 1.6, + 0.06, + 0.28, + 0.09, + -0.1, + 0.25, + 0.4, + 0.09, + 0.3, + 0.5, + 0.8, + 1.0, + 2e-4, + 8e-5, + 0.7, + 0.35, + 1.9, + 520.0, + 105.0, + 0, + 5.1, + ], +] + +_MINIMAL_HEADER = [ + "epoch", + "stage1/train/loss", + "stage1/train/loss_gen", + "stage1/val/loss", + "stage1/val/loss_gen", + "stage1/lr", + "val/loss", + "grad_norm", + "gpu_mem_mb", + "samples_per_sec", + "is_best", + "epoch_time_s", +] + +_MINIMAL_ROWS = [ + [1, 1.0, 0.8, 0.9, 0.7, 3e-4, 0.85, 0.4, 0.0, 100.0, 0, 5.0], + [2, 0.8, 0.6, 0.7, 0.5, 2e-4, 0.7, 0.35, 0.0, 105.0, 1, 5.1], +] + + +def _write_csv(path: Path, header: list[str], rows: list[list]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(header) + writer.writerows(rows) + + +# --- MetricsTable -------------------------------------------------------- + + +def test_metrics_table_load_round_trips(tmp_path: Path): + csv_path = tmp_path / "metrics.csv" + _write_csv(csv_path, _MINIMAL_HEADER, _MINIMAL_ROWS) + table = MetricsTable.load(csv_path) + assert table.epochs == [1, 2] + assert table.columns["stage1/train/loss"] == [1.0, 0.8] + assert "epoch" not in table.columns + assert table.best_epochs() == [2] + + +def test_metrics_table_best_epochs_empty_without_is_best_column(): + table = MetricsTable(epochs=[1, 2], columns={"stage1/train/loss": [1.0, 0.5]}) + assert table.best_epochs() == [] + + +# --- column classification (pure functions, no matplotlib) -------------- + + +def _rich_columns() -> dict[str, list]: + return {name: [0.0] for name in _RICH_HEADER if name != "epoch"} + + +def test_stages_detects_only_stages_present(): + assert plots_mod._stages(_rich_columns()) == ["stage1", "stage2"] + assert plots_mod._stages({"stage2/train/loss": [0.0]}) == ["stage2"] + assert plots_mod._stages({"val/loss": [0.0]}) == [] + + +def test_split_matches_stage_and_split_prefix_only(): + cols = _rich_columns() + train = plots_mod._split(cols, "stage1", "train") + assert train == { + "loss": "stage1/train/loss", + "loss_gen": "stage1/train/loss_gen", + "nsec_acc": "stage1/train/nsec_acc", + "grad_norm": "stage1/train/grad_norm", + } + assert plots_mod._split(cols, "stage2", "val") == {} + + +def test_point_in_time_excludes_train_val_router(): + cols = _rich_columns() + pit = plots_mod._point_in_time(cols, "stage1") + assert pit == {"lr": "stage1/lr"} + pit2 = plots_mod._point_in_time(cols, "stage2") + assert pit2 == {"lr": "stage2/lr", "critic_lr": "stage2/critic_lr"} + + +def test_router_columns(): + cols = _rich_columns() + assert plots_mod._router(cols, "stage1") == { + "entropy": "stage1/router/entropy", + "util_min": "stage1/router/util_min", + "util_max": "stage1/router/util_max", + "util_std": "stage1/router/util_std", + } + assert plots_mod._router(cols, "stage2") == {} + + +def test_run_level_excludes_stage_prefixed_columns_including_val_loss_lookalike(): + cols = _rich_columns() + run_level = plots_mod._run_level(cols) + assert set(run_level) == { + "val/loss", + "val/marginal_kl", + "grad_norm", + "gpu_mem_mb", + "samples_per_sec", + "is_best", + "epoch_time_s", + } + # stage-prefixed "val/loss" lookalike (stage1/val/loss) must not leak in + assert "stage1/val/loss" not in run_level + + +def test_loss_keys_excludes_acc_and_wgan_and_grad_norm(): + train = {"loss": "x", "loss_gen": "x", "nsec_acc": "x", "grad_norm": "x", "d_loss": "x"} + val = {"loss": "x", "loss_gen": "x"} + assert plots_mod._loss_keys(train, val) == ["loss", "loss_gen"] + + +# --- derive_metrics_dir --------------------------------------------------- + + +def test_derive_metrics_dir_explicit_out_dir_wins(): + assert derive_metrics_dir("runs/my-run", out_dir="/somewhere") == Path("/somewhere") + + +def test_derive_metrics_dir_default_base(): + assert derive_metrics_dir("runs/my-run", default_base="/data/analysis_runs") == Path( + "/data/analysis_runs/metrics_my-run" + ) + + +def test_derive_metrics_dir_falls_back_to_cwd_analysis_runs(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + assert derive_metrics_dir("runs/my-run") == tmp_path / "analysis_runs" / "metrics_my-run" + + +# --- render_metrics end to end ------------------------------------------- + + +def _try_render(run_dir: Path, out_dir: Path) -> list[Path]: + try: + return render_metrics(run_dir, out_dir) + except RuntimeError as e: # LaTeX missing at render time + pytest.skip(f"LaTeX rendering unavailable: {e}") + + +def test_render_metrics_rich_run_produces_expected_plots_outside_run_dir(tmp_path: Path): + run_dir = tmp_path / "run" + out_dir = tmp_path / "out" + _write_csv(run_dir / "metrics.csv", _RICH_HEADER, _RICH_ROWS) + + paths = _try_render(run_dir, out_dir) + + names = {p.stem for p in paths} + assert names == { + "overview", + "stage1_loss", + "stage2_loss", + "lr", + "stage1_accuracy", + "stage2_accuracy", + "grad_norm", + "stage1_router", + "stage2_wgan_balance", + "throughput", + } + assert all(p.exists() for p in paths) + assert all(p.is_relative_to(out_dir) for p in paths) + # nothing written into the training run directory itself + assert not any(run_dir.rglob("*.pdf")) + + +def test_render_metrics_minimal_run_omits_router_wgan_accuracy(tmp_path: Path): + run_dir = tmp_path / "run" + out_dir = tmp_path / "out" + _write_csv(run_dir / "metrics.csv", _MINIMAL_HEADER, _MINIMAL_ROWS) + + paths = _try_render(run_dir, out_dir) + + names = {p.stem for p in paths} + assert names == {"overview", "stage1_loss", "lr", "grad_norm", "throughput"} + assert "stage1_accuracy" not in names + assert "stage1_router" not in names + assert "stage1_wgan_balance" not in names + + +def test_render_metrics_default_out_dir_uses_analysis_runs_convention(tmp_path: Path): + run_dir = tmp_path / "runs" / "my-run" + _write_csv(run_dir / "metrics.csv", _MINIMAL_HEADER, _MINIMAL_ROWS) + default_base = tmp_path / "analysis_runs" + + try: + paths = render_metrics(run_dir, default_base=default_base) + except RuntimeError as e: + pytest.skip(f"LaTeX rendering unavailable: {e}") + + assert paths + assert all(p.is_relative_to(default_base / "metrics_my-run") for p in paths) + + +# --- CLI ------------------------------------------------------------------- + + +def test_cli_analyze_metrics_smoke(tmp_path: Path): + from typer.testing import CliRunner + + from giant.cli import app + + run_dir = tmp_path / "run" + out_dir = tmp_path / "out" + _write_csv(run_dir / "metrics.csv", _MINIMAL_HEADER, _MINIMAL_ROWS) + + runner = CliRunner() + result = runner.invoke(app, ["analyze", "metrics", str(run_dir), "--out", str(out_dir)]) + + if result.exit_code != 0 and "LaTeX" in str(result.output): + pytest.skip("LaTeX rendering unavailable") + assert result.exit_code == 0, result.output + assert any(out_dir.glob("*.pdf")) From 7560e2bff076c00babc80c4fb1fa44402a6370f1 Mon Sep 17 00:00:00 2001 From: Lars Bogner Date: Mon, 24 Aug 2026 11:15:51 +0200 Subject: [PATCH 2/2] Fix LaTeX-unavailable skip check in analyze metrics smoke test CliRunner stores an uncaught exception in result.exception, not result.output, so the skip condition never matched and the test failed outright on CI machines without LaTeX installed. Co-Authored-By: Claude Sonnet 5 --- tests/test_training_plots.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_training_plots.py b/tests/test_training_plots.py index e46d0bd..b095e70 100644 --- a/tests/test_training_plots.py +++ b/tests/test_training_plots.py @@ -327,7 +327,7 @@ def test_cli_analyze_metrics_smoke(tmp_path: Path): runner = CliRunner() result = runner.invoke(app, ["analyze", "metrics", str(run_dir), "--out", str(out_dir)]) - if result.exit_code != 0 and "LaTeX" in str(result.output): + if result.exit_code != 0 and "latex" in (str(result.output) + str(result.exception)).lower(): pytest.skip("LaTeX rendering unavailable") - assert result.exit_code == 0, result.output + assert result.exit_code == 0, result.output or result.exception assert any(out_dir.glob("*.pdf"))