Add KL divergence to marginal validation and hook it into the training loop

validate_marginals now estimates a per-dimension KL(real || generated) via
a shared histogram, alongside the existing mean/std comparison, so
distribution-shape drift shows up even when the first two moments match.

Wire it into giant/train.py: every validate_every epochs (default 10, 0
disables), the training loop runs validate_marginals against val_loader and
prints the table. validate_every flows through DEFAULT_CONFIG/config.toml
and is exposed as --validate-every on both giant train and scripts/train.py.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-18 13:45:12 +02:00
parent 43634ef77a
commit 893d91e749
6 changed files with 47 additions and 6 deletions
+3 -1
View File
@@ -20,6 +20,8 @@ def main() -> None:
parser.add_argument("--emb-dim", type=int)
parser.add_argument("--val-fraction", type=float)
parser.add_argument("--seed", type=int, help="Random seed for reproducibility")
parser.add_argument("--validate-every", type=int,
help="Run marginal+KL validation every N epochs (0 disables)")
parser.add_argument("--shuffle-buffer", type=int, default=65536,
help="Rows held in RAM for shuffling per worker (default: 65536)")
parser.add_argument("--out", default=None, help="Checkpoint output directory (default: auto from hyperparams)")
@@ -31,7 +33,7 @@ def main() -> None:
cli_train = {k: v for k, v in {
"mode": args.mode, "epochs": args.epochs, "batch_size": args.batch_size,
"lr": args.lr, "val_fraction": args.val_fraction, "num_workers": args.num_workers,
"seed": args.seed,
"seed": args.seed, "validate_every": args.validate_every,
}.items() if v is not None}
cli_model = {k: v for k, v in {
"hidden_dim": args.hidden_dim, "n_blocks": args.n_blocks, "emb_dim": args.emb_dim,