Add KL divergence to marginal validation and hook it into the training loop
validate_marginals now estimates a per-dimension KL(real || generated) via a shared histogram, alongside the existing mean/std comparison, so distribution-shape drift shows up even when the first two moments match. Wire it into giant/train.py: every validate_every epochs (default 10, 0 disables), the training loop runs validate_marginals against val_loader and prints the table. validate_every flows through DEFAULT_CONFIG/config.toml and is exposed as --validate-every on both giant train and scripts/train.py. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+3
-1
@@ -20,6 +20,8 @@ def main() -> None:
|
||||
parser.add_argument("--emb-dim", type=int)
|
||||
parser.add_argument("--val-fraction", type=float)
|
||||
parser.add_argument("--seed", type=int, help="Random seed for reproducibility")
|
||||
parser.add_argument("--validate-every", type=int,
|
||||
help="Run marginal+KL validation every N epochs (0 disables)")
|
||||
parser.add_argument("--shuffle-buffer", type=int, default=65536,
|
||||
help="Rows held in RAM for shuffling per worker (default: 65536)")
|
||||
parser.add_argument("--out", default=None, help="Checkpoint output directory (default: auto from hyperparams)")
|
||||
@@ -31,7 +33,7 @@ def main() -> None:
|
||||
cli_train = {k: v for k, v in {
|
||||
"mode": args.mode, "epochs": args.epochs, "batch_size": args.batch_size,
|
||||
"lr": args.lr, "val_fraction": args.val_fraction, "num_workers": args.num_workers,
|
||||
"seed": args.seed,
|
||||
"seed": args.seed, "validate_every": args.validate_every,
|
||||
}.items() if v is not None}
|
||||
cli_model = {k: v for k, v in {
|
||||
"hidden_dim": args.hidden_dim, "n_blocks": args.n_blocks, "emb_dim": args.emb_dim,
|
||||
|
||||
Reference in New Issue
Block a user