Add opt-in straight-through Gumbel-softmax combine weights to MoE router #26

Merged
lars merged 4 commits from feat/router-gumbel-softmax into master 2026-07-30 17:05:09 +02:00
3 changed files with 67 additions and 0 deletions
Showing only changes of commit af2ee7c7ce - Show all commits
@@ -0,0 +1,22 @@
[train]
mode = "flow"
epochs = 30
lr = 3e-4
warmup_epochs = 3
val_fraction = 0.1
num_workers = 4
[model]
conditioning = "physical"
dropout = 0.0
[model.router]
enabled = true
type = "energy"
n_experts = 10
expert_hidden_dim = 128
expert_n_blocks = 4
temperature = 0.05
lambda_balance = 0.035
learn_centers = true
gumbel = true
@@ -0,0 +1,23 @@
[train]
mode = "flow"
epochs = 30
lr = 3e-4
warmup_epochs = 3
val_fraction = 0.1
num_workers = 4
[model]
conditioning = "physical"
dropout = 0.0
[model.router]
enabled = true
type = "energy"
n_experts = 10
expert_hidden_dim = 128
expert_n_blocks = 4
temperature = 0.05
lambda_balance = 0.035
learn_centers = true
learn_temperature = true
gumbel = true
@@ -0,0 +1,22 @@
[train]
mode = "flow"
epochs = 30
lr = 3e-4
warmup_epochs = 3
val_fraction = 0.1
num_workers = 4
[model]
conditioning = "physical"
dropout = 0.0
[model.router]
enabled = true
type = "energy"
n_experts = 10
expert_hidden_dim = 128
expert_n_blocks = 4
temperature = 0.05
lambda_balance = 0.035
learn_centers = false
gumbel = true