router: seed EnergyRouter centers from data quantiles instead of a fixed linspace
CI / Lint (ruff check) (push) Successful in 1m1s
CI / Format (ruff format) (push) Successful in 1m6s
CI / Type check (ty) (push) Successful in 59s
CI / Tests (push) Successful in 1m45s
CI / Lint (ruff check) (pull_request) Successful in 1m4s
CI / Format (ruff format) (pull_request) Successful in 1m5s
CI / Type check (ty) (pull_request) Successful in 1m4s
CI / Tests (pull_request) Successful in 1m55s
CI / Bump version, build & publish wheel (push) Has been skipped
CI / Bump version, build & publish wheel (pull_request) Has been skipped
CI / Lint (ruff check) (push) Successful in 1m1s
CI / Format (ruff format) (push) Successful in 1m6s
CI / Type check (ty) (push) Successful in 59s
CI / Tests (push) Successful in 1m45s
CI / Lint (ruff check) (pull_request) Successful in 1m4s
CI / Format (ruff format) (pull_request) Successful in 1m5s
CI / Type check (ty) (pull_request) Successful in 1m4s
CI / Tests (pull_request) Successful in 1m55s
CI / Bump version, build & publish wheel (push) Has been skipped
CI / Bump version, build & publish wheel (pull_request) Has been skipped
The 2026-07-22 rollout benchmark's router_gating diagnostic showed the 10-expert EnergyRouter's default linspace(-2, 2, n_experts) init assumes a roughly uniform z-normalized energy distribution, leaving experts heavily overlapping instead of partitioning the range. Add an optional centers_init kwarg (backward compatible, defaults to the old linspace) and have giant train estimate it from a reservoir sample of the real energy column, collected during the existing normalizer-fitting pass.
This commit is contained in:
@@ -97,6 +97,42 @@ def test_build_router_ignores_unrecognized_kwargs():
|
||||
assert router.temperature == 0.3
|
||||
|
||||
|
||||
def test_energy_router_default_centers_are_linspace():
|
||||
router = EnergyRouter(n_experts=4)
|
||||
torch.testing.assert_close(router.centers, torch.linspace(-2.0, 2.0, 4))
|
||||
|
||||
|
||||
def test_energy_router_centers_init_overrides_default():
|
||||
centers_init = [-1.0, 0.0, 0.5, 3.0]
|
||||
router = EnergyRouter(n_experts=4, centers_init=centers_init)
|
||||
torch.testing.assert_close(router.centers, torch.tensor(centers_init))
|
||||
|
||||
|
||||
def test_energy_router_centers_init_wrong_length_raises():
|
||||
try:
|
||||
EnergyRouter(n_experts=4, centers_init=[0.0, 1.0])
|
||||
except ValueError:
|
||||
return
|
||||
raise AssertionError("expected ValueError for centers_init length mismatch")
|
||||
|
||||
|
||||
def test_energy_router_centers_init_respects_learn_centers_flag():
|
||||
learned = EnergyRouter(
|
||||
n_experts=3, centers_init=[-1.0, 0.0, 1.0], learn_centers=True
|
||||
)
|
||||
fixed = EnergyRouter(
|
||||
n_experts=3, centers_init=[-1.0, 0.0, 1.0], learn_centers=False
|
||||
)
|
||||
assert isinstance(learned.centers, torch.nn.Parameter)
|
||||
assert not isinstance(fixed.centers, torch.nn.Parameter)
|
||||
|
||||
|
||||
def test_build_router_threads_centers_init_through_energy_router():
|
||||
centers_init = [-1.5, -0.5, 0.5, 1.5]
|
||||
router = build_router("energy", 4, centers_init=centers_init)
|
||||
torch.testing.assert_close(router.centers, torch.tensor(centers_init))
|
||||
|
||||
|
||||
def test_build_router_unknown_type_raises():
|
||||
try:
|
||||
build_router("nonexistent", 4)
|
||||
|
||||
Reference in New Issue
Block a user