Update stats reporter PPO hyperparam keys

Author: relh

metta/rl/training/stats_reporter.py

@@ -529,17 +529,20 @@ def _collect_hyperparameters(
 
         losses = getattr(trainer_cfg, "losses", None)
         loss_configs = getattr(losses, "loss_configs", {}) if losses else {}
-        ppo_cfg = loss_configs.get("ppo") if isinstance(loss_configs, dict) else None
-        if ppo_cfg is not None:
-            for attr in (
-                "clip_coef",
-                "vf_clip_coef",
-                "ent_coef",
-                "l2_reg_loss_coef",
-                "l2_init_loss_coef",
-            ):
-                value = getattr(ppo_cfg, attr, None)
-                if value is None:
-                    continue
-                hyperparameters[f"ppo_{attr}"] = value
+        if isinstance(loss_configs, dict):
+            ppo_actor_cfg = loss_configs.get("ppo_actor")
+            if ppo_actor_cfg is not None:
+                for attr in ("clip_coef", "ent_coef", "norm_adv", "target_kl"):
+                    value = getattr(ppo_actor_cfg, attr, None)
+                    if value is None:
+                        continue
+                    hyperparameters[f"ppo_actor_{attr}"] = value
+
+            ppo_critic_cfg = loss_configs.get("ppo_critic")
+            if ppo_critic_cfg is not None:
+                for attr in ("vf_coef", "vf_clip_coef", "clip_vloss"):
+                    value = getattr(ppo_critic_cfg, attr, None)
+                    if value is None:
+                        continue
+                    hyperparameters[f"ppo_critic_{attr}"] = value
         return hyperparameters

tests/rl/test_stats_reporter_defaults.py

@@ -24,7 +24,10 @@ def _reporter(existing_heart: float | None = None) -> StatsReporter:
     reporter._context = SimpleNamespace(  # type: ignore[attr-defined, assignment]
         config=SimpleNamespace(
             optimizer=SimpleNamespace(learning_rate=0.0, type="adam"),
-            losses=SimpleNamespace(ppo=SimpleNamespace(enabled=False)),
+            losses=SimpleNamespace(
+                ppo_actor=SimpleNamespace(enabled=False),
+                ppo_critic=SimpleNamespace(enabled=False),
+            ),
         ),
         stopwatch=timer,
         experience=SimpleNamespace(stats=lambda: {}),