From f29441bf099735946e9b8ea967b7dba7b0e26a6b Mon Sep 17 00:00:00 2001 From: Gordie Date: Tue, 30 Jun 2026 18:51:10 -0500 Subject: [PATCH] fix(webui): split MoA picker and Copilot catalog fixes - Keep resolve_moa_preset optional so older hermes-agent installs still degrade gracefully. - Surface MoA presets in the WebUI model picker as a virtual provider. - Keep configured Copilot per-model settings from collapsing the built-in catalog. - Refresh the Copilot static fallback used when the live catalog probe misses. Verification: ./scripts/test.sh tests/test_issue5057_moa_webui_route.py tests/test_copilot_provider_model_settings_not_allowlist.py tests/test_moa_model_picker_provider.py -q 11 passed, 1 skipped --- api/commands.py | 42 +++++-- api/config.py | 63 ++++++++-- api/routes.py | 15 +++ ...t_provider_model_settings_not_allowlist.py | 79 +++++++++++++ tests/test_moa_model_picker_provider.py | 110 ++++++++++++++++++ 5 files changed, 291 insertions(+), 18 deletions(-) create mode 100644 tests/test_copilot_provider_model_settings_not_allowlist.py create mode 100644 tests/test_moa_model_picker_provider.py diff --git a/api/commands.py b/api/commands.py index d81a2cfd..a70acd20 100644 --- a/api/commands.py +++ b/api/commands.py @@ -385,20 +385,48 @@ def _run_credits_command() -> str: return "\n".join(lines) -def resolve_moa_config() -> dict: +def _load_config_for_moa_resolution() -> dict: + from hermes_cli.config import load_config + + cfg = load_config() + return cfg if isinstance(cfg, dict) else {} + + +def resolve_moa_config(preset: str | None = None) -> dict: try: from hermes_cli.moa_config import moa_usage, normalize_moa_config except ImportError as exc: raise RuntimeError("MoA runtime unavailable (hermes-agent not installed or too old)") from exc try: - from hermes_cli.config import load_config - cfg = load_config() - moa_cfg = normalize_moa_config(cfg.get("moa") if isinstance(cfg, dict) else {}) + from hermes_cli.moa_config import resolve_moa_preset + except ImportError: + resolve_moa_preset = None + + try: + cfg = _load_config_for_moa_resolution() + moa_raw = cfg.get("moa") if isinstance(cfg, dict) else {} + moa_cfg = normalize_moa_config(moa_raw) except Exception: + moa_raw = {} moa_cfg = normalize_moa_config({}) - moa_cfg["preset"] = moa_cfg["default_preset"] - moa_cfg["usage"] = moa_usage() - return moa_cfg + + preset_name = str(preset or moa_cfg.get("default_preset") or "default").strip() + if preset_name not in (moa_cfg.get("presets") or {}): + preset_name = str(moa_cfg.get("default_preset") or "default") + + selected = {} + if resolve_moa_preset is not None: + try: + selected = resolve_moa_preset(moa_raw, preset_name) + except Exception: + selected = resolve_moa_preset({}, None) + preset_name = "default" + + resolved = dict(moa_cfg) + resolved.update(selected) + resolved["preset"] = preset_name + resolved["usage"] = moa_usage() + return resolved def execute_plugin_command(command: str) -> str: diff --git a/api/config.py b/api/config.py index f8577a8c..8feb0980 100644 --- a/api/config.py +++ b/api/config.py @@ -1139,6 +1139,7 @@ _PROVIDER_DISPLAY = { "openai-codex": "OpenAI Codex", "xai-oauth": "xAI Grok OAuth", "copilot": "GitHub Copilot", + "moa": "Mixture of Agents", "cursor-acp": "Cursor ACP", "zai": "Z.AI / GLM", "kimi-coding": "Kimi / Moonshot", @@ -1678,15 +1679,30 @@ _PROVIDER_MODELS = { {"id": "MiniMax-M2.7", "label": "MiniMax M2.7"}, ], # GitHub Copilot — model IDs served via the Copilot API + # Fallback ONLY — the live GitHub Copilot catalog + # (hermes_cli.models.provider_model_ids("copilot")) is authoritative and is + # tried first by _read_live_provider_model_ids(). This static list is the + # safety net shown when the live probe fails (cold start / token blip). Keep + # it in sync with the real integrator allowlist so a probe miss never renders + # legacy junk (GPT-4o / gpt-3.5-turbo). Last synced 2026-06-30 from the live + # copilot-4-cli catalog (16 models). "copilot": [ + {"id": "claude-opus-4.8", "label": "Claude Opus 4.8"}, + {"id": "claude-opus-4.7", "label": "Claude Opus 4.7"}, + {"id": "claude-opus-4.6", "label": "Claude Opus 4.6"}, + {"id": "claude-sonnet-5", "label": "Claude Sonnet 5"}, + {"id": "claude-sonnet-4.6", "label": "Claude Sonnet 4.6"}, + {"id": "claude-sonnet-4.5", "label": "Claude Sonnet 4.5"}, + {"id": "claude-haiku-4.5", "label": "Claude Haiku 4.5"}, {"id": "gpt-5.5", "label": "GPT-5.5"}, - {"id": "gpt-5.5-mini", "label": "GPT-5.5 Mini"}, {"id": "gpt-5.4", "label": "GPT-5.4"}, {"id": "gpt-5.4-mini", "label": "GPT-5.4 Mini"}, - {"id": "gpt-4o", "label": "GPT-4o"}, - {"id": "claude-opus-4.6", "label": "Claude Opus 4.6"}, - {"id": "claude-sonnet-4.6", "label": "Claude Sonnet 4.6"}, - {"id": "gemini-3-flash-preview", "label": "Gemini 3 Flash Preview"}, + {"id": "gpt-5.3-codex", "label": "GPT-5.3 Codex"}, + {"id": "gpt-5-mini", "label": "GPT-5 Mini"}, + {"id": "gemini-3.1-pro-preview", "label": "Gemini 3.1 Pro Preview"}, + {"id": "gemini-3.5-flash", "label": "Gemini 3.5 Flash"}, + {"id": "gemini-2.5-pro", "label": "Gemini 2.5 Pro"}, + {"id": "mai-code-1-flash-picker", "label": "MAI Code 1 Flash"}, ], # Cursor ACP — models served via Cursor CLI agent acp "cursor-acp": [ @@ -5964,7 +5980,13 @@ def get_available_models(*, prefer_cache: bool = False, force_refresh: bool = Fa or _is_plugin_model_provider(_canonical) ) _is_provider_config = isinstance(_provider_cfg, dict) - if not (_is_known_provider or _is_provider_config): + _has_provider_route = False + if _is_provider_config: + _has_provider_route = any( + str(_provider_cfg.get(_route_key) or "").strip() + for _route_key in ("api", "base_url", "api_key", "key_env") + ) + if not (_is_known_provider or _has_provider_route): continue _canonical_to_raw_provider_key.setdefault(_canonical, _pid_key) @@ -6401,6 +6423,16 @@ def get_available_models(*, prefer_cache: bool = False, force_refresh: bool = Fa _canonicalised_detected.add(_c) detected_providers = _canonicalised_detected + try: + _moa_cfg = cfg.get("moa") if isinstance(cfg, dict) else None + if isinstance(_moa_cfg, dict): + _moa_enabled = bool(_moa_cfg.get("enabled", True)) + _moa_presets = _moa_cfg.get("presets") + if _moa_enabled and isinstance(_moa_presets, dict) and _moa_presets: + detected_providers.add("moa") + except Exception: + logger.debug("Failed to inspect MoA presets for model picker", exc_info=True) + # 5. Build model groups if detected_providers: _picker_selected_model_id = ( @@ -6740,6 +6772,7 @@ def get_available_models(*, prefer_cache: bool = False, force_refresh: bool = Fa _append_picker_group(provider_name, pid, raw_models) elif ( pid in _PROVIDER_MODELS + or pid in _PROVIDER_DISPLAY or pid in _canonical_to_raw_provider_key or _is_plugin_model_provider(pid) ): @@ -6753,11 +6786,19 @@ def get_available_models(*, prefer_cache: bool = False, force_refresh: bool = Fa raw_models = [] # User-configured model allowlists are explicit local - # source-of-truth and should still beat auto-discovery. - # Otherwise, ask Hermes CLI first so WebUI tracks the same - # live catalog as the agent/CLI picker; WebUI's static - # _PROVIDER_MODELS table is now a fallback only (#1240). - if isinstance(provider_cfg, dict) and "models" in provider_cfg: + # source-of-truth for custom/plugin providers, but built-in + # Hermes providers also use providers..models as a + # per-model settings map (reasoning_effort, limits, etc.). + # Treating that settings map as an allowlist collapsed the + # Copilot picker to whichever model had local settings. + # Built-ins should ask Hermes CLI for the live catalog first; + # WebUI's static _PROVIDER_MODELS table remains fallback only. + _is_builtin_catalog_provider = pid in _PROVIDER_MODELS + if ( + not _is_builtin_catalog_provider + and isinstance(provider_cfg, dict) + and "models" in provider_cfg + ): cfg_models = provider_cfg["models"] if isinstance(cfg_models, dict): raw_models = [{"id": k, "label": k} for k in cfg_models.keys()] diff --git a/api/routes.py b/api/routes.py index 74a978fb..753ca21a 100644 --- a/api/routes.py +++ b/api/routes.py @@ -5930,6 +5930,12 @@ def _resolve_compatible_session_model_state( """ model = str(model_id or "").strip() requested_provider = _clean_session_model_provider(model_provider) + if model and requested_provider == "moa": + if model.startswith("@moa:"): + return model.split(":", 1)[1].strip(), "moa", True + if model.lower().startswith("moa/"): + return model.split("/", 1)[1].strip(), "moa", True + return model, "moa", False if model and requested_provider and model.startswith(f"@{requested_provider}:"): try: from api.config import cfg as _active_cfg @@ -19149,6 +19155,15 @@ def _handle_chat_start(handler, body, diag=None): profile_default_model=_pp_default, explicit_model_pick=explicit_model_pick, ) + if model_provider == "moa" and moa_config is None: + if webui_gateway_chat_enabled(get_config()): + return bad(handler, "MoA override is unavailable on gateway-backed sessions", 409) + from api.commands import resolve_moa_config + + try: + moa_config = resolve_moa_config(model) + except RuntimeError as e: + return bad(handler, str(e), 503) # NOTE: runtime-adapter selection is delegated to _start_run (shared # with start_session_turn so both entry points behave identically # under runtime_adapter_enabled() / runtime_adapter_runner_enabled() diff --git a/tests/test_copilot_provider_model_settings_not_allowlist.py b/tests/test_copilot_provider_model_settings_not_allowlist.py new file mode 100644 index 00000000..7f4260e5 --- /dev/null +++ b/tests/test_copilot_provider_model_settings_not_allowlist.py @@ -0,0 +1,79 @@ +"""Regression coverage for Copilot model settings vs picker catalog. + +Hermes config uses providers..models as a per-model settings map for +some built-in providers. WebUI must not mistake that for a picker allowlist for +Copilot, or a single reasoning_effort entry collapses the model dropdown. +""" + + +def _provider_group(payload: dict, provider_id: str) -> dict: + for group in payload.get("groups", []): + if group.get("provider_id") == provider_id: + return group + raise AssertionError(f"provider group {provider_id!r} not found: {payload.get('groups')!r}") + + +def test_copilot_provider_models_settings_do_not_replace_live_catalog(monkeypatch, tmp_path): + import api.config as config + + cfg = { + "model": {"default": "gpt-5.5", "provider": "copilot"}, + "providers": { + "copilot": { + "name": "GitHub Copilot", + "api": "https://api.githubcopilot.com", + "transport": "chat_completions", + "models": { + "claude-opus-4.8": {"reasoning_effort": "xhigh"}, + }, + } + }, + } + + monkeypatch.setattr(config, "cfg", cfg, raising=False) + monkeypatch.setattr(config, "_get_config_path", lambda: tmp_path / "config.yaml") + monkeypatch.setattr(config, "_get_auth_store_path", lambda: tmp_path / "auth.json") + monkeypatch.setattr(config, "_get_models_cache_path", lambda: tmp_path / "models_cache.json") + monkeypatch.setattr(config, "_models_cache_source_fingerprint", lambda: {"test": "fingerprint"}) + monkeypatch.setattr(config, "reload_config_if_stale", lambda: None) + monkeypatch.setattr(config, "reload_config", lambda: None) + monkeypatch.setattr(config, "_cfg_mtime", 0.0, raising=False) + monkeypatch.setattr(config, "_LIVE_REBUILD_BUDGET_SECONDS", 0.0, raising=False) + monkeypatch.setattr(config, "_read_live_provider_model_ids", lambda pid: ["gpt-5.5", "claude-opus-4.8", "gpt-5.4"] if pid == "copilot" else []) + + config.invalidate_models_cache() + payload = config.get_available_models(force_refresh=True) + group = _provider_group(payload, "copilot") + ids = [m["id"] for m in group["models"]] + + assert ids == ["gpt-5.5", "claude-opus-4.8", "gpt-5.4"] + + +def test_unknown_duplicate_copilot_provider_config_is_not_rendered(monkeypatch, tmp_path): + import api.config as config + + cfg = { + "model": {"default": "gpt-5.5", "provider": "copilot"}, + "providers": { + "copilot": {"models": {"claude-opus-4.8": {}}}, + "copilot-2": {"name": "copilot", "models": {"gpt-5.5": {}}}, + }, + } + + monkeypatch.setattr(config, "cfg", cfg, raising=False) + monkeypatch.setattr(config, "_get_config_path", lambda: tmp_path / "config.yaml") + monkeypatch.setattr(config, "_get_auth_store_path", lambda: tmp_path / "auth.json") + monkeypatch.setattr(config, "_get_models_cache_path", lambda: tmp_path / "models_cache.json") + monkeypatch.setattr(config, "_models_cache_source_fingerprint", lambda: {"test": "fingerprint"}) + monkeypatch.setattr(config, "reload_config_if_stale", lambda: None) + monkeypatch.setattr(config, "reload_config", lambda: None) + monkeypatch.setattr(config, "_cfg_mtime", 0.0, raising=False) + monkeypatch.setattr(config, "_LIVE_REBUILD_BUDGET_SECONDS", 0.0, raising=False) + monkeypatch.setattr(config, "_read_live_provider_model_ids", lambda pid: ["gpt-5.5", "claude-opus-4.8"] if pid == "copilot" else []) + + config.invalidate_models_cache() + payload = config.get_available_models(force_refresh=True) + provider_ids = [g.get("provider_id") for g in payload.get("groups", [])] + + assert "copilot" in provider_ids + assert "copilot-2" not in provider_ids diff --git a/tests/test_moa_model_picker_provider.py b/tests/test_moa_model_picker_provider.py new file mode 100644 index 00000000..b5ef1796 --- /dev/null +++ b/tests/test_moa_model_picker_provider.py @@ -0,0 +1,110 @@ +def test_moa_presets_render_as_virtual_provider_models(monkeypatch, tmp_path): + import api.config as config + + cfg = { + "model": {"default": "gpt-5.5", "provider": "copilot"}, + "moa": { + "default_preset": "Frontier Tuned", + "presets": { + "default": {"enabled": True}, + "Frontier Tuned": {"enabled": True}, + "Disabled Preset": {"enabled": False}, + }, + }, + } + + monkeypatch.setattr(config, "cfg", cfg, raising=False) + monkeypatch.setattr(config, "_get_config_path", lambda: tmp_path / "config.yaml") + monkeypatch.setattr(config, "_get_auth_store_path", lambda: tmp_path / "auth.json") + monkeypatch.setattr(config, "_get_models_cache_path", lambda: tmp_path / "models_cache.json") + monkeypatch.setattr(config, "_models_cache_source_fingerprint", lambda: {"test": "fingerprint"}) + monkeypatch.setattr(config, "reload_config_if_stale", lambda: None) + monkeypatch.setattr(config, "reload_config", lambda: None) + monkeypatch.setattr(config, "_cfg_mtime", 0.0, raising=False) + monkeypatch.setattr(config, "_LIVE_REBUILD_BUDGET_SECONDS", 0.0, raising=False) + + def fake_live_models(provider_id): + if provider_id == "copilot": + return ["gpt-5.5"] + if provider_id == "moa": + return ["default", "Frontier Tuned"] + return [] + + monkeypatch.setattr(config, "_read_live_provider_model_ids", fake_live_models) + + config.invalidate_models_cache() + payload = config.get_available_models(force_refresh=True) + moa_group = next(g for g in payload["groups"] if g.get("provider_id") == "moa") + + assert moa_group["provider"] == "Mixture of Agents" + assert [m["id"] for m in moa_group["models"]] == ["@moa:default", "@moa:Frontier Tuned"] + + +def test_moa_picker_selection_resolves_to_preset_and_provider(): + from api.routes import _resolve_compatible_session_model_state + + model, provider, normalized = _resolve_compatible_session_model_state( + "moa/Frontier Tuned", + "moa", + profile_provider="copilot", + profile_default_model="gpt-5.5", + explicit_model_pick=True, + ) + + assert model == "Frontier Tuned" + assert provider == "moa" + assert normalized is True + + +def test_resolve_moa_config_uses_selected_preset(monkeypatch): + import sys + from types import ModuleType + from typing import Any, cast + + import api.commands as commands + + cfg = { + "moa": { + "default_preset": "default", + "presets": { + "default": { + "enabled": True, + "reference_models": [{"provider": "copilot", "model": "claude-sonnet-4.6"}], + "aggregator": {"provider": "copilot", "model": "gpt-5.5"}, + }, + "Frontier Tuned": { + "enabled": True, + "reference_models": [{"provider": "copilot", "model": "claude-opus-4.8"}], + "aggregator": {"provider": "copilot", "model": "gpt-5.4"}, + }, + }, + } + } + + hermes_cli_pkg = sys.modules.get("hermes_cli") or ModuleType("hermes_cli") + hermes_cli_pkg.__path__ = [] + moa_config = ModuleType("hermes_cli.moa_config") + + def normalize_moa_config(raw): + return { + "default_preset": raw.get("default_preset", "default"), + "presets": raw.get("presets", {}), + } + + def resolve_moa_preset(raw, preset_name): + presets = raw.get("presets", {}) if isinstance(raw, dict) else {} + return dict(presets.get(preset_name or raw.get("default_preset") or "default", {})) + + moa_config_any = cast(Any, moa_config) + moa_config_any.normalize_moa_config = normalize_moa_config + moa_config_any.resolve_moa_preset = resolve_moa_preset + moa_config_any.moa_usage = lambda: "Usage: /moa " + monkeypatch.setitem(sys.modules, "hermes_cli", hermes_cli_pkg) + monkeypatch.setitem(sys.modules, "hermes_cli.moa_config", moa_config) + monkeypatch.setattr(commands, "_load_config_for_moa_resolution", lambda: cfg, raising=False) + + resolved = commands.resolve_moa_config("Frontier Tuned") + + assert resolved["preset"] == "Frontier Tuned" + assert resolved["reference_models"] == [{"provider": "copilot", "model": "claude-opus-4.8"}] + assert resolved["aggregator"] == {"provider": "copilot", "model": "gpt-5.4"}