"""Tests for auxiliary models settings UI — panels.js + index.html + i18n.js. Verifies that the auxiliary models card is present in the settings HTML, that the JS loading/saving logic is wired up, and that all locales have the required i18n keys. """ import json import shutil import subprocess from pathlib import Path from types import SimpleNamespace import pytest ROOT = Path(__file__).parent.parent PANELS_JS_PATH = ROOT / "static" / "panels.js" PANELS_JS = PANELS_JS_PATH.read_text(encoding="utf-8") INDEX_HTML = (ROOT / "static" / "index.html").read_text(encoding="utf-8") I18N_JS = (ROOT / "static" / "i18n.js").read_text(encoding="utf-8") STREAMING_PY = (ROOT / "api" / "streaming.py").read_text(encoding="utf-8") NODE = shutil.which("node") class TestAuxiliaryModelsHTML: """The auxiliary models card must be present in the settings preferences pane.""" def test_aux_models_container_exists(self): """The #auxModelsContainer div must exist in the preferences pane.""" assert 'id="auxModelsContainer"' in INDEX_HTML, ( "Missing #auxModelsContainer in index.html — auxiliary models card not rendered" ) def test_reset_button_exists(self): assert 'id="btnResetAuxModels"' in INDEX_HTML, ( "Missing #btnResetAuxModels button in index.html" ) def test_apply_button_exists(self): assert 'id="btnApplyAuxModels"' in INDEX_HTML, ( "Missing #btnApplyAuxModels button in index.html" ) def test_aux_card_after_default_model(self): """Auxiliary Models card should come after the Default Model card in the DOM.""" model_idx = INDEX_HTML.find('id="settingsModel"') aux_idx = INDEX_HTML.find('id="auxModelsContainer"') assert model_idx >= 0, "Default Model select not found in index.html" assert aux_idx >= 0, "Auxiliary Models container not found in index.html" assert aux_idx > model_idx, ( "Auxiliary Models container must appear after Default Model in the DOM" ) def test_i18n_label_on_aux_card(self): """The auxiliary models card label must use data-i18n attribute.""" assert 'data-i18n="settings_label_auxiliary_models"' in INDEX_HTML, ( "Missing data-i18n='settings_label_auxiliary_models' on auxiliary card label" ) class TestAuxiliaryModelsJS: """The JS logic for loading and saving auxiliary models must be in panels.js.""" def test_load_function_exists(self): assert "async function _loadAuxiliaryModels" in PANELS_JS, ( "Missing _loadAuxiliaryModels() in panels.js" ) def test_apply_function_exists(self): assert "async function _applyAuxModels" in PANELS_JS, ( "Missing _applyAuxModels() in panels.js" ) def test_auxiliary_task_metadata_is_normalized(self): """Frontend should keep auxiliary task rows backed by normalized metadata.""" assert "let _auxTasks=[]" in PANELS_JS, "Missing _auxTasks cache in panels.js" assert "function _normalizeAuxiliaryTasks" in PANELS_JS, ( "Missing _normalizeAuxiliaryTasks() in panels.js" ) assert "function _auxTaskLabelFromMeta" in PANELS_JS, ( "Missing _auxTaskLabelFromMeta() in panels.js" ) assert "_auxTasks=_normalizeAuxiliaryTasks((auxData&&auxData.tasks)||[])" in PANELS_JS, ( "Auxiliary load flow must normalize API task payload" ) assert "for(const task of _auxTasks)" in PANELS_JS, ( "Auxiliary rows should be rendered from normalized API payload" ) @pytest.mark.skipif(NODE is None, reason="node not on PATH") def test_normalize_auxiliary_tasks_keeps_first_wins_and_unknown_metadata(self): """Normalization should keep first occurrence and preserve unknown metadata.""" script = r""" const fs = require('fs'); const src = fs.readFileSync(process.argv[1], 'utf8'); function extract(name){ const re = new RegExp('function\\s+' + name + '\\s*\\('); const start = src.search(re); if(start < 0) throw new Error(name + ' not found'); let i = src.indexOf('{', start); let depth = 0; while(i < src.length){ const ch = src[i]; if(ch === '{') depth += 1; else if(ch === '}') { depth -= 1; if(depth === 0){ break; } } i += 1; } if(depth !== 0) throw new Error(name + ' parse failed'); return src.slice(start, i + 1); } global.t = (key) => { const dict = { settings_aux_task_vision: 'Vision', settings_aux_task_vision_desc: 'image/screenshot analysis', }; return Object.prototype.hasOwnProperty.call(dict, key) ? dict[key] : key; }; eval(extract('_auxTaskLabelFromMeta')); eval(extract('_normalizeAuxiliaryTasks')); const normalized = _normalizeAuxiliaryTasks([ {task: 'vision', provider: 'openai', model: 'gpt-5.5', label: 'Backend Vision', description: 'backend desc'}, {task: 'future_task', provider: 'openai', model: 'future-1', label: 'Future Task', description: 'future tool'}, {task: 'vision', provider: 'openai', model: 'gpt-5.6'}, null, {}, ]); const vision = normalized.find((entry) => entry.task === 'vision'); const futureTask = normalized.find((entry) => entry.task === 'future_task'); console.log(JSON.stringify({ tasks: normalized.map((entry) => entry.task), visionLabel: vision ? vision.label : null, visionDescription: vision ? vision.description : null, futureLabel: futureTask ? futureTask.label : null, futureDescription: futureTask ? futureTask.description : null, })); """ proc = subprocess.run( [NODE, "-e", script, str(PANELS_JS_PATH)], capture_output=True, text=True, timeout=20, ) assert proc.returncode == 0, f"node probe failed:\n{proc.stderr}" result = json.loads(proc.stdout.strip().splitlines()[-1]) assert result["tasks"] == ["vision", "future_task"], ( "normalize should keep duplicate removal and payload order" ) assert result["visionLabel"] == "Vision" assert result["visionDescription"] == "image/screenshot analysis" assert result["futureLabel"] == "Future Task" assert result["futureDescription"] == "future tool" def test_no_hardcoded_aux_task_slot_array(self): """Auxiliary rows must not come from a hardcoded task-slot list.""" assert "_AUX_TASK_SLOTS" not in PANELS_JS assert "const _AUX_TASK_SLOTS" not in PANELS_JS assert "session_search" not in PANELS_JS def test_advanced_options_button_and_modal_wiring(self): """Each auxiliary row and the main model should expose gear-driven advanced config editing.""" for marker in ( "aux-advanced-btn", "model-advanced-row", "model-advanced-btn", "mainAdvancedBtn", "_bindMainAdvancedOptionsButton", "document.createElement('button')", "row.appendChild(btn)", "_openAuxAdvancedOptions", "_mainAdvancedConfig=null", "btn.disabled=_mainAdvancedConfig===null", "if(_mainAdvancedConfig!==null)", "Object.prototype.hasOwnProperty.call(auxData,'main')", "_mainAdvancedConfig=null;", "auxAdvancedOverlay", "auxAdvancedBaseUrl", "auxAdvancedTimeout", "auxAdvancedDownloadTimeout", "auxAdvancedMaxConcurrency", "auxAdvancedExtraBody", "auxAdvancedApiKey", "api_key_clear", "Object.keys(cfg.extra_body).length", ): assert marker in PANELS_JS def test_normalize_auxiliary_tasks_deduplicates_and_rejects_malformed(self): """Frontend normalization should keep only first occurrence and valid payload entries.""" assert "if(!rawTask||typeof rawTask!=='object') continue;" in PANELS_JS assert "if(!task||seen.has(task)) continue;" in PANELS_JS def test_auxiliary_load_uses_api_payload_order(self): """Row rendering should iterate exactly the normalized API payload order.""" idx = PANELS_JS.find("_auxTasks=_normalizeAuxiliaryTasks((auxData&&auxData.tasks)||[])") assert idx >= 0 assert "for(const task of _auxTasks)" in PANELS_JS[idx:idx + 380] def test_main_advanced_modal_hides_unsupported_timing_fields_but_keeps_request_body(self): """Main-model modal should not advertise timing knobs that the chat agent cannot apply.""" open_idx = PANELS_JS.find("function _openAuxAdvancedOptions") assert open_idx >= 0 modal_body = PANELS_JS[open_idx:open_idx + 4600] assert "const timingFields=isMain?'':(" in modal_body assert "auxAdvancedExtraBody" in modal_body assert "auxAdvancedBaseUrl" in modal_body def test_main_advanced_modal_exposes_service_tier_selector(self): """Main-model advanced modal should expose service-tier control.""" open_idx = PANELS_JS.find("function _openAuxAdvancedOptions") assert open_idx >= 0 modal_body = PANELS_JS[open_idx:open_idx + 3600] helper_idx = PANELS_JS.find("function _mainModelSupportsServiceTier") assert helper_idx >= 0 helper_body = PANELS_JS[helper_idx:helper_idx + 1300] assert "_mainModelSupportsServiceTier" in PANELS_JS assert "selectedOpt.dataset.fast" in helper_body assert "return cfg&&cfg.supports_fast_tier===true" in helper_body assert "return fastSupport==='1'||fastSupport==='true'" in helper_body assert "provider!=='openai'&&provider!=='openai-api'&&provider!=='openai-codex')" in helper_body assert "provider==='openai-codex') return false" not in helper_body assert "auxAdvancedServiceTier" in modal_body assert "isMain&&_mainModelSupportsServiceTier(cfg)" in modal_body assert "settings_main_advanced_service_tier" in modal_body assert "settings_main_advanced_service_tier_default" in modal_body assert "settings_main_advanced_service_tier_priority" in modal_body assert "m.supports_fast_tier" in PANELS_JS assert "opt.dataset.fast='1'" in PANELS_JS assert "opt.dataset.fast='0'" in PANELS_JS def test_main_advanced_save_omits_unsupported_timing_keys(self): """Saving main-model options must not send blank timing keys that backend treats as clears.""" save_idx = PANELS_JS.find("const advanced={") assert save_idx >= 0 save_body = PANELS_JS[save_idx:save_idx + 900] object_literal = save_body[:save_body.find("};") + 2] assert "timeout:" not in object_literal assert "download_timeout:" not in object_literal assert "max_concurrency:" not in object_literal assert "if(!isMain){" in save_body assert "advanced.timeout=$('auxAdvancedTimeout')?.value||''" in save_body assert "advanced.download_timeout=$('auxAdvancedDownloadTimeout')?.value||''" in save_body assert "advanced.max_concurrency=$('auxAdvancedMaxConcurrency')?.value||''" in save_body def test_main_extra_body_flows_to_agent_request_overrides(self): """Persisted main extra_body must be passed to AIAgent, not only shown in Settings.""" assert "_main_model_request_overrides" in STREAMING_PY assert "'request_overrides' in _agent_params" in STREAMING_PY assert "_agent_kwargs['request_overrides'] = _main_request_overrides" in STREAMING_PY assert "_main_request_overrides or {}" in STREAMING_PY def test_advanced_modal_uses_defined_theme_tokens_and_inline_button_styles(self): """The modal is appended outside #mainSettings, so scoped button CSS must not be required.""" modal_idx = PANELS_JS.find("function _ensureAuxAdvancedModal") assert modal_idx >= 0 modal_body = PANELS_JS[modal_idx:modal_idx + 2400] assert "var(--panel)" not in modal_body, "--panel is not a defined WebUI theme token" assert "class=\"settings-btn\"" not in modal_body, "settings-btn is scoped under #mainSettings" assert "background:var(--surface)" in modal_body assert "background:var(--input-bg)" in modal_body assert ":-webkit-autofill" in PANELS_JS assert "settings_aux_advanced_title" in PANELS_JS assert "settings_aux_advanced_button_aria" in PANELS_JS def test_advanced_modal_inputs_disable_browser_autofill(self): """Advanced modal fields must not be mistaken for browser login/password fields.""" input_helper_idx = PANELS_JS.find("function _auxAdvancedInputHtml") assert input_helper_idx >= 0 input_helper = PANELS_JS[input_helper_idx:input_helper_idx + 1200] assert 'autocomplete="off"' in input_helper assert 'data-lpignore="true"' in input_helper assert 'data-1p-ignore="true"' in input_helper assert "aux-manual-override-value" in input_helper assert "aux-${id}" not in input_helper assert "autocompleteAttr" in input_helper api_key_idx = PANELS_JS.find("_auxAdvancedInputHtml('auxAdvancedApiKey'") assert api_key_idx >= 0 api_key_call = PANELS_JS[api_key_idx:api_key_idx + 450] assert "'password'" not in api_key_call assert 'autocomplete="one-time-code"' in api_key_call assert "-webkit-text-security:disc" in api_key_call def test_calls_model_auxiliary_api(self): """_loadAuxiliaryModels must call /api/model/auxiliary.""" assert "/api/model/auxiliary" in PANELS_JS, ( "panels.js must call /api/model/auxiliary to fetch current config" ) def test_calls_model_set_api(self): """_applyAuxModels must call /api/model/set to save changes.""" assert "/api/model/set" in PANELS_JS, ( "panels.js must call /api/model/set to save auxiliary model changes" ) def test_provider_cascade(self): """Changing provider must rebuild model dropdown.""" assert "_onAuxProviderChange" in PANELS_JS, ( "Missing _onAuxProviderChange() for provider→model cascade" ) assert "_buildAuxModelOptions" in PANELS_JS, ( "Missing _buildAuxModelOptions() for model dropdown rebuild" ) def test_custom_model_prompt(self): """Selecting 'Custom model…' must prompt for model ID.""" assert "__custom__" in PANELS_JS, ( "Missing __custom__ sentinel option for custom model input" ) def test_reset_calls_api_with_reset_task(self): """Reset button must call /api/model/set with task='__reset__'.""" idx = PANELS_JS.find("btnResetAuxModels") assert idx >= 0, "btnResetAuxModels not found in panels.js" # Check that __reset__ is sent in the reset handler body_after = PANELS_JS[idx:idx + 2000] assert "__reset__" in body_after, ( "Reset handler must send task='__reset__' to /api/model/set" ) def test_load_called_from_loadSettingsPanel(self): """_loadAuxiliaryModels must be called from loadSettingsPanel.""" assert "_loadAuxiliaryModels()" in PANELS_JS, ( "_loadAuxiliaryModels() is not called from loadSettingsPanel" ) def test_dirty_flag_marking(self): """Changing an auxiliary dropdown must mark settings dirty.""" assert "_markAuxDirty" in PANELS_JS, ( "Missing _markAuxDirty() for dirty detection" ) # _markAuxDirty should call _markSettingsDirty idx = PANELS_JS.find("function _markAuxDirty") body = PANELS_JS[idx:idx + 200] assert "_markSettingsDirty" in body, ( "_markAuxDirty must call _markSettingsDirty" ) class TestAuxiliaryModelsI18n: """All locales must have the auxiliary model i18n keys.""" REQUIRED_KEYS = [ "settings_label_auxiliary_models", "settings_desc_auxiliary_models", "settings_btn_reset_aux_models", "settings_btn_apply_aux_models", "settings_aux_provider_auto", "settings_aux_model_auto", "settings_aux_model_custom", "settings_aux_model_custom_prompt", "settings_aux_loading", "settings_aux_load_failed", "settings_aux_reset_confirm_title", "settings_aux_reset_confirm_msg", "settings_aux_reset_done", "settings_aux_save_failed", "settings_aux_saved", "settings_aux_no_changes", "settings_aux_advanced_button_title", "settings_aux_advanced_button_aria", "settings_aux_advanced_title", "settings_aux_advanced_subtitle", "settings_aux_advanced_save", "settings_aux_advanced_base_url", "settings_aux_advanced_base_url_desc", "settings_aux_advanced_timeout", "settings_aux_advanced_timeout_desc", "settings_aux_advanced_download_timeout", "settings_aux_advanced_download_timeout_desc", "settings_aux_advanced_max_concurrency", "settings_aux_advanced_max_concurrency_desc", "settings_aux_advanced_extra_body", "settings_aux_advanced_extra_body_desc", "settings_aux_advanced_api_key", "settings_aux_advanced_api_key_set_hint", "settings_aux_advanced_api_key_empty_hint", "settings_aux_advanced_api_key_clear", "settings_aux_advanced_extra_body_invalid_json", "settings_aux_advanced_extra_body_object_required", "settings_aux_advanced_saved", "settings_aux_advanced_save_failed", "settings_main_advanced_button_aria", "settings_main_advanced_title", "settings_main_advanced_subtitle", "settings_main_advanced_saved", "settings_main_advanced_save_failed", "settings_main_advanced_service_tier", "settings_main_advanced_service_tier_desc", "settings_main_advanced_service_tier_default", "settings_main_advanced_service_tier_priority", "settings_aux_task_vision", "settings_aux_task_vision_desc", "settings_aux_task_compression", "settings_aux_task_compression_desc", "settings_aux_task_web_extract", "settings_aux_task_web_extract_desc", "settings_aux_task_approval", "settings_aux_task_approval_desc", "settings_aux_task_mcp", "settings_aux_task_mcp_desc", "settings_aux_task_title_generation", "settings_aux_task_title_generation_desc", "settings_aux_task_skills_hub", "settings_aux_task_skills_hub_desc", "settings_aux_task_curator", "settings_aux_task_curator_desc", "settings_aux_task_kanban_decomposer", "settings_aux_task_kanban_decomposer_desc", "settings_aux_task_profile_describer", "settings_aux_task_profile_describer_desc", "settings_aux_task_triage_specifier", "settings_aux_task_triage_specifier_desc", ] def test_all_i18n_keys_present(self): """Every required key must exist in i18n.js at least once.""" for key in self.REQUIRED_KEYS: assert key in I18N_JS, ( f"Missing i18n key '{key}' in i18n.js" ) def test_all_locales_have_auxiliary_keys(self): """Count of each key should equal the number of supported locales.""" for key in self.REQUIRED_KEYS: count = I18N_JS.count(f"{key}:") assert count == 15, ( f"i18n key '{key}' found {count} times — expected 15 (one per locale)" ) def test_session_search_aux_task_i18n_keys_removed(self): """session_search auxiliary labels were retired from the canonical set.""" assert "settings_aux_task_session_search" not in I18N_JS assert "settings_aux_task_session_search_desc" not in I18N_JS class TestAuxiliaryModelsBackend: """WebUI backend must expose /api/model/auxiliary and /api/model/set.""" ROUTES_PY = (ROOT / "api" / "routes.py").read_text(encoding="utf-8") CONFIG_PY = (ROOT / "api" / "config.py").read_text(encoding="utf-8") def test_model_auxiliary_route_exists(self): """/api/model/auxiliary route must be registered in routes.py.""" assert '"/api/model/auxiliary"' in self.ROUTES_PY, ( "Missing /api/model/auxiliary route in routes.py" ) def test_model_set_route_exists(self): """/api/model/set route must be registered in routes.py.""" assert '"/api/model/set"' in self.ROUTES_PY, ( "Missing /api/model/set route in routes.py" ) def test_default_model_routes_drop_auxiliary_auto_provider_sentinel(self, monkeypatch): from api import routes seen = [] monkeypatch.setattr(routes, "_csrf_exempt_path", lambda _path: True) monkeypatch.setattr(routes, "j", lambda _handler, payload, **_kwargs: payload) def fake_set_default_model(model, provider=None, advanced=None): seen.append({ "model": model, "provider": provider, "advanced": advanced, }) return {"ok": True, "model": model, "provider": provider} monkeypatch.setattr(routes, "set_hermes_default_model", fake_set_default_model) bodies = { "/api/default-model": { "model": "gpt-5.5", "provider": "auto", "advanced": {"base_url": "https://example.invalid/v1"}, }, "/api/model/set": { "scope": "main", "model": "gpt-5.5", "provider": "auto", "advanced": {"base_url": "https://example.invalid/v1"}, }, } for path, body in bodies.items(): monkeypatch.setattr(routes, "read_body", lambda _handler, payload=body: payload) routes.handle_post(object(), SimpleNamespace(path=path, query="")) assert seen == [ { "model": "gpt-5.5", "provider": None, "advanced": {"base_url": "https://example.invalid/v1"}, }, { "model": "gpt-5.5", "provider": None, "advanced": {"base_url": "https://example.invalid/v1"}, }, ] def test_get_auxiliary_models_function_exists(self): """get_auxiliary_models() must exist in api/config.py.""" assert "def get_auxiliary_models" in self.CONFIG_PY, ( "Missing get_auxiliary_models() in api/config.py" ) def test_backend_aux_task_slots_include_agent_defaults(self): """Backend allow-list must include newer Hermes auxiliary slots.""" for key in ("kanban_decomposer", "profile_describer", "triage_specifier"): assert f'"{key}"' in self.CONFIG_PY def test_backend_surfaces_advanced_fields_without_api_key_value(self, monkeypatch): """Advanced fields should be visible, but API keys remain write-only.""" from api import config monkeypatch.setattr(config, "reload_config", lambda: None) monkeypatch.setattr(config, "cfg", { "model": {"provider": "openai", "default": "gpt-5.5"}, "auxiliary": { "vision": { "provider": "openai", "model": "gpt-5.5", "base_url": "https://example.invalid/v1", "timeout": 42, "download_timeout": 7, "max_concurrency": 2, "extra_body": {"reasoning_effort": "none"}, "api_key": "DUMMY_KEY_DO_NOT_RETURN", } }, }) data = config.get_auxiliary_models() vision = next(t for t in data["tasks"] if t["task"] == "vision") assert vision["base_url"] == "https://example.invalid/v1" assert vision["timeout"] == 42 assert vision["download_timeout"] == 7 assert vision["max_concurrency"] == 2 assert vision["extra_body"] == {"reasoning_effort": "none"} assert vision["api_key_set"] is True assert "api_key" not in vision def test_get_auxiliary_models_omits_session_search_and_includes_metadata(self, monkeypatch): """Backend payload should omit unknown keys and keep canonical metadata.""" from api import config monkeypatch.setattr(config, "reload_config", lambda: None) monkeypatch.setattr(config, "cfg", { "model": {"provider": "openai", "default": "gpt-5.5"}, "auxiliary": { "vision": {"provider": "openai", "model": "gpt-5.5"}, "future_task": { "provider": "openai", "model": "gpt-5.5", "label": "Future Task", "description": "future tool", }, "monitor": {"provider": "openai", "model": "gpt-5.5"}, "tts_audio_tags": {"provider": "openai", "model": "gpt-5.5"}, }, }) data = config.get_auxiliary_models() task_keys = [t["task"] for t in data["tasks"]] assert task_keys == [ "vision", "web_extract", "compression", "approval", "mcp", "title_generation", "skills_hub", "curator", "kanban_decomposer", "profile_describer", "triage_specifier", ] assert "session_search" not in task_keys assert "future_task" not in task_keys assert "monitor" not in task_keys assert "tts_audio_tags" not in task_keys vision = next(t for t in data["tasks"] if t["task"] == "vision") assert vision["label"] == "Vision" assert vision["description"] == "image/screenshot analysis" def test_set_auxiliary_model_rejects_existing_unknown_task(self, monkeypatch, tmp_path): """Existing unknown keys must be rejected by the auxiliary setter.""" from api import config config_path = tmp_path / "config.yaml" config_path.write_text( "auxiliary:\n future_task:\n provider: openai\n model: gpt-5.5\n", encoding="utf-8", ) monkeypatch.setattr(config, "_get_config_path", lambda: config_path) monkeypatch.setattr(config, "reload_config", lambda: None) with pytest.raises(ValueError, match="Unknown auxiliary task slot") as excinfo: config.set_auxiliary_model("future_task", "openai", "gpt-5.6") assert "future_task" in str(excinfo.value) def test_reset_removes_retired_slot_but_preserves_other_unknown_mappings( self, monkeypatch, tmp_path ): """Reset cleans known retired slots without treating config as task schema.""" from api import config config_path = tmp_path / "config.yaml" config_path.write_text( "auxiliary:\n" " vision:\n" " provider: openai\n" " model: gpt-5.5\n" " session_search:\n" " provider: openai\n" " model: gpt-5.5\n" " monitor:\n" " provider: openai\n" " model: gpt-5.5\n", encoding="utf-8", ) monkeypatch.setattr(config, "_get_config_path", lambda: config_path) monkeypatch.setattr(config, "reload_config", lambda: None) config.set_auxiliary_model("__reset__", "auto", "") saved = config._load_yaml_config_file(config_path)["auxiliary"] assert "session_search" not in saved assert saved["monitor"] == {"provider": "openai", "model": "gpt-5.5"} assert saved["vision"]["provider"] == "auto" assert saved["vision"]["model"] == "" def test_backend_surfaces_main_advanced_fields_without_api_key_value(self, monkeypatch): """Main model advanced fields should be visible, but API keys remain write-only.""" from api import config monkeypatch.setattr(config, "reload_config", lambda: None) monkeypatch.setattr(config, "cfg", { "model": { "provider": "openai", "default": "gpt-5.5", "base_url": "https://example.invalid/v1", "timeout": 42, "download_timeout": 7, "max_concurrency": 2, "extra_body": {"reasoning_effort": "none"}, "api_key": "DUMMY_KEY_DO_NOT_RETURN", }, "auxiliary": {}, }) data = config.get_auxiliary_models() main = data["main"] assert main["base_url"] == "https://example.invalid/v1" assert main["timeout"] == 42 assert main["download_timeout"] == 7 assert main["max_concurrency"] == 2 assert main["extra_body"] == {"reasoning_effort": "none"} assert main["api_key_set"] is True assert "api_key" not in main def test_set_auxiliary_model_function_exists(self): """set_auxiliary_model() must exist in api/config.py.""" assert "def set_auxiliary_model" in self.CONFIG_PY, ( "Missing set_auxiliary_model() in api/config.py" ) def test_aux_task_slots_constant_exists(self): """AUX_TASK_SLOTS must be defined in api/config.py.""" assert "AUX_TASK_SLOTS" in self.CONFIG_PY, ( "Missing AUX_TASK_SLOTS constant in api/config.py" ) def test_js_uses_models_endpoint_not_options(self): """Frontend must use /api/models (WebUI's own API) not /api/model/options (agent API).""" # _loadAuxiliaryModels should call /api/models, not /api/model/options idx = PANELS_JS.find("async function _loadAuxiliaryModels") assert idx >= 0, "_loadAuxiliaryModels not found" body = PANELS_JS[idx:idx + 800] assert "/api/models" in body, ( "_loadAuxiliaryModels must call /api/models for provider/model lists" ) assert "/api/model/options" not in body, ( "_loadAuxiliaryModels must NOT call /api/model/options (agent-only endpoint)" ) def test_set_auxiliary_model_rejects_unknown_task(self, monkeypatch, tmp_path): """Unknown auxiliary task names must not pollute config.yaml.""" from api import config config_path = tmp_path / "config.yaml" config_path.write_text("auxiliary: {}\n", encoding="utf-8") monkeypatch.setattr(config, "_get_config_path", lambda: config_path) try: config.set_auxiliary_model("arbitrary_key", "openai", "gpt-5.5") except ValueError as exc: assert "Unknown auxiliary task slot" in str(exc) assert "vision" in str(exc) else: raise AssertionError("set_auxiliary_model accepted an unknown task") assert "arbitrary_key" not in config_path.read_text(encoding="utf-8") def test_set_hermes_default_model_persists_advanced_options(self, monkeypatch, tmp_path): """Main-model gear-modal payload should persist supported model options.""" from api import config config_path = tmp_path / "config.yaml" config_path.write_text("model:\n provider: openai\n default: gpt-5.5\n", encoding="utf-8") monkeypatch.setattr(config, "_get_config_path", lambda: config_path) monkeypatch.setattr(config, "reload_config", lambda: None) monkeypatch.setattr(config, "invalidate_models_cache", lambda: None) monkeypatch.setattr(config, "resolve_model_provider", lambda model: (model, "openai", None)) result = config.set_hermes_default_model( "gpt-5.5", advanced={ "base_url": "https://example.invalid/v1/", "timeout": "45", "download_timeout": "9", "max_concurrency": "2", "extra_body": {"reasoning_effort": "none"}, "api_key": "DUMMY_KEY_DO_NOT_PRINT", }, ) assert result["ok"] is True text = config_path.read_text(encoding="utf-8") assert "https://example.invalid/v1" in text assert "timeout: 45" in text assert "download_timeout: 9" in text assert "max_concurrency: 2" in text assert "reasoning_effort: none" in text assert "DUMMY_KEY_DO_NOT_PRINT" in text def test_set_hermes_default_model_persists_explicit_provider_override(self, monkeypatch, tmp_path): from api import config config_path = tmp_path / "config.yaml" config_path.write_text("model:\n provider: openai\n default: gpt-5.5\n", encoding="utf-8") monkeypatch.setattr(config, "_get_config_path", lambda: config_path) monkeypatch.setattr(config, "reload_config", lambda: None) monkeypatch.setattr(config, "invalidate_models_cache", lambda: None) monkeypatch.setattr(config, "resolve_model_provider", lambda model: (model, "", None)) result = config.set_hermes_default_model("gpt-5.5", provider="anthropic") assert result["ok"] is True assert result["provider"] == "anthropic" text = config_path.read_text(encoding="utf-8") assert "provider: anthropic" in text def test_set_hermes_default_model_provider_override_replaces_stale_custom_base_url(self, monkeypatch, tmp_path): from api import config config_path = tmp_path / "config.yaml" config_path.write_text("model:\n provider: custom\n default: gpt-5.5\n base_url: http://old.local/v1\n", encoding="utf-8") monkeypatch.setattr(config, "_get_config_path", lambda: config_path) monkeypatch.setattr(config, "reload_config", lambda: None) monkeypatch.setattr(config, "invalidate_models_cache", lambda: None) monkeypatch.setattr(config, "resolve_model_provider", lambda model: (model, "custom", "http://old.local/v1")) result = config.set_hermes_default_model("gpt-5.5", provider="openai") assert result["ok"] is True saved = config_path.read_text(encoding="utf-8") assert "provider: openai" in saved assert "base_url: https://api.openai.com/v1" in saved assert "http://old.local/v1" not in saved def test_set_auxiliary_model_persists_advanced_options(self, monkeypatch, tmp_path): """Gear-modal payload should persist supported per-slot options.""" from api import config config_path = tmp_path / "config.yaml" config_path.write_text("auxiliary:\n vision:\n provider: auto\n model: ''\n", encoding="utf-8") monkeypatch.setattr(config, "_get_config_path", lambda: config_path) monkeypatch.setattr(config, "reload_config", lambda: None) result = config.set_auxiliary_model( "vision", "openai", "gpt-5.5", advanced={ "base_url": "https://example.invalid/v1/", "timeout": "45", "download_timeout": "9", "max_concurrency": "2", "extra_body": {"reasoning_effort": "none"}, "api_key": "DUMMY_KEY_DO_NOT_PRINT", }, ) assert result["ok"] is True text = config_path.read_text(encoding="utf-8") assert "https://example.invalid/v1" in text assert "timeout: 45" in text assert "download_timeout: 9" in text assert "max_concurrency: 2" in text assert "reasoning_effort: none" in text assert "DUMMY_KEY_DO_NOT_PRINT" in text def test_set_auxiliary_model_explicit_advanced_base_url_wins_over_custom_resolution(self, monkeypatch, tmp_path): """Custom-provider auto-resolution must not clobber an explicit gear base_url.""" from api import config config_path = tmp_path / "config.yaml" config_path.write_text("auxiliary:\n vision:\n provider: auto\n model: ''\n", encoding="utf-8") monkeypatch.setattr(config, "_get_config_path", lambda: config_path) monkeypatch.setattr(config, "reload_config", lambda: None) monkeypatch.setattr( config, "resolve_model_provider", lambda model: (model, "custom:demo", "https://resolved.invalid/v1"), ) result = config.set_auxiliary_model( "vision", "custom:demo", "demo/model", advanced={"base_url": "https://manual.invalid/v1/"}, ) assert result["ok"] is True text = config_path.read_text(encoding="utf-8") assert "https://manual.invalid/v1" in text assert "https://resolved.invalid/v1" not in text def test_main_extra_body_becomes_runtime_request_overrides(self): """The main-model extra_body option is live only if it reaches request_overrides.""" from api import config cfg = { "model": { "provider": "openai", "default": "gpt-5.5", "extra_body": {"reasoning_effort": "none"}, } } overrides = config._main_model_request_overrides(cfg) assert overrides == {"extra_body": {"reasoning_effort": "none"}} assert overrides["extra_body"] is not cfg["model"]["extra_body"] def test_set_hermes_default_model_clear_api_key_removes_key(self, monkeypatch, tmp_path): """Clearing a write-only API key override should remove the key, not persist api_key: ''.""" from api import config config_path = tmp_path / "config.yaml" config_path.write_text( "model:\n provider: openai\n default: gpt-5.5\n api_key: old-secret\n", encoding="utf-8", ) monkeypatch.setattr(config, "_get_config_path", lambda: config_path) monkeypatch.setattr(config, "reload_config", lambda: None) monkeypatch.setattr(config, "invalidate_models_cache", lambda: None) monkeypatch.setattr(config, "resolve_model_provider", lambda model: (model, "openai", None)) result = config.set_hermes_default_model( "gpt-5.5", advanced={"api_key_clear": True, "api_key": ""}, ) assert result["ok"] is True text = config_path.read_text(encoding="utf-8") assert "old-secret" not in text assert "api_key" not in text def test_set_auxiliary_model_clears_empty_extra_body(self, monkeypatch, tmp_path): """Blank extra_body from the modal should remove config noise instead of writing {}.""" from api import config config_path = tmp_path / "config.yaml" config_path.write_text( "auxiliary:\n vision:\n provider: openai\n model: gpt-5.5\n extra_body:\n reasoning_effort: none\n", encoding="utf-8", ) monkeypatch.setattr(config, "_get_config_path", lambda: config_path) monkeypatch.setattr(config, "reload_config", lambda: None) result = config.set_auxiliary_model("vision", "openai", "gpt-5.5", advanced={"extra_body": {}}) assert result["ok"] is True assert "extra_body" not in config_path.read_text(encoding="utf-8") def test_set_auxiliary_model_validates_extra_body_object(self, monkeypatch, tmp_path): """extra_body must stay an object, not arbitrary JSON.""" from api import config config_path = tmp_path / "config.yaml" config_path.write_text("auxiliary: {}\n", encoding="utf-8") monkeypatch.setattr(config, "_get_config_path", lambda: config_path) try: config.set_auxiliary_model("vision", "openai", "gpt-5.5", advanced={"extra_body": ["bad"]}) except ValueError as exc: assert "extra_body must be a JSON object" in str(exc) else: raise AssertionError("set_auxiliary_model accepted non-object extra_body") def test_model_set_route_returns_400_for_unknown_auxiliary_task(self, monkeypatch): """The route should surface invalid auxiliary task names as a client error.""" from types import SimpleNamespace from api import routes monkeypatch.setattr(routes, "_check_csrf", lambda _handler: True) monkeypatch.setattr(routes, "read_body", lambda _handler: { "scope": "auxiliary", "task": "arbitrary_key", "provider": "openai", "model": "gpt-5.5", }) monkeypatch.setattr( routes, "bad", lambda _handler, msg, status=400: {"ok": False, "error": msg, "status": status}, ) result = routes.handle_post(object(), SimpleNamespace(path="/api/model/set")) assert result["status"] == 400 assert "Unknown auxiliary task slot" in result["error"]