4f4c8fd183
Resolve review feedback on PR #5313: - Read reasoning_efforts from named custom_providers entries for custom:<name> providers instead of looking only in providers:<name>. - Preserve the nested-route deny ordering before the provider-config shortcut so Gemini image/embedding routes cannot be re-enabled by config. - Deduplicate configured effort values before returning them. Add focused regression tests for bare provider config, named custom provider config, all-invalid fallthrough, ACP hard guards, and nested route denies.
268 lines
8.5 KiB
Python
268 lines
8.5 KiB
Python
"""Tests for model-aware reasoning effort chip visibility."""
|
|
|
|
from api import config as cfg
|
|
|
|
|
|
def test_cursor_acp_models_do_not_support_reasoning_effort_levels():
|
|
assert cfg.resolve_model_reasoning_efforts(
|
|
"cursor/composer-2.5",
|
|
provider_id="cursor-acp",
|
|
) == []
|
|
|
|
|
|
def test_openai_codex_gpt5_supports_reasoning_effort_levels():
|
|
efforts = cfg.resolve_model_reasoning_efforts(
|
|
"gpt-5.5",
|
|
provider_id="openai-codex",
|
|
)
|
|
assert "medium" in efforts
|
|
assert "high" in efforts
|
|
assert "xhigh" in efforts
|
|
assert "max" not in efforts
|
|
|
|
|
|
def test_openai_codex_prefixed_gpt5_supports_reasoning_effort_levels():
|
|
efforts = cfg.resolve_model_reasoning_efforts(
|
|
"@openai-codex:gpt-5.5",
|
|
provider_id="openai-codex",
|
|
)
|
|
assert "medium" in efforts
|
|
assert "high" in efforts
|
|
assert "xhigh" in efforts
|
|
assert "max" not in efforts
|
|
|
|
|
|
def test_openai_codex_max_effort_is_clamped_before_streaming():
|
|
assert cfg.coerce_reasoning_effort_for_model(
|
|
"max",
|
|
"gpt-5.5",
|
|
provider_id="openai-codex",
|
|
) == "xhigh"
|
|
|
|
|
|
def test_unsupported_xhigh_degrades_to_high_not_disabled():
|
|
# o1/o3/o4 on openai-codex cap at low/medium/high. A configured xhigh (or
|
|
# max) must clamp DOWN to the highest supported level (high), not silently
|
|
# disable reasoning by returning "".
|
|
assert cfg.coerce_reasoning_effort_for_model(
|
|
"xhigh",
|
|
"o3-mini",
|
|
provider_id="openai-codex",
|
|
) == "high"
|
|
assert cfg.coerce_reasoning_effort_for_model(
|
|
"max",
|
|
"o3-mini",
|
|
provider_id="openai-codex",
|
|
) == "high"
|
|
|
|
|
|
def test_coerce_never_escalates_above_configured_effort():
|
|
# A supported lower effort is returned verbatim; coercion only degrades.
|
|
assert cfg.coerce_reasoning_effort_for_model(
|
|
"low",
|
|
"gpt-5.5",
|
|
provider_id="openai-codex",
|
|
) == "low"
|
|
|
|
|
|
def test_coerce_preserves_effort_for_unrecognized_model():
|
|
# #3505 review: resolve_model_reasoning_efforts() returns [] for BOTH
|
|
# known-unsupported AND simply-unrecognized models (custom providers,
|
|
# aggregator-rewritten ids, brand-new releases). Coercion must NOT silently
|
|
# drop a configured effort just because we don't recognize the model — that
|
|
# would be a behavior change vs sending it verbatim (master). Preserve the
|
|
# configured level for an empty/unknown capability set; the provider stays
|
|
# the final authority. The known-bad CLAMP paths return a NON-empty set, so
|
|
# they are unaffected (covered by the openai-codex tests above).
|
|
assert cfg.coerce_reasoning_effort_for_model(
|
|
"high",
|
|
"some-unknown-model-xyz",
|
|
provider_id="some-custom-provider",
|
|
) == "high"
|
|
assert cfg.coerce_reasoning_effort_for_model(
|
|
"max",
|
|
"brand-new-model-2099",
|
|
provider_id="some-custom-provider",
|
|
) == "xhigh"
|
|
# 'none' / unset still pass through unchanged for unknown models.
|
|
assert cfg.coerce_reasoning_effort_for_model(
|
|
"none", "some-unknown-model-xyz", provider_id="custom"
|
|
) == "none"
|
|
assert cfg.coerce_reasoning_effort_for_model(
|
|
"", "some-unknown-model-xyz", provider_id="custom"
|
|
) == ""
|
|
|
|
|
|
def test_github_copilot_gpt5_supports_reasoning_effort_levels():
|
|
efforts = cfg.resolve_model_reasoning_efforts(
|
|
"gpt-5.5",
|
|
provider_id="github-copilot",
|
|
)
|
|
assert "medium" in efforts
|
|
assert "high" in efforts
|
|
|
|
|
|
def test_openrouter_anthropic_models_keep_reasoning_effort_levels():
|
|
efforts = cfg.resolve_model_reasoning_efforts(
|
|
"anthropic/claude-sonnet-4.5",
|
|
provider_id="openrouter",
|
|
)
|
|
assert "medium" in efforts
|
|
assert "high" in efforts
|
|
|
|
|
|
def test_non_reasoning_http_models_hide_reasoning_effort_levels():
|
|
assert cfg.resolve_model_reasoning_efforts(
|
|
"meta-llama/llama-3.1-8b-instruct",
|
|
provider_id="openrouter",
|
|
) == []
|
|
|
|
|
|
def test_provider_config_reasoning_efforts_return_filtered_deduped(monkeypatch):
|
|
original = cfg.cfg.get("providers")
|
|
monkeypatch.setitem(
|
|
cfg.cfg,
|
|
"providers",
|
|
{
|
|
"wandb": {
|
|
"reasoning_efforts": [
|
|
" none ",
|
|
"HIGH",
|
|
"bogus",
|
|
"high",
|
|
"xhigh",
|
|
]
|
|
}
|
|
},
|
|
)
|
|
try:
|
|
assert cfg.resolve_model_reasoning_efforts(
|
|
"zai-org/GLM-5.2",
|
|
provider_id="wandb",
|
|
) == ["none", "high", "xhigh"]
|
|
finally:
|
|
if original is None:
|
|
cfg.cfg.pop("providers", None)
|
|
else:
|
|
monkeypatch.setitem(cfg.cfg, "providers", original)
|
|
|
|
|
|
def test_provider_config_all_invalid_falls_through(monkeypatch):
|
|
original = cfg.cfg.get("providers")
|
|
monkeypatch.setitem(
|
|
cfg.cfg,
|
|
"providers",
|
|
{"wandb": {"reasoning_efforts": ["bogus", "typo"]}},
|
|
)
|
|
try:
|
|
result = cfg.resolve_model_reasoning_efforts(
|
|
"zai-org/GLM-5.2",
|
|
provider_id="wandb",
|
|
)
|
|
assert result != []
|
|
assert "bogus" not in result
|
|
assert "typo" not in result
|
|
finally:
|
|
if original is None:
|
|
cfg.cfg.pop("providers", None)
|
|
else:
|
|
monkeypatch.setitem(cfg.cfg, "providers", original)
|
|
|
|
|
|
def test_named_custom_provider_config_reasoning_efforts(monkeypatch):
|
|
original = cfg.cfg.get("custom_providers")
|
|
monkeypatch.setitem(
|
|
cfg.cfg,
|
|
"custom_providers",
|
|
[{"name": "llm-proxy", "reasoning_efforts": ["none", "high", "xhigh"]}],
|
|
)
|
|
try:
|
|
assert cfg.resolve_model_reasoning_efforts(
|
|
"some-model",
|
|
provider_id="custom:llm-proxy",
|
|
) == ["none", "high", "xhigh"]
|
|
finally:
|
|
if original is None:
|
|
cfg.cfg.pop("custom_providers", None)
|
|
else:
|
|
monkeypatch.setitem(cfg.cfg, "custom_providers", original)
|
|
|
|
|
|
def test_acp_guards_win_over_configured_reasoning_efforts(monkeypatch):
|
|
original = cfg.cfg.get("providers")
|
|
monkeypatch.setitem(
|
|
cfg.cfg,
|
|
"providers",
|
|
{"copilot-acp": {"reasoning_efforts": ["high"]}},
|
|
)
|
|
try:
|
|
assert cfg.resolve_model_reasoning_efforts(
|
|
"some-model",
|
|
provider_id="copilot-acp",
|
|
) == []
|
|
finally:
|
|
if original is None:
|
|
cfg.cfg.pop("providers", None)
|
|
else:
|
|
monkeypatch.setitem(cfg.cfg, "providers", original)
|
|
|
|
|
|
def test_nested_route_deny_wins_over_configured_reasoning_efforts(monkeypatch):
|
|
original = cfg.cfg.get("custom_providers")
|
|
monkeypatch.setitem(
|
|
cfg.cfg,
|
|
"custom_providers",
|
|
[{"name": "agg", "reasoning_efforts": ["low", "high"]}],
|
|
)
|
|
try:
|
|
for model in ("vertex/gemini-image-1.0", "vertex/gemini-embedding-001"):
|
|
assert cfg.resolve_model_reasoning_efforts(
|
|
model,
|
|
provider_id="custom:agg",
|
|
) == []
|
|
finally:
|
|
if original is None:
|
|
cfg.cfg.pop("custom_providers", None)
|
|
else:
|
|
monkeypatch.setitem(cfg.cfg, "custom_providers", original)
|
|
|
|
|
|
def test_get_reasoning_status_includes_supported_efforts(monkeypatch):
|
|
monkeypatch.setattr(
|
|
cfg,
|
|
"resolve_model_reasoning_efforts",
|
|
lambda *a, **k: ["low", "medium", "high"],
|
|
)
|
|
status = cfg.get_reasoning_status(
|
|
model_id="gpt-5.5",
|
|
provider_id="openai-codex",
|
|
)
|
|
assert status["supported_efforts"] == ["low", "medium", "high"]
|
|
assert status["supports_reasoning_effort"] is True
|
|
|
|
|
|
def test_get_reasoning_status_for_reasoning_capable_model_has_no_max():
|
|
status = cfg.get_reasoning_status(
|
|
model_id="gpt-5.5",
|
|
provider_id="openai-codex",
|
|
)
|
|
assert status["supported_efforts"] == ["minimal", "low", "medium", "high", "xhigh"]
|
|
assert status["supports_reasoning_effort"] is True
|
|
assert "max" not in status["supported_efforts"]
|
|
|
|
|
|
def test_get_reasoning_status_coerces_stale_max_to_xhigh(monkeypatch):
|
|
"""A previously-saved `agent.reasoning_effort: max` (no longer a valid effort)
|
|
must be reported as the coerced `xhigh`, not the raw stale `max`, so the
|
|
boot/status/chip read paths agree with what streaming actually sends."""
|
|
monkeypatch.setattr(
|
|
cfg,
|
|
"_load_yaml_config_file",
|
|
lambda *a, **k: {"agent": {"reasoning_effort": "max"}},
|
|
)
|
|
status = cfg.get_reasoning_status(
|
|
model_id="gpt-5.5",
|
|
provider_id="openai-codex",
|
|
)
|
|
assert status["reasoning_effort"] == "xhigh"
|
|
assert status["reasoning_effort"] != "max"
|