From da73c00f06577159ee820d8ab6acad5c233a7385 Mon Sep 17 00:00:00 2001 From: Frank Song Date: Wed, 13 May 2026 18:57:56 +0800 Subject: [PATCH] Harden session tail redaction prefilter --- CHANGELOG.md | 2 +- api/helpers.py | 58 ++++++++++++++++++++++++++++---- tests/test_security_redaction.py | 51 ++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1097bebf..0988dc30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Fixed -- Large session responses now skip the full credential-redaction pass for strings that contain no credential markers, and tail-window session loads no longer send historical `tool_calls` when the returned messages already carry per-message tool metadata. Both changes preserve the existing redaction and tool-card behavior while reducing session-switch latency on long transcripts. +- Large session responses now skip the full credential-redaction pass for strings that contain no credential markers, and tail-window session loads no longer send historical `tool_calls` when the returned messages already carry per-message tool metadata. The redaction prefilter and local fallback now mirror the full prefixed-token set handled by the agent redactor, so the faster path keeps the existing hard API redaction boundary while reducing session-switch latency on long transcripts. - **PR #2179** (self-built, closes #2177) — NVIDIA NIM no longer 404s when WebUI is configured with `provider: nvidia` and a `nvidia/` id. `resolve_model_provider()` in `api/config.py` had the `_PORTAL_PROVIDERS` guard (Nous, OpenCode-Zen, OpenCode-Go, NVIDIA NIM — providers whose APIs require the full namespaced `provider/model` wire format) sitting **after** the `prefix == config_provider` strip branch. For `model_id="nvidia/nemotron-3-super-120b-a12b"` + `config_provider="nvidia"`, the strip branch fired first and returned the bare `nemotron-...` to NIM, which then 404'd because NIM requires the full path. Same bug class as #854 / #894 (Nous portal). The guard was originally added with NVIDIA in mind but the structural ordering was wrong. Fix is a pure reorder of two `if` blocks — hoist `_PORTAL_PROVIDERS` ahead of the strip — so all portal providers always preserve the full `provider/model` id regardless of whether the prefix happens to equal the provider name. Also closes a latent symmetric bug for the Nous case if a `nous/` id ever entered the catalog. Cross-tool trace against hermes-agent's `hermes_cli/models.py` (line 59, 177, 237-239) and `agent/model_metadata.py:68` confirms the agent CLI sends `nvidia/nemotron-...` verbatim — both tools now agree on the wire format. 118-line regression suite covering the reported case, cross-namespace `qwen/` and `meta/` ids, every static nvidia model in `_PROVIDER_MODELS`, the latent `nous/` ordering pin, and a non-portal-provider regression pin for the anthropic strip behavior. nesquena APPROVED with 200-line end-to-end trace + 12-shape behavioural harness + cross-tool wire-format verification. Reported on Discord by @vishnu in #report-bugs. diff --git a/api/helpers.py b/api/helpers.py index 664dc104..28873e69 100644 --- a/api/helpers.py +++ b/api/helpers.py @@ -112,7 +112,8 @@ MAX_BODY_BYTES = 20 * 1024 * 1024 # 20MB limit for non-upload POST bodies def _build_redact_fn(): """Return a redactor backed by hermes-agent plus local fallback patterns.""" - # Minimal fallback covering the most common credential prefixes. + # Fallback mirrors the agent's known credential prefixes so WebUI API + # responses remain a hard redaction boundary even without hermes-agent. # Keep this active even when hermes-agent is importable so API responses do # not regress if the agent redactor misses a token shape. _CRED_RE = _re.compile( @@ -124,10 +125,34 @@ def _build_redact_fn(): r"|ghu_[A-Za-z0-9]{10,}" # GitHub user-to-server token r"|ghs_[A-Za-z0-9]{10,}" # GitHub server-to-server token r"|ghr_[A-Za-z0-9]{10,}" # GitHub refresh token + r"|xox[baprs]-[A-Za-z0-9-]{10,}" # Slack tokens + r"|AIza[A-Za-z0-9_-]{30,}" # Google API keys + r"|pplx-[A-Za-z0-9]{10,}" # Perplexity + r"|fal_[A-Za-z0-9_-]{10,}" # Fal.ai + r"|fc-[A-Za-z0-9]{10,}" # Firecrawl + r"|bb_live_[A-Za-z0-9_-]{10,}" # BrowserBase + r"|gAAAA[A-Za-z0-9_=-]{20,}" # Codex encrypted tokens r"|AKIA[A-Z0-9]{16}" # AWS Access Key ID - r"|xox[baprs]-[A-Za-z0-9-]{10,}" # Slack tokens - r"|hf_[A-Za-z0-9]{10,}" # HuggingFace token - r"|SG\.[A-Za-z0-9_-]{10,}" # SendGrid API key + r"|sk_live_[A-Za-z0-9]{10,}" # Stripe secret key (live) + r"|sk_test_[A-Za-z0-9]{10,}" # Stripe secret key (test) + r"|rk_live_[A-Za-z0-9]{10,}" # Stripe restricted key + r"|SG\.[A-Za-z0-9_-]{10,}" # SendGrid API key + r"|hf_[A-Za-z0-9]{10,}" # HuggingFace token + r"|r8_[A-Za-z0-9]{10,}" # Replicate API token + r"|npm_[A-Za-z0-9]{10,}" # npm access token + r"|pypi-[A-Za-z0-9_-]{10,}" # PyPI API token + r"|dop_v1_[A-Za-z0-9]{10,}" # DigitalOcean PAT + r"|doo_v1_[A-Za-z0-9]{10,}" # DigitalOcean OAuth + r"|am_[A-Za-z0-9_-]{10,}" # AgentMail API key + r"|sk_[A-Za-z0-9_]{10,}" # ElevenLabs TTS key + r"|tvly-[A-Za-z0-9]{10,}" # Tavily search API key + r"|exa_[A-Za-z0-9]{10,}" # Exa search API key + r"|gsk_[A-Za-z0-9]{10,}" # Groq Cloud API key + r"|syt_[A-Za-z0-9]{10,}" # Matrix access token + r"|retaindb_[A-Za-z0-9]{10,}" # RetainDB API key + r"|hsk-[A-Za-z0-9]{10,}" # Hindsight API key + r"|mem0_[A-Za-z0-9]{10,}" # Mem0 Platform API key + r"|brv_[A-Za-z0-9]{10,}" # ByteRover API key r")(?![A-Za-z0-9_-])" ) _AUTH_HDR_RE = _re.compile(r"(Authorization:\s*Bearer\s+)(\S+)", _re.IGNORECASE) @@ -193,9 +218,30 @@ _SENSITIVE_CASE_MARKERS = ( "xoxp-", "xoxr-", "xoxs-", - "hf_", - "SG.", "AIza", + "pplx-", + "fal_", + "fc-", + "bb_live_", + "gAAAA", + "sk_live_", + "sk_test_", + "rk_live_", + "SG.", + "hf_", + "r8_", + "npm_", + "pypi-", + "dop_v1_", + "doo_v1_", + "am_", + "sk_", + "tvly-", + "exa_", + "gsk_", + "syt_", + "retaindb_", + "hsk-", "mem0_", "brv_", "eyJ", diff --git a/tests/test_security_redaction.py b/tests/test_security_redaction.py index 81d63af7..51a6033a 100644 --- a/tests/test_security_redaction.py +++ b/tests/test_security_redaction.py @@ -137,6 +137,57 @@ def test_redact_text_still_runs_full_redactor_for_sensitive_markers(monkeypatch) assert calls == [f"token={_FAKE_SK_KEY}"] +@pytest.mark.parametrize("prefix,suffix", [ + ("sk-", "TestCredential1234567890"), + ("ghp_", "TestCredential1234567890"), + ("github_pat_", "TestCredential_1234567890"), + ("gho_", "TestCredential1234567890"), + ("ghu_", "TestCredential1234567890"), + ("ghs_", "TestCredential1234567890"), + ("ghr_", "TestCredential1234567890"), + ("xoxb-", "TestCredential1234567890"), + ("xoxa-", "TestCredential1234567890"), + ("xoxp-", "TestCredential1234567890"), + ("xoxr-", "TestCredential1234567890"), + ("xoxs-", "TestCredential1234567890"), + ("AIza", "TestCredential1234567890abcdefghi"), + ("pplx-", "TestCredential1234567890"), + ("fal_", "TestCredential1234567890"), + ("fc-", "TestCredential1234567890"), + ("bb_live_", "TestCredential1234567890"), + ("gAAAA", "TestCredential1234567890abcd"), + ("AKIA", "TESTCREDENTIAL12"), + ("sk_" + "live_", "TestCredential1234567890"), + ("sk_" + "test_", "TestCredential1234567890"), + ("rk_" + "live_", "TestCredential1234567890"), + ("SG.", "TestCredential1234567890"), + ("hf_", "TestCredential1234567890"), + ("r8_", "TestCredential1234567890"), + ("npm_", "TestCredential1234567890"), + ("pypi-", "TestCredential1234567890"), + ("dop_v1_", "TestCredential1234567890"), + ("doo_v1_", "TestCredential1234567890"), + ("am_", "TestCredential1234567890"), + ("sk_", "TestCredential1234567890"), + ("tvly-", "TestCredential1234567890"), + ("exa_", "TestCredential1234567890"), + ("gsk_", "TestCredential1234567890"), + ("syt_", "TestCredential1234567890"), + ("retaindb_", "TestCredential1234567890"), + ("hsk-", "TestCredential1234567890"), + ("mem0_", "TestCredential1234567890"), + ("brv_", "TestCredential1234567890"), +]) +def test_redact_text_prefilter_covers_known_prefixed_credentials(prefix, suffix): + """Every known prefix must still reach the hard redactor.""" + import api.helpers as helpers + + token = prefix + suffix + result = helpers._redact_text(f"credential={token}", _enabled=True) + + assert token not in result + + def test_redact_value_works_with_legacy_agent_redact_signature(monkeypatch): """_redact_text must tolerate older redact_sensitive_text(text) signatures.""" fake_agent = types.ModuleType("agent")