From 5df6b1fe8a103546e59d073c1bbecdc5d3ae9575 Mon Sep 17 00:00:00 2001 From: pxxD1998 <214340659+pxxD1998@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:01:22 +0800 Subject: [PATCH] fix: stop stale agent runtime after source updates --- api/agent_runtime.py | 79 +++++++++++++++ api/routes.py | 19 ++-- api/streaming.py | 22 +++-- docs/troubleshooting.md | 22 +++++ tests/test_agent_runtime_revision_guard.py | 101 ++++++++++++++++++++ tests/test_agent_source_dependency_audit.py | 2 +- tests/test_regressions.py | 8 +- tests/test_update_banner_fixes.py | 2 +- 8 files changed, 232 insertions(+), 23 deletions(-) create mode 100644 api/agent_runtime.py create mode 100644 tests/test_agent_runtime_revision_guard.py diff --git a/api/agent_runtime.py b/api/agent_runtime.py new file mode 100644 index 00000000..be3139f6 --- /dev/null +++ b/api/agent_runtime.py @@ -0,0 +1,79 @@ +"""Fail-closed guard for in-process Hermes Agent source revisions. + +Hermes WebUI currently imports ``run_agent.AIAgent`` into its long-lived server +process. If the Agent checkout changes while that process is alive, Python may +combine already-cached modules with newly-read source. Refuse to reuse that +mixed runtime and require a clean WebUI restart instead. +""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +import threading + +from api.config import _AGENT_DIR + + +_RESTART_MESSAGE = ( + "Hermes Agent was updated while Hermes WebUI was running. " + "Restart Hermes WebUI before retrying this action." +) + + +def _read_agent_revision(agent_dir: Path) -> str | None: + """Return the checkout HEAD, or ``None`` for a non-Git/unavailable source.""" + try: + result = subprocess.run( + ["git", "-C", str(agent_dir), "rev-parse", "--verify", "HEAD"], + check=False, + capture_output=True, + text=True, + timeout=2, + ) + except (OSError, subprocess.TimeoutExpired): + return None + revision = result.stdout.strip() + return revision if result.returncode == 0 and revision else None + + +_AGENT_REVISION = _read_agent_revision(_AGENT_DIR) +_AIAgent = None +_RUNTIME_LOCK = threading.Lock() + + +class AgentRuntimeChangedError(RuntimeError): + """Raised when the loaded Agent runtime no longer matches its source tree.""" + + +def ensure_agent_runtime_current() -> None: + """Reject a known Git checkout change instead of mixing Python modules.""" + if _AGENT_REVISION is None: + return + if _read_agent_revision(_AGENT_DIR) != _AGENT_REVISION: + raise AgentRuntimeChangedError(_RESTART_MESSAGE) + + +def require_ai_agent_class(): + """Import ``AIAgent`` after proving the loaded source revision is current.""" + ensure_agent_runtime_current() + from run_agent import AIAgent # noqa: PLC0415 + + return AIAgent + + +def get_ai_agent_class(): + """Return ``AIAgent`` while preserving the existing lazy-import retry.""" + global _AIAgent, _AGENT_REVISION + + with _RUNTIME_LOCK: + ensure_agent_runtime_current() + if _AIAgent is None: + try: + agent_class = require_ai_agent_class() + except ImportError: + return None + _AIAgent = agent_class + if _AGENT_REVISION is None: + _AGENT_REVISION = _read_agent_revision(_AGENT_DIR) + return _AIAgent diff --git a/api/routes.py b/api/routes.py index 2a654ab5..93b63aa7 100644 --- a/api/routes.py +++ b/api/routes.py @@ -34,6 +34,7 @@ from contextlib import closing from urllib.parse import parse_qs, quote, unquote, urljoin, urlsplit from urllib.error import HTTPError, URLError from urllib.request import HTTPRedirectHandler, HTTPSHandler, ProxyHandler, Request, build_opener +from api.agent_runtime import ensure_agent_runtime_current, require_ai_agent_class from api.agent_sessions import ( MESSAGING_SOURCES, _looks_like_default_cli_title, @@ -16052,6 +16053,7 @@ def handle_post(handler, parsed) -> bool: "api_key": _main_api_key, } + ensure_agent_runtime_current() try: from agent.auxiliary_client import get_text_auxiliary_client @@ -16072,7 +16074,7 @@ def handle_post(handler, parsed) -> bool: except Exception as _e: logger.debug("update summary auxiliary model failed; falling back to main model: %s", _e) - from run_agent import AIAgent + AIAgent = require_ai_agent_class() agent = AIAgent( model=_main_model, @@ -21991,7 +21993,7 @@ def _handle_chat_sync(handler, body): os.environ["HERMES_EXEC_ASK"] = "1" os.environ["HERMES_SESSION_KEY"] = s.session_id try: - from run_agent import AIAgent + AIAgent = require_ai_agent_class() with CHAT_LOCK: from api.config import ( @@ -22622,6 +22624,7 @@ def _llm_git_commit_message(system_prompt: str, user_prompt: str, session=None) "base_url": _main_base_url, "api_key": _main_api_key, } + ensure_agent_runtime_current() try: from agent.auxiliary_client import get_text_auxiliary_client @@ -22638,7 +22641,7 @@ def _llm_git_commit_message(system_prompt: str, user_prompt: str, session=None) except Exception as _e: logger.debug("git commit message auxiliary model failed; falling back to main model: %s", _e) - from run_agent import AIAgent + AIAgent = require_ai_agent_class() agent = AIAgent( model=_main_model, @@ -24035,10 +24038,11 @@ def _handle_session_compress(handler, body): focus_topic, ) + ensure_agent_runtime_current() import api.config as _cfg from api.oauth import resolve_runtime_provider_with_anthropic_env_lock import hermes_cli.runtime_provider as _runtime_provider - import run_agent as _run_agent + AIAgent = require_ai_agent_class() resolved_model, resolved_provider, resolved_base_url = _cfg.resolve_model_provider( _cfg.model_with_provider_context(s.model, getattr(s, "model_provider", None)) @@ -24081,7 +24085,7 @@ def _handle_session_compress(handler, body): ) approx_tokens = _estimate_messages_tokens_rough(original_messages) - agent = _run_agent.AIAgent( + agent = AIAgent( model=resolved_model, provider=resolved_provider, base_url=resolved_base_url, @@ -24685,10 +24689,11 @@ def _handle_handoff_summary(handler, body): # Call LLM for summary. try: + ensure_agent_runtime_current() import api.config as _cfg from api.oauth import resolve_runtime_provider_with_anthropic_env_lock import hermes_cli.runtime_provider as _runtime_provider - import run_agent as _run_agent + AIAgent = require_ai_agent_class() # Try to resolve model from an existing session, fall back to default. resolved_model = None @@ -24744,7 +24749,7 @@ def _handle_handoff_summary(handler, body): "fallback": True, }) - agent = _run_agent.AIAgent( + agent = AIAgent( model=resolved_model, provider=resolved_provider, base_url=resolved_base_url, diff --git a/api/streaming.py b/api/streaming.py index c733a8f9..1b8fc70b 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -564,10 +564,13 @@ def _prewarm_skill_tool_modules(): # Lazy import to avoid circular deps -- hermes-agent is on sys.path via api/config.py -try: - from run_agent import AIAgent -except ImportError: - AIAgent = None +from api.agent_runtime import ensure_agent_runtime_current, get_ai_agent_class + + +# Eagerly attempt the import at startup, matching the pre-guard behavior. If +# dependencies are not ready yet, _get_ai_agent() retries when a chat starts. +AIAgent = get_ai_agent_class() + def _get_ai_agent(): """Return AIAgent class, retrying the import if the initial attempt failed. @@ -575,15 +578,13 @@ def _get_ai_agent(): auto_install_agent_deps() in server.py may install missing packages after this module is first imported (common in Docker with a volume-mounted agent). Re-attempting the import here picks up the newly installed packages without - requiring a server restart. + requiring a server restart. The shared runtime guard also refuses to reuse + cached Agent modules after the source checkout changes. """ global AIAgent + ensure_agent_runtime_current() if AIAgent is None: - try: - from run_agent import AIAgent as _cls # noqa: PLC0415 - AIAgent = _cls - except ImportError: - pass + AIAgent = get_ai_agent_class() return AIAgent @@ -7213,6 +7214,7 @@ def _run_agent_streaming( logger.debug("register_process_session failed", exc_info=True) # first-time module initialisation (which can be slow) does not # block other concurrent sessions waiting on _ENV_LOCK (#2024). + ensure_agent_runtime_current() _prewarm_skill_tool_modules() _install_streaming_cronjob_profile_wrapper() # Still set process-level env as fallback for tools that bypass thread-local diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 71d63d73..da8a5dcf 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -177,6 +177,28 @@ turn in the exhausted session instead of being blocked with recovery guidance. --- +## "Hermes Agent was updated while Hermes WebUI was running" + +**Symptom.** An action that uses the in-process Agent runtime stops with a message telling you to restart Hermes WebUI. This can happen after `hermes update`, a Git checkout/pull in the Agent source tree, or another tool updates Hermes Agent without restarting the already-running WebUI backend. + +**Why.** WebUI currently imports `run_agent.AIAgent` into its long-lived Python process. Python keeps imported modules in memory. Continuing after the Agent Git revision changes could combine cached modules from the old revision with source read from the new revision, producing misleading `ImportError`s or inconsistent runtime state. WebUI therefore fails closed instead of attempting a partial in-process reload. + +**Diagnostic.** Compare the running WebUI process start time with the Agent checkout revision and recent update history. If the Agent was updated after WebUI started, restart WebUI before investigating individual missing-symbol errors. + +**Fix.** Restart using the same launch method that started WebUI: + +```bash +./ctl.sh restart +# Or, for a user systemd service: +systemctl --user restart hermes-webui.service +``` + +If you launched `python3 bootstrap.py` in the foreground, stop it with Ctrl-C and start it again. Restarting the whole computer or WSL is not required when restarting the WebUI backend succeeds. + +**When to file a bug.** File a WebUI bug if the restart-required message appears even though the Agent revision did not change, or if a clean WebUI restart still produces the same import error. Include the WebUI launch method, WebUI revision, Agent revision, and the sanitized error text. + +--- + ## Other troubleshooting This document grows over time. If a recurring failure mode isn't covered here yet, add it via PR. The format for each entry: **Symptom → Why → Diagnostic commands → Fix → When to file a bug**. diff --git a/tests/test_agent_runtime_revision_guard.py b/tests/test_agent_runtime_revision_guard.py new file mode 100644 index 00000000..f9cc1ae6 --- /dev/null +++ b/tests/test_agent_runtime_revision_guard.py @@ -0,0 +1,101 @@ +"""Regression coverage for Hermes Agent source changes during a WebUI process lifetime.""" + +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys + + +REPO = Path(__file__).resolve().parents[1] + + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", *args], + cwd=repo, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def test_loaded_agent_runtime_fails_closed_after_source_revision_changes(tmp_path: Path): + agent_dir = tmp_path / "hermes-agent" + agent_dir.mkdir() + (agent_dir / "run_agent.py").write_text( + "class AIAgent:\n revision = 'before'\n", + encoding="utf-8", + ) + _git(agent_dir, "init", "-q") + _git(agent_dir, "add", "run_agent.py") + _git(agent_dir, "commit", "-qm", "before") + + probe = tmp_path / "probe.py" + probe.write_text( + """ +from pathlib import Path +import subprocess + +import api.streaming as streaming +from api import agent_runtime + +agent_dir = Path(__file__).parent / "hermes-agent" +assert agent_runtime._AGENT_DIR == agent_dir.resolve() +assert streaming._get_ai_agent().revision == "before" + +(agent_dir / "run_agent.py").write_text( + "class AIAgent:\\n revision = 'after'\\n", + encoding="utf-8", +) +subprocess.run(["git", "add", "run_agent.py"], cwd=agent_dir, check=True) +subprocess.run( + [ + "git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "commit", "-qm", "after", + ], + cwd=agent_dir, + check=True, +) + +try: + streaming._get_ai_agent() +except RuntimeError as exc: + message = str(exc) + assert "Hermes Agent was updated" in message + assert "Restart Hermes WebUI" in message +else: + raise AssertionError("stale in-process AIAgent was reused after its source revision changed") + +try: + agent_runtime.require_ai_agent_class() +except agent_runtime.AgentRuntimeChangedError as exc: + assert "Restart Hermes WebUI" in str(exc) +else: + raise AssertionError("unguarded AIAgent import was allowed after its source revision changed") +""".strip() + + "\n", + encoding="utf-8", + ) + + env = os.environ.copy() + env.update( + { + "HERMES_WEBUI_AGENT_DIR": str(agent_dir), + "HERMES_HOME": str(tmp_path / "hermes-home"), + "HERMES_WEBUI_STATE_DIR": str(tmp_path / "webui-state"), + "PYTHONPATH": os.pathsep.join((str(REPO), str(agent_dir))), + } + ) + result = subprocess.run( + [sys.executable, str(probe)], + cwd=REPO, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/test_agent_source_dependency_audit.py b/tests/test_agent_source_dependency_audit.py index 9f56f5fb..05267306 100644 --- a/tests/test_agent_source_dependency_audit.py +++ b/tests/test_agent_source_dependency_audit.py @@ -120,7 +120,7 @@ def test_audit_reports_runtime_agent_execution_imports(): classes = _class_by_id(_run_audit()) anchors = _anchors(classes["runtime_agent_execution"]) - assert ("api/streaming.py", "run_agent") in anchors + assert ("api/agent_runtime.py", "run_agent") in anchors assert ("api/routes.py", "tools.skills_tool") in anchors assert ("api/streaming.py", "tools.approval") in anchors assert ("api/routes.py", "cron.jobs") in anchors diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 82725429..8d38e449 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -166,13 +166,13 @@ def test_streaming_py_imports_has_pending(cleanup_test_sessions): def test_aiagent_imported_in_streaming(cleanup_test_sessions): - """R2b: api/streaming.py must import AIAgent. + """R2b: api/streaming.py must resolve AIAgent through the runtime guard. When missing, the streaming thread crashed immediately after being spawned. """ src = (REPO_ROOT / "api/streaming.py").read_text() - assert "AIAgent" in src, "AIAgent not referenced in api/streaming.py" - assert "from run_agent import AIAgent" in src or "import AIAgent" in src, \ - "AIAgent must be imported in api/streaming.py" + assert "get_ai_agent_class" in src, "guarded AIAgent resolver not referenced in api/streaming.py" + assert "from api.agent_runtime import" in src and "get_ai_agent_class" in src, \ + "AIAgent must be resolved through api.agent_runtime in api/streaming.py" # ── R5: SSE loop did not break on cancel event (Sprint 10 bug) ─────────────── diff --git a/tests/test_update_banner_fixes.py b/tests/test_update_banner_fixes.py index 0ab7a051..0da5ad41 100644 --- a/tests/test_update_banner_fixes.py +++ b/tests/test_update_banner_fixes.py @@ -1635,7 +1635,7 @@ class TestUpdateSummaryRouteModelSelection: assert '"update_summary"' not in src assert 'main_runtime=main_runtime' in src assert 'update summary auxiliary model failed; falling back to main model' in src - assert 'from run_agent import AIAgent' in src + assert 'require_ai_agent_class()' in src def test_summary_route_auxiliary_model_uses_active_profile_env(self, monkeypatch, tmp_path): import api.config as cfg