fix: stop stale agent runtime after source updates

api/agent_runtime.py

@@ -0,0 +1,79 @@
+"""Fail-closed guard for in-process Hermes Agent source revisions.
+
+Hermes WebUI currently imports ``run_agent.AIAgent`` into its long-lived server
+process. If the Agent checkout changes while that process is alive, Python may
+combine already-cached modules with newly-read source. Refuse to reuse that
+mixed runtime and require a clean WebUI restart instead.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+import subprocess
+import threading
+
+from api.config import _AGENT_DIR
+
+
+_RESTART_MESSAGE = (
+    "Hermes Agent was updated while Hermes WebUI was running. "
+    "Restart Hermes WebUI before retrying this action."
+)
+
+
+def _read_agent_revision(agent_dir: Path) -> str | None:
+    """Return the checkout HEAD, or ``None`` for a non-Git/unavailable source."""
+    try:
+        result = subprocess.run(
+            ["git", "-C", str(agent_dir), "rev-parse", "--verify", "HEAD"],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=2,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    revision = result.stdout.strip()
+    return revision if result.returncode == 0 and revision else None
+
+
+_AGENT_REVISION = _read_agent_revision(_AGENT_DIR)
+_AIAgent = None
+_RUNTIME_LOCK = threading.Lock()
+
+
+class AgentRuntimeChangedError(RuntimeError):
+    """Raised when the loaded Agent runtime no longer matches its source tree."""
+
+
+def ensure_agent_runtime_current() -> None:
+    """Reject a known Git checkout change instead of mixing Python modules."""
+    if _AGENT_REVISION is None:
+        return
+    if _read_agent_revision(_AGENT_DIR) != _AGENT_REVISION:
+        raise AgentRuntimeChangedError(_RESTART_MESSAGE)
+
+
+def require_ai_agent_class():
+    """Import ``AIAgent`` after proving the loaded source revision is current."""
+    ensure_agent_runtime_current()
+    from run_agent import AIAgent  # noqa: PLC0415
+
+    return AIAgent
+
+
+def get_ai_agent_class():
+    """Return ``AIAgent`` while preserving the existing lazy-import retry."""
+    global _AIAgent, _AGENT_REVISION
+
+    with _RUNTIME_LOCK:
+        ensure_agent_runtime_current()
+        if _AIAgent is None:
+            try:
+                agent_class = require_ai_agent_class()
+            except ImportError:
+                return None
+            _AIAgent = agent_class
+            if _AGENT_REVISION is None:
+                _AGENT_REVISION = _read_agent_revision(_AGENT_DIR)
+        return _AIAgent

api/routes.py

@@ -34,6 +34,7 @@ from contextlib import closing
 from urllib.parse import parse_qs, quote, unquote, urljoin, urlsplit
 from urllib.error import HTTPError, URLError
 from urllib.request import HTTPRedirectHandler, HTTPSHandler, ProxyHandler, Request, build_opener
+from api.agent_runtime import ensure_agent_runtime_current, require_ai_agent_class
 from api.agent_sessions import (
     MESSAGING_SOURCES,
     _looks_like_default_cli_title,
@@ -16052,6 +16053,7 @@ def handle_post(handler, parsed) -> bool:
                     "api_key": _main_api_key,
                 }
 
+                ensure_agent_runtime_current()
                 try:
                     from agent.auxiliary_client import get_text_auxiliary_client
 
@@ -16072,7 +16074,7 @@ def handle_post(handler, parsed) -> bool:
                 except Exception as _e:
                     logger.debug("update summary auxiliary model failed; falling back to main model: %s", _e)
 
-                from run_agent import AIAgent
+                AIAgent = require_ai_agent_class()
 
                 agent = AIAgent(
                     model=_main_model,
@@ -21991,7 +21993,7 @@ def _handle_chat_sync(handler, body):
         os.environ["HERMES_EXEC_ASK"] = "1"
         os.environ["HERMES_SESSION_KEY"] = s.session_id
     try:
-        from run_agent import AIAgent
+        AIAgent = require_ai_agent_class()
 
         with CHAT_LOCK:
             from api.config import (
@@ -22622,6 +22624,7 @@ def _llm_git_commit_message(system_prompt: str, user_prompt: str, session=None)
             "base_url": _main_base_url,
             "api_key": _main_api_key,
         }
+        ensure_agent_runtime_current()
         try:
             from agent.auxiliary_client import get_text_auxiliary_client
 
@@ -22638,7 +22641,7 @@ def _llm_git_commit_message(system_prompt: str, user_prompt: str, session=None)
         except Exception as _e:
             logger.debug("git commit message auxiliary model failed; falling back to main model: %s", _e)
 
-        from run_agent import AIAgent
+        AIAgent = require_ai_agent_class()
 
         agent = AIAgent(
             model=_main_model,
@@ -24035,10 +24038,11 @@ def _handle_session_compress(handler, body):
                     focus_topic,
                 )
 
+        ensure_agent_runtime_current()
         import api.config as _cfg
         from api.oauth import resolve_runtime_provider_with_anthropic_env_lock
         import hermes_cli.runtime_provider as _runtime_provider
-        import run_agent as _run_agent
+        AIAgent = require_ai_agent_class()
 
         resolved_model, resolved_provider, resolved_base_url = _cfg.resolve_model_provider(
             _cfg.model_with_provider_context(s.model, getattr(s, "model_provider", None))
@@ -24081,7 +24085,7 @@ def _handle_session_compress(handler, body):
         )
         approx_tokens = _estimate_messages_tokens_rough(original_messages)
 
-        agent = _run_agent.AIAgent(
+        agent = AIAgent(
             model=resolved_model,
             provider=resolved_provider,
             base_url=resolved_base_url,
@@ -24685,10 +24689,11 @@ def _handle_handoff_summary(handler, body):
 
         # Call LLM for summary.
     try:
+        ensure_agent_runtime_current()
         import api.config as _cfg
         from api.oauth import resolve_runtime_provider_with_anthropic_env_lock
         import hermes_cli.runtime_provider as _runtime_provider
-        import run_agent as _run_agent
+        AIAgent = require_ai_agent_class()
 
         # Try to resolve model from an existing session, fall back to default.
         resolved_model = None
@@ -24744,7 +24749,7 @@ def _handle_handoff_summary(handler, body):
                 "fallback": True,
             })
 
-        agent = _run_agent.AIAgent(
+        agent = AIAgent(
             model=resolved_model,
             provider=resolved_provider,
             base_url=resolved_base_url,

api/streaming.py

@@ -564,10 +564,13 @@ def _prewarm_skill_tool_modules():
 
 
 # Lazy import to avoid circular deps -- hermes-agent is on sys.path via api/config.py
-try:
-    from run_agent import AIAgent
-except ImportError:
-    AIAgent = None
+from api.agent_runtime import ensure_agent_runtime_current, get_ai_agent_class
+
+
+# Eagerly attempt the import at startup, matching the pre-guard behavior. If
+# dependencies are not ready yet, _get_ai_agent() retries when a chat starts.
+AIAgent = get_ai_agent_class()
+
 
 def _get_ai_agent():
     """Return AIAgent class, retrying the import if the initial attempt failed.
@@ -575,15 +578,13 @@ def _get_ai_agent():
     auto_install_agent_deps() in server.py may install missing packages after
     this module is first imported (common in Docker with a volume-mounted agent).
     Re-attempting the import here picks up the newly installed packages without
-    requiring a server restart.
+    requiring a server restart. The shared runtime guard also refuses to reuse
+    cached Agent modules after the source checkout changes.
     """
     global AIAgent
+    ensure_agent_runtime_current()
     if AIAgent is None:
-        try:
-            from run_agent import AIAgent as _cls  # noqa: PLC0415
-            AIAgent = _cls
-        except ImportError:
-            pass
+        AIAgent = get_ai_agent_class()
     return AIAgent
 
 
@@ -7213,6 +7214,7 @@ def _run_agent_streaming(
             logger.debug("register_process_session failed", exc_info=True)
         # first-time module initialisation (which can be slow) does not
         # block other concurrent sessions waiting on _ENV_LOCK (#2024).
+        ensure_agent_runtime_current()
         _prewarm_skill_tool_modules()
         _install_streaming_cronjob_profile_wrapper()
         # Still set process-level env as fallback for tools that bypass thread-local

docs/troubleshooting.md

@@ -177,6 +177,28 @@ turn in the exhausted session instead of being blocked with recovery guidance.
 
 ---
 
+## "Hermes Agent was updated while Hermes WebUI was running"
+
+**Symptom.** An action that uses the in-process Agent runtime stops with a message telling you to restart Hermes WebUI. This can happen after `hermes update`, a Git checkout/pull in the Agent source tree, or another tool updates Hermes Agent without restarting the already-running WebUI backend.
+
+**Why.** WebUI currently imports `run_agent.AIAgent` into its long-lived Python process. Python keeps imported modules in memory. Continuing after the Agent Git revision changes could combine cached modules from the old revision with source read from the new revision, producing misleading `ImportError`s or inconsistent runtime state. WebUI therefore fails closed instead of attempting a partial in-process reload.
+
+**Diagnostic.** Compare the running WebUI process start time with the Agent checkout revision and recent update history. If the Agent was updated after WebUI started, restart WebUI before investigating individual missing-symbol errors.
+
+**Fix.** Restart using the same launch method that started WebUI:
+
+```bash
+./ctl.sh restart
+# Or, for a user systemd service:
+systemctl --user restart hermes-webui.service
+```
+
+If you launched `python3 bootstrap.py` in the foreground, stop it with Ctrl-C and start it again. Restarting the whole computer or WSL is not required when restarting the WebUI backend succeeds.
+
+**When to file a bug.** File a WebUI bug if the restart-required message appears even though the Agent revision did not change, or if a clean WebUI restart still produces the same import error. Include the WebUI launch method, WebUI revision, Agent revision, and the sanitized error text.
+
+---
+
 ## Other troubleshooting
 
 This document grows over time. If a recurring failure mode isn't covered here yet, add it via PR. The format for each entry: **Symptom → Why → Diagnostic commands → Fix → When to file a bug**.

tests/test_agent_runtime_revision_guard.py

@@ -0,0 +1,101 @@
+"""Regression coverage for Hermes Agent source changes during a WebUI process lifetime."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+
+REPO = Path(__file__).resolve().parents[1]
+
+
+def _git(repo: Path, *args: str) -> str:
+    result = subprocess.run(
+        ["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", *args],
+        cwd=repo,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return result.stdout.strip()
+
+
+def test_loaded_agent_runtime_fails_closed_after_source_revision_changes(tmp_path: Path):
+    agent_dir = tmp_path / "hermes-agent"
+    agent_dir.mkdir()
+    (agent_dir / "run_agent.py").write_text(
+        "class AIAgent:\n    revision = 'before'\n",
+        encoding="utf-8",
+    )
+    _git(agent_dir, "init", "-q")
+    _git(agent_dir, "add", "run_agent.py")
+    _git(agent_dir, "commit", "-qm", "before")
+
+    probe = tmp_path / "probe.py"
+    probe.write_text(
+        """
+from pathlib import Path
+import subprocess
+
+import api.streaming as streaming
+from api import agent_runtime
+
+agent_dir = Path(__file__).parent / "hermes-agent"
+assert agent_runtime._AGENT_DIR == agent_dir.resolve()
+assert streaming._get_ai_agent().revision == "before"
+
+(agent_dir / "run_agent.py").write_text(
+    "class AIAgent:\\n    revision = 'after'\\n",
+    encoding="utf-8",
+)
+subprocess.run(["git", "add", "run_agent.py"], cwd=agent_dir, check=True)
+subprocess.run(
+    [
+        "git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid",
+        "commit", "-qm", "after",
+    ],
+    cwd=agent_dir,
+    check=True,
+)
+
+try:
+    streaming._get_ai_agent()
+except RuntimeError as exc:
+    message = str(exc)
+    assert "Hermes Agent was updated" in message
+    assert "Restart Hermes WebUI" in message
+else:
+    raise AssertionError("stale in-process AIAgent was reused after its source revision changed")
+
+try:
+    agent_runtime.require_ai_agent_class()
+except agent_runtime.AgentRuntimeChangedError as exc:
+    assert "Restart Hermes WebUI" in str(exc)
+else:
+    raise AssertionError("unguarded AIAgent import was allowed after its source revision changed")
+""".strip()
+        + "\n",
+        encoding="utf-8",
+    )
+
+    env = os.environ.copy()
+    env.update(
+        {
+            "HERMES_WEBUI_AGENT_DIR": str(agent_dir),
+            "HERMES_HOME": str(tmp_path / "hermes-home"),
+            "HERMES_WEBUI_STATE_DIR": str(tmp_path / "webui-state"),
+            "PYTHONPATH": os.pathsep.join((str(REPO), str(agent_dir))),
+        }
+    )
+    result = subprocess.run(
+        [sys.executable, str(probe)],
+        cwd=REPO,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+
+    assert result.returncode == 0, result.stdout + result.stderr

tests/test_agent_source_dependency_audit.py

@@ -120,7 +120,7 @@ def test_audit_reports_runtime_agent_execution_imports():
     classes = _class_by_id(_run_audit())
     anchors = _anchors(classes["runtime_agent_execution"])
 
-    assert ("api/streaming.py", "run_agent") in anchors
+    assert ("api/agent_runtime.py", "run_agent") in anchors
     assert ("api/routes.py", "tools.skills_tool") in anchors
     assert ("api/streaming.py", "tools.approval") in anchors
     assert ("api/routes.py", "cron.jobs") in anchors

tests/test_regressions.py

@@ -166,13 +166,13 @@ def test_streaming_py_imports_has_pending(cleanup_test_sessions):
 
 
 def test_aiagent_imported_in_streaming(cleanup_test_sessions):
-    """R2b: api/streaming.py must import AIAgent.
+    """R2b: api/streaming.py must resolve AIAgent through the runtime guard.
     When missing, the streaming thread crashed immediately after being spawned.
     """
     src = (REPO_ROOT / "api/streaming.py").read_text()
-    assert "AIAgent" in src, "AIAgent not referenced in api/streaming.py"
-    assert "from run_agent import AIAgent" in src or "import AIAgent" in src, \
-        "AIAgent must be imported in api/streaming.py"
+    assert "get_ai_agent_class" in src, "guarded AIAgent resolver not referenced in api/streaming.py"
+    assert "from api.agent_runtime import" in src and "get_ai_agent_class" in src, \
+        "AIAgent must be resolved through api.agent_runtime in api/streaming.py"
 
 
 # ── R5: SSE loop did not break on cancel event (Sprint 10 bug) ───────────────

tests/test_update_banner_fixes.py

@@ -1635,7 +1635,7 @@ class TestUpdateSummaryRouteModelSelection:
         assert '"update_summary"' not in src
         assert 'main_runtime=main_runtime' in src
         assert 'update summary auxiliary model failed; falling back to main model' in src
-        assert 'from run_agent import AIAgent' in src
+        assert 'require_ai_agent_class()' in src
 
     def test_summary_route_auxiliary_model_uses_active_profile_env(self, monkeypatch, tmp_path):
         import api.config as cfg