From b25e99c1b47ec328ed8ecec8570594b7f053fbab Mon Sep 17 00:00:00 2001 From: Frank Song Date: Tue, 7 Jul 2026 12:33:57 +0800 Subject: [PATCH] fix(wakeup): revalidate credential pause on refresh --- api/gateway_chat.py | 3 +- api/models.py | 44 ++++++++++++ api/routes.py | 12 ++++ tests/test_issue3929_process_wakeup_pause.py | 74 ++++++++++++++++++++ 4 files changed, 132 insertions(+), 1 deletion(-) diff --git a/api/gateway_chat.py b/api/gateway_chat.py index b0bc7aaa..548bd60e 100644 --- a/api/gateway_chat.py +++ b/api/gateway_chat.py @@ -30,7 +30,7 @@ from api.config import ( update_active_run, ) from api.helpers import _redact_text, redact_session_data -from api.models import get_session, merge_session_messages_append_only +from api.models import clear_process_wakeup_pause, get_session, merge_session_messages_append_only from api.run_journal import RunJournalWriter logger = logging.getLogger(__name__) @@ -901,6 +901,7 @@ def _run_gateway_chat_streaming( s.workspace = str(workspace) s.model = model s.model_provider = model_provider + clear_process_wakeup_pause(s, reason="run_completed") s.save() try: from api.goals import evaluate_goal_after_turn, has_active_goal diff --git a/api/models.py b/api/models.py index ffa3c1a8..bd832dc0 100644 --- a/api/models.py +++ b/api/models.py @@ -1643,6 +1643,7 @@ def record_process_wakeup_provider_unavailable_pause( return None now = time.time() key = _process_wakeup_pause_key(model, provider, classification) + credential_state_fingerprint = process_wakeup_credential_state_fingerprint(session) existing = getattr(session, 'process_wakeup_pause', None) same_window = ( isinstance(existing, dict) @@ -1672,6 +1673,7 @@ def record_process_wakeup_provider_unavailable_pause( 'last_error_at': now, 'visible_error_count': visible_error_count, 'suppressed_count': suppressed_count, + 'credential_state_fingerprint': credential_state_fingerprint, } return session.process_wakeup_pause @@ -1698,6 +1700,48 @@ def suppress_process_wakeup_for_provider_pause( session.process_wakeup_pause = pause return pause + +def process_wakeup_credential_state_fingerprint(session) -> str: + """Return a metadata-only fingerprint for credential/config state.""" + try: + hermes_home = _get_profile_home(getattr(session, 'profile', None)) + except Exception: + hermes_home = Path(os.environ.get('HERMES_HOME') or HOME).expanduser() + files = [] + for name in ('auth.json', 'config.yaml', 'config.yml', '.env'): + path = hermes_home / name + try: + stat = path.stat() + except FileNotFoundError: + files.append((name, 'missing')) + except OSError as exc: + files.append((name, 'error', exc.__class__.__name__)) + else: + kind = 'file' if path.is_file() else 'other' + files.append((name, kind, int(stat.st_mtime_ns), int(stat.st_size))) + payload = { + 'version': 1, + 'profile': _process_wakeup_pause_part(getattr(session, 'profile', None)), + 'files': files, + } + encoded = json.dumps(payload, sort_keys=True, separators=(',', ':')).encode('utf-8') + return hashlib.sha256(encoded).hexdigest() + + +def process_wakeup_pause_credential_state_changed(session) -> bool: + """Return True when a stored credential pause should be revalidated.""" + pause = getattr(session, 'process_wakeup_pause', None) + if not isinstance(pause, dict) or not pause.get('paused'): + return False + classification = _process_wakeup_pause_part(pause.get('classification')) + if classification not in PROCESS_WAKEUP_PROVIDER_UNAVAILABLE_TYPES: + return False + previous = str(pause.get('credential_state_fingerprint') or '').strip() + if not previous: + return True + return process_wakeup_credential_state_fingerprint(session) != previous + + def _get_profile_home(profile) -> Path: """Resolve the hermes agent home directory for the given profile. diff --git a/api/routes.py b/api/routes.py index 450b202e..074f2d12 100644 --- a/api/routes.py +++ b/api/routes.py @@ -8672,7 +8672,9 @@ from api.models import ( is_cron_session, is_safe_session_id, PROCESS_WAKEUP_PAUSE_ERROR, + clear_process_wakeup_pause, clear_process_wakeup_pause_if_model_changed, + process_wakeup_pause_credential_state_changed, suppress_process_wakeup_for_provider_pause, ) @@ -19902,6 +19904,16 @@ def start_session_turn( ) _paused_wakeup = None if turn_source == "process_wakeup": + if process_wakeup_pause_credential_state_changed(s): + if clear_process_wakeup_pause(s, reason='credential_state_changed'): + try: + s.save(touch_updated_at=False) + except Exception: + logger.debug( + "failed to persist process_wakeup credential-state reset for session %s", + session_id, + exc_info=True, + ) _paused_wakeup = suppress_process_wakeup_for_provider_pause( s, model=model, diff --git a/tests/test_issue3929_process_wakeup_pause.py b/tests/test_issue3929_process_wakeup_pause.py index f17cf134..5d987e0f 100644 --- a/tests/test_issue3929_process_wakeup_pause.py +++ b/tests/test_issue3929_process_wakeup_pause.py @@ -165,6 +165,7 @@ def test_credential_empty_process_wakeup_pauses_repeated_automatic_turns(tmp_pat assert saved.process_wakeup_pause["paused"] is True assert saved.process_wakeup_pause["classification"] == "credential_pool_empty" assert saved.process_wakeup_pause["suppressed_count"] == 0 + assert saved.process_wakeup_pause["credential_state_fingerprint"] context_before = list(saved.context_messages) messages_before = list(saved.messages) @@ -196,6 +197,70 @@ def test_credential_empty_process_wakeup_pauses_repeated_automatic_turns(tmp_pat assert "last_suppressed_at" in saved_after.process_wakeup_pause +def test_process_wakeup_pause_revalidates_when_credential_state_changes(tmp_path, monkeypatch): + hermes_home = tmp_path / "hermes-home" + hermes_home.mkdir() + auth_json = hermes_home / "auth.json" + auth_json.write_text('{"credential_pool": {}}\n', encoding="utf-8") + monkeypatch.setattr(models, "_get_profile_home", lambda _profile: hermes_home) + session = Session( + session_id="wakeup_pause_credential_refresh", + workspace=str(tmp_path), + model="test-model", + model_provider="test-provider", + ) + pause = models.record_process_wakeup_provider_unavailable_pause( + session, + classification="credential_pool_empty", + model="test-model", + provider="test-provider", + ) + assert pause is not None + paused_fingerprint = pause["credential_state_fingerprint"] + session.save() + models.SESSIONS[session.session_id] = session + + auth_json.write_text( + '{"credential_pool": {"test-provider": [{"id": "refilled-token"}]}}\n', + encoding="utf-8", + ) + assert models.process_wakeup_credential_state_fingerprint(session) != paused_fingerprint + + captured = {} + + def _fake_start_run(s, **kwargs): + captured["source"] = kwargs.get("source") + captured["model"] = kwargs.get("model") + captured["model_provider"] = kwargs.get("model_provider") + return {"stream_id": "stream-credential-refresh", "session_id": s.session_id, "_status": 200} + + monkeypatch.setattr(routes, "_resolve_chat_workspace_with_recovery", lambda _s, _w: str(tmp_path)) + monkeypatch.setattr(routes, "_read_profile_model_config", lambda _s, _p: (None, None, {})) + monkeypatch.setattr( + routes, + "_resolve_compatible_session_model_state", + lambda *_args, **_kwargs: ("test-model", "test-provider", False), + ) + monkeypatch.setattr(routes, "_start_run", _fake_start_run) + + response = routes.start_session_turn( + session.session_id, + "[IMPORTANT: Background process completed after credential refill.]", + source="process_wakeup", + ) + + assert response["_status"] == 200 + assert response["stream_id"] == "stream-credential-refresh" + assert captured == { + "source": "process_wakeup", + "model": "test-model", + "model_provider": "test-provider", + } + saved = Session.load(session.session_id) + assert saved is not None + assert saved.process_wakeup_pause == {} + + def test_stale_credential_empty_process_wakeup_still_records_pause(tmp_path): session = Session( session_id="wakeup_pause_stale", @@ -304,6 +369,15 @@ def test_success_path_clears_process_wakeup_pause_after_late_cancel_checks(): assert state_sync_idx < final_cancel_idx < pause_clear_idx < done_payload_idx +def test_gateway_success_path_clears_process_wakeup_pause_before_save(): + src = Path(__file__).parent.parent.joinpath("api", "gateway_chat.py").read_text(encoding="utf-8") + pending_clear_idx = src.index("s.pending_user_source = None") + pause_clear_idx = src.index('clear_process_wakeup_pause(s, reason="run_completed")') + save_idx = src.index("s.save()", pause_clear_idx) + + assert pending_clear_idx < pause_clear_idx < save_idx + + def test_process_wakeup_pause_does_not_suppress_explicit_non_wakeup_turn(tmp_path, monkeypatch): session = Session( session_id="wakeup_pause_manual_recover",