fix(wakeup): revalidate credential pause on refresh

This commit is contained in:
Frank Song 2026-07-07 12:33:57 +08:00
parent d72698b75d
commit b25e99c1b4
4 changed files with 132 additions and 1 deletions

View file

@ -30,7 +30,7 @@ from api.config import (
update_active_run,
)
from api.helpers import _redact_text, redact_session_data
from api.models import get_session, merge_session_messages_append_only
from api.models import clear_process_wakeup_pause, get_session, merge_session_messages_append_only
from api.run_journal import RunJournalWriter
logger = logging.getLogger(__name__)
@ -901,6 +901,7 @@ def _run_gateway_chat_streaming(
s.workspace = str(workspace)
s.model = model
s.model_provider = model_provider
clear_process_wakeup_pause(s, reason="run_completed")
s.save()
try:
from api.goals import evaluate_goal_after_turn, has_active_goal

View file

@ -1643,6 +1643,7 @@ def record_process_wakeup_provider_unavailable_pause(
return None
now = time.time()
key = _process_wakeup_pause_key(model, provider, classification)
credential_state_fingerprint = process_wakeup_credential_state_fingerprint(session)
existing = getattr(session, 'process_wakeup_pause', None)
same_window = (
isinstance(existing, dict)
@ -1672,6 +1673,7 @@ def record_process_wakeup_provider_unavailable_pause(
'last_error_at': now,
'visible_error_count': visible_error_count,
'suppressed_count': suppressed_count,
'credential_state_fingerprint': credential_state_fingerprint,
}
return session.process_wakeup_pause
@ -1698,6 +1700,48 @@ def suppress_process_wakeup_for_provider_pause(
session.process_wakeup_pause = pause
return pause
def process_wakeup_credential_state_fingerprint(session) -> str:
"""Return a metadata-only fingerprint for credential/config state."""
try:
hermes_home = _get_profile_home(getattr(session, 'profile', None))
except Exception:
hermes_home = Path(os.environ.get('HERMES_HOME') or HOME).expanduser()
files = []
for name in ('auth.json', 'config.yaml', 'config.yml', '.env'):
path = hermes_home / name
try:
stat = path.stat()
except FileNotFoundError:
files.append((name, 'missing'))
except OSError as exc:
files.append((name, 'error', exc.__class__.__name__))
else:
kind = 'file' if path.is_file() else 'other'
files.append((name, kind, int(stat.st_mtime_ns), int(stat.st_size)))
payload = {
'version': 1,
'profile': _process_wakeup_pause_part(getattr(session, 'profile', None)),
'files': files,
}
encoded = json.dumps(payload, sort_keys=True, separators=(',', ':')).encode('utf-8')
return hashlib.sha256(encoded).hexdigest()
def process_wakeup_pause_credential_state_changed(session) -> bool:
"""Return True when a stored credential pause should be revalidated."""
pause = getattr(session, 'process_wakeup_pause', None)
if not isinstance(pause, dict) or not pause.get('paused'):
return False
classification = _process_wakeup_pause_part(pause.get('classification'))
if classification not in PROCESS_WAKEUP_PROVIDER_UNAVAILABLE_TYPES:
return False
previous = str(pause.get('credential_state_fingerprint') or '').strip()
if not previous:
return True
return process_wakeup_credential_state_fingerprint(session) != previous
def _get_profile_home(profile) -> Path:
"""Resolve the hermes agent home directory for the given profile.

View file

@ -8672,7 +8672,9 @@ from api.models import (
is_cron_session,
is_safe_session_id,
PROCESS_WAKEUP_PAUSE_ERROR,
clear_process_wakeup_pause,
clear_process_wakeup_pause_if_model_changed,
process_wakeup_pause_credential_state_changed,
suppress_process_wakeup_for_provider_pause,
)
@ -19902,6 +19904,16 @@ def start_session_turn(
)
_paused_wakeup = None
if turn_source == "process_wakeup":
if process_wakeup_pause_credential_state_changed(s):
if clear_process_wakeup_pause(s, reason='credential_state_changed'):
try:
s.save(touch_updated_at=False)
except Exception:
logger.debug(
"failed to persist process_wakeup credential-state reset for session %s",
session_id,
exc_info=True,
)
_paused_wakeup = suppress_process_wakeup_for_provider_pause(
s,
model=model,

View file

@ -165,6 +165,7 @@ def test_credential_empty_process_wakeup_pauses_repeated_automatic_turns(tmp_pat
assert saved.process_wakeup_pause["paused"] is True
assert saved.process_wakeup_pause["classification"] == "credential_pool_empty"
assert saved.process_wakeup_pause["suppressed_count"] == 0
assert saved.process_wakeup_pause["credential_state_fingerprint"]
context_before = list(saved.context_messages)
messages_before = list(saved.messages)
@ -196,6 +197,70 @@ def test_credential_empty_process_wakeup_pauses_repeated_automatic_turns(tmp_pat
assert "last_suppressed_at" in saved_after.process_wakeup_pause
def test_process_wakeup_pause_revalidates_when_credential_state_changes(tmp_path, monkeypatch):
hermes_home = tmp_path / "hermes-home"
hermes_home.mkdir()
auth_json = hermes_home / "auth.json"
auth_json.write_text('{"credential_pool": {}}\n', encoding="utf-8")
monkeypatch.setattr(models, "_get_profile_home", lambda _profile: hermes_home)
session = Session(
session_id="wakeup_pause_credential_refresh",
workspace=str(tmp_path),
model="test-model",
model_provider="test-provider",
)
pause = models.record_process_wakeup_provider_unavailable_pause(
session,
classification="credential_pool_empty",
model="test-model",
provider="test-provider",
)
assert pause is not None
paused_fingerprint = pause["credential_state_fingerprint"]
session.save()
models.SESSIONS[session.session_id] = session
auth_json.write_text(
'{"credential_pool": {"test-provider": [{"id": "refilled-token"}]}}\n',
encoding="utf-8",
)
assert models.process_wakeup_credential_state_fingerprint(session) != paused_fingerprint
captured = {}
def _fake_start_run(s, **kwargs):
captured["source"] = kwargs.get("source")
captured["model"] = kwargs.get("model")
captured["model_provider"] = kwargs.get("model_provider")
return {"stream_id": "stream-credential-refresh", "session_id": s.session_id, "_status": 200}
monkeypatch.setattr(routes, "_resolve_chat_workspace_with_recovery", lambda _s, _w: str(tmp_path))
monkeypatch.setattr(routes, "_read_profile_model_config", lambda _s, _p: (None, None, {}))
monkeypatch.setattr(
routes,
"_resolve_compatible_session_model_state",
lambda *_args, **_kwargs: ("test-model", "test-provider", False),
)
monkeypatch.setattr(routes, "_start_run", _fake_start_run)
response = routes.start_session_turn(
session.session_id,
"[IMPORTANT: Background process completed after credential refill.]",
source="process_wakeup",
)
assert response["_status"] == 200
assert response["stream_id"] == "stream-credential-refresh"
assert captured == {
"source": "process_wakeup",
"model": "test-model",
"model_provider": "test-provider",
}
saved = Session.load(session.session_id)
assert saved is not None
assert saved.process_wakeup_pause == {}
def test_stale_credential_empty_process_wakeup_still_records_pause(tmp_path):
session = Session(
session_id="wakeup_pause_stale",
@ -304,6 +369,15 @@ def test_success_path_clears_process_wakeup_pause_after_late_cancel_checks():
assert state_sync_idx < final_cancel_idx < pause_clear_idx < done_payload_idx
def test_gateway_success_path_clears_process_wakeup_pause_before_save():
src = Path(__file__).parent.parent.joinpath("api", "gateway_chat.py").read_text(encoding="utf-8")
pending_clear_idx = src.index("s.pending_user_source = None")
pause_clear_idx = src.index('clear_process_wakeup_pause(s, reason="run_completed")')
save_idx = src.index("s.save()", pause_clear_idx)
assert pending_clear_idx < pause_clear_idx < save_idx
def test_process_wakeup_pause_does_not_suppress_explicit_non_wakeup_turn(tmp_path, monkeypatch):
session = Session(
session_id="wakeup_pause_manual_recover",