fix(wakeup): revalidate credential pause on refresh
This commit is contained in:
parent
d72698b75d
commit
b25e99c1b4
4 changed files with 132 additions and 1 deletions
|
|
@ -30,7 +30,7 @@ from api.config import (
|
|||
update_active_run,
|
||||
)
|
||||
from api.helpers import _redact_text, redact_session_data
|
||||
from api.models import get_session, merge_session_messages_append_only
|
||||
from api.models import clear_process_wakeup_pause, get_session, merge_session_messages_append_only
|
||||
from api.run_journal import RunJournalWriter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -901,6 +901,7 @@ def _run_gateway_chat_streaming(
|
|||
s.workspace = str(workspace)
|
||||
s.model = model
|
||||
s.model_provider = model_provider
|
||||
clear_process_wakeup_pause(s, reason="run_completed")
|
||||
s.save()
|
||||
try:
|
||||
from api.goals import evaluate_goal_after_turn, has_active_goal
|
||||
|
|
|
|||
|
|
@ -1643,6 +1643,7 @@ def record_process_wakeup_provider_unavailable_pause(
|
|||
return None
|
||||
now = time.time()
|
||||
key = _process_wakeup_pause_key(model, provider, classification)
|
||||
credential_state_fingerprint = process_wakeup_credential_state_fingerprint(session)
|
||||
existing = getattr(session, 'process_wakeup_pause', None)
|
||||
same_window = (
|
||||
isinstance(existing, dict)
|
||||
|
|
@ -1672,6 +1673,7 @@ def record_process_wakeup_provider_unavailable_pause(
|
|||
'last_error_at': now,
|
||||
'visible_error_count': visible_error_count,
|
||||
'suppressed_count': suppressed_count,
|
||||
'credential_state_fingerprint': credential_state_fingerprint,
|
||||
}
|
||||
return session.process_wakeup_pause
|
||||
|
||||
|
|
@ -1698,6 +1700,48 @@ def suppress_process_wakeup_for_provider_pause(
|
|||
session.process_wakeup_pause = pause
|
||||
return pause
|
||||
|
||||
|
||||
def process_wakeup_credential_state_fingerprint(session) -> str:
|
||||
"""Return a metadata-only fingerprint for credential/config state."""
|
||||
try:
|
||||
hermes_home = _get_profile_home(getattr(session, 'profile', None))
|
||||
except Exception:
|
||||
hermes_home = Path(os.environ.get('HERMES_HOME') or HOME).expanduser()
|
||||
files = []
|
||||
for name in ('auth.json', 'config.yaml', 'config.yml', '.env'):
|
||||
path = hermes_home / name
|
||||
try:
|
||||
stat = path.stat()
|
||||
except FileNotFoundError:
|
||||
files.append((name, 'missing'))
|
||||
except OSError as exc:
|
||||
files.append((name, 'error', exc.__class__.__name__))
|
||||
else:
|
||||
kind = 'file' if path.is_file() else 'other'
|
||||
files.append((name, kind, int(stat.st_mtime_ns), int(stat.st_size)))
|
||||
payload = {
|
||||
'version': 1,
|
||||
'profile': _process_wakeup_pause_part(getattr(session, 'profile', None)),
|
||||
'files': files,
|
||||
}
|
||||
encoded = json.dumps(payload, sort_keys=True, separators=(',', ':')).encode('utf-8')
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def process_wakeup_pause_credential_state_changed(session) -> bool:
|
||||
"""Return True when a stored credential pause should be revalidated."""
|
||||
pause = getattr(session, 'process_wakeup_pause', None)
|
||||
if not isinstance(pause, dict) or not pause.get('paused'):
|
||||
return False
|
||||
classification = _process_wakeup_pause_part(pause.get('classification'))
|
||||
if classification not in PROCESS_WAKEUP_PROVIDER_UNAVAILABLE_TYPES:
|
||||
return False
|
||||
previous = str(pause.get('credential_state_fingerprint') or '').strip()
|
||||
if not previous:
|
||||
return True
|
||||
return process_wakeup_credential_state_fingerprint(session) != previous
|
||||
|
||||
|
||||
def _get_profile_home(profile) -> Path:
|
||||
"""Resolve the hermes agent home directory for the given profile.
|
||||
|
||||
|
|
|
|||
|
|
@ -8672,7 +8672,9 @@ from api.models import (
|
|||
is_cron_session,
|
||||
is_safe_session_id,
|
||||
PROCESS_WAKEUP_PAUSE_ERROR,
|
||||
clear_process_wakeup_pause,
|
||||
clear_process_wakeup_pause_if_model_changed,
|
||||
process_wakeup_pause_credential_state_changed,
|
||||
suppress_process_wakeup_for_provider_pause,
|
||||
)
|
||||
|
||||
|
|
@ -19902,6 +19904,16 @@ def start_session_turn(
|
|||
)
|
||||
_paused_wakeup = None
|
||||
if turn_source == "process_wakeup":
|
||||
if process_wakeup_pause_credential_state_changed(s):
|
||||
if clear_process_wakeup_pause(s, reason='credential_state_changed'):
|
||||
try:
|
||||
s.save(touch_updated_at=False)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"failed to persist process_wakeup credential-state reset for session %s",
|
||||
session_id,
|
||||
exc_info=True,
|
||||
)
|
||||
_paused_wakeup = suppress_process_wakeup_for_provider_pause(
|
||||
s,
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ def test_credential_empty_process_wakeup_pauses_repeated_automatic_turns(tmp_pat
|
|||
assert saved.process_wakeup_pause["paused"] is True
|
||||
assert saved.process_wakeup_pause["classification"] == "credential_pool_empty"
|
||||
assert saved.process_wakeup_pause["suppressed_count"] == 0
|
||||
assert saved.process_wakeup_pause["credential_state_fingerprint"]
|
||||
context_before = list(saved.context_messages)
|
||||
messages_before = list(saved.messages)
|
||||
|
||||
|
|
@ -196,6 +197,70 @@ def test_credential_empty_process_wakeup_pauses_repeated_automatic_turns(tmp_pat
|
|||
assert "last_suppressed_at" in saved_after.process_wakeup_pause
|
||||
|
||||
|
||||
def test_process_wakeup_pause_revalidates_when_credential_state_changes(tmp_path, monkeypatch):
|
||||
hermes_home = tmp_path / "hermes-home"
|
||||
hermes_home.mkdir()
|
||||
auth_json = hermes_home / "auth.json"
|
||||
auth_json.write_text('{"credential_pool": {}}\n', encoding="utf-8")
|
||||
monkeypatch.setattr(models, "_get_profile_home", lambda _profile: hermes_home)
|
||||
session = Session(
|
||||
session_id="wakeup_pause_credential_refresh",
|
||||
workspace=str(tmp_path),
|
||||
model="test-model",
|
||||
model_provider="test-provider",
|
||||
)
|
||||
pause = models.record_process_wakeup_provider_unavailable_pause(
|
||||
session,
|
||||
classification="credential_pool_empty",
|
||||
model="test-model",
|
||||
provider="test-provider",
|
||||
)
|
||||
assert pause is not None
|
||||
paused_fingerprint = pause["credential_state_fingerprint"]
|
||||
session.save()
|
||||
models.SESSIONS[session.session_id] = session
|
||||
|
||||
auth_json.write_text(
|
||||
'{"credential_pool": {"test-provider": [{"id": "refilled-token"}]}}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert models.process_wakeup_credential_state_fingerprint(session) != paused_fingerprint
|
||||
|
||||
captured = {}
|
||||
|
||||
def _fake_start_run(s, **kwargs):
|
||||
captured["source"] = kwargs.get("source")
|
||||
captured["model"] = kwargs.get("model")
|
||||
captured["model_provider"] = kwargs.get("model_provider")
|
||||
return {"stream_id": "stream-credential-refresh", "session_id": s.session_id, "_status": 200}
|
||||
|
||||
monkeypatch.setattr(routes, "_resolve_chat_workspace_with_recovery", lambda _s, _w: str(tmp_path))
|
||||
monkeypatch.setattr(routes, "_read_profile_model_config", lambda _s, _p: (None, None, {}))
|
||||
monkeypatch.setattr(
|
||||
routes,
|
||||
"_resolve_compatible_session_model_state",
|
||||
lambda *_args, **_kwargs: ("test-model", "test-provider", False),
|
||||
)
|
||||
monkeypatch.setattr(routes, "_start_run", _fake_start_run)
|
||||
|
||||
response = routes.start_session_turn(
|
||||
session.session_id,
|
||||
"[IMPORTANT: Background process completed after credential refill.]",
|
||||
source="process_wakeup",
|
||||
)
|
||||
|
||||
assert response["_status"] == 200
|
||||
assert response["stream_id"] == "stream-credential-refresh"
|
||||
assert captured == {
|
||||
"source": "process_wakeup",
|
||||
"model": "test-model",
|
||||
"model_provider": "test-provider",
|
||||
}
|
||||
saved = Session.load(session.session_id)
|
||||
assert saved is not None
|
||||
assert saved.process_wakeup_pause == {}
|
||||
|
||||
|
||||
def test_stale_credential_empty_process_wakeup_still_records_pause(tmp_path):
|
||||
session = Session(
|
||||
session_id="wakeup_pause_stale",
|
||||
|
|
@ -304,6 +369,15 @@ def test_success_path_clears_process_wakeup_pause_after_late_cancel_checks():
|
|||
assert state_sync_idx < final_cancel_idx < pause_clear_idx < done_payload_idx
|
||||
|
||||
|
||||
def test_gateway_success_path_clears_process_wakeup_pause_before_save():
|
||||
src = Path(__file__).parent.parent.joinpath("api", "gateway_chat.py").read_text(encoding="utf-8")
|
||||
pending_clear_idx = src.index("s.pending_user_source = None")
|
||||
pause_clear_idx = src.index('clear_process_wakeup_pause(s, reason="run_completed")')
|
||||
save_idx = src.index("s.save()", pause_clear_idx)
|
||||
|
||||
assert pending_clear_idx < pause_clear_idx < save_idx
|
||||
|
||||
|
||||
def test_process_wakeup_pause_does_not_suppress_explicit_non_wakeup_turn(tmp_path, monkeypatch):
|
||||
session = Session(
|
||||
session_id="wakeup_pause_manual_recover",
|
||||
|
|
|
|||
Loading…
Reference in a new issue