From a61880b977b4b791682995f7925464913f4d67ee Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Mon, 20 Jul 2026 03:52:18 -0700 Subject: [PATCH] Release: terminal-error lifecycle gate row (#6354, @franksong2702) (#6358) * test: add terminal-error lifecycle matrix row * test: reject empty terminal process rows --------- Co-authored-by: Frank Song Co-authored-by: nesquena-hermes --- .github/workflows/conversation-lifecycle.yml | 7 +- TESTING.md | 31 +- tests/browser_conversation_lifecycle.py | 451 ++++++++++++++++--- 3 files changed, 421 insertions(+), 68 deletions(-) diff --git a/.github/workflows/conversation-lifecycle.yml b/.github/workflows/conversation-lifecycle.yml index b3853fbd..fe9fd1c5 100644 --- a/.github/workflows/conversation-lifecycle.yml +++ b/.github/workflows/conversation-lifecycle.yml @@ -31,6 +31,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 continue-on-error: true + strategy: + fail-fast: false + matrix: + scenario: [normal, terminal-error] steps: - uses: actions/checkout@v4 @@ -53,12 +57,13 @@ jobs: - name: Run conversation lifecycle gate env: LIFECYCLE_ARTIFACT_DIR: lifecycle-artifacts + LIFECYCLE_SCENARIO: ${{ matrix.scenario }} run: python tests/browser_conversation_lifecycle.py - name: Upload failure artifacts if: failure() uses: actions/upload-artifact@v4 with: - name: conversation-lifecycle-failure + name: conversation-lifecycle-failure-${{ matrix.scenario }} path: lifecycle-artifacts if-no-files-found: ignore diff --git a/TESTING.md b/TESTING.md index 5a42edb9..3748ded7 100644 --- a/TESTING.md +++ b/TESTING.md @@ -93,18 +93,24 @@ that breaks the page for everyone). ## Public conversation lifecycle gate -`tests/browser_conversation_lifecycle.py` adds the first public, deterministic -chat golden path. It drives the real composer and real WebUI server in Chromium, -while a localhost-only fixture supplies reasoning, tool, and final-answer events -through the existing Hermes Gateway Runs API. The gate asserts semantic activity -during live streaming, after settlement, and after a hard reload, including +`tests/browser_conversation_lifecycle.py` adds a public deterministic +multi-row lifecycle gate. It drives the real composer and real WebUI server in Chromium, +while a localhost-only fixture supplies reasoning, tool, process, and final/error +events through the existing Hermes Gateway Runs API. The gate now covers both +normal and terminal-error proof-matrix rows, asserting semantic activity during +live streaming, after settlement, and after hard reload, including transcript-backed `activity_scene_v1` persistence and zero unexpected browser errors. It uses isolated temporary state and no provider credentials. ```bash pip install -r requirements.txt playwright python -m playwright install --with-deps chromium + +# Normal-path deterministic conversation lifecycle gate. python tests/browser_conversation_lifecycle.py + +# Terminal-error lifecycle gate (new row in the proof matrix). +LIFECYCLE_SCENARIO=terminal-error python tests/browser_conversation_lifecycle.py ``` To certify that the gate catches its target failure, the test owns an opt-in @@ -114,12 +120,19 @@ must fail at the hard-reload boundary: ```bash LIFECYCLE_TEST_BITE=drop-anchor-persistence \ python tests/browser_conversation_lifecycle.py + +# Terminal-state-specific mutation bite: remove terminal row from persisted scene +# so hard reload cannot recover terminal status. +LIFECYCLE_SCENARIO=terminal-error \ +LIFECYCLE_TEST_BITE=drop-terminal-anchor-row \ + python tests/browser_conversation_lifecycle.py ``` -The dedicated `Conversation lifecycle (informational)` workflow runs this test -without blocking merges while the public matrix is being established. The -maintainer's private QA harness remains broader; later public slices will add -session switching, reconnect/replay, cancellation, compression, and recovery. +The dedicated `Conversation lifecycle (informational)` workflow runs both current +proof rows (`normal` and `terminal-error`) and stays non-blocking while the public +matrix expands to additional behavior rows. The maintainer's private QA harness +remains broader; later public slices will add session switching, reconnect/replay, +cancellation, compression, and recovery. `tests/test_static_js_runtime_lint.py` runs this automatically when eslint is present diff --git a/tests/browser_conversation_lifecycle.py b/tests/browser_conversation_lifecycle.py index ea534713..539d7804 100644 --- a/tests/browser_conversation_lifecycle.py +++ b/tests/browser_conversation_lifecycle.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Public browser gate for the normal conversation lifecycle. +"""Public browser gate for the normal + terminal-error lifecycle. This test boots the real WebUI server with isolated state, drives the real chat composer in Chromium, and supplies deterministic runtime events through the @@ -28,9 +28,12 @@ from urllib.parse import urlsplit PROMPT = "Exercise the public conversation lifecycle gate." REASONING_TEXT = "Checking the persistent assistant turn." FINAL_TEXT = "Lifecycle gate final answer." +FINAL_ACK_TEXT = "Lifecycle" FINAL_PREFIX = "Lifecycle gate " FINAL_SUFFIX = "final answer." -FINAL_ACK_TEXT = "Lifecycle" +TERMINAL_PROCESS_TEXT = "Lifecycle terminal process check" +TERMINAL_ERROR_TEXT = "Lifecycle gate encountered a terminal-side error." +SCENARIO = os.environ.get("LIFECYCLE_SCENARIO", "normal").strip() or "normal" TOOL_NAME = "read_file" TOOL_ID = "lifecycle-tool-1" TEST_BITE = os.environ.get("LIFECYCLE_TEST_BITE", "").strip() @@ -39,6 +42,52 @@ ANCHOR_SCENE_PERSIST_TIMEOUT = 60.0 ANCHOR_SCENE_PROJECTION_TIMEOUT = 10_000 +def _latest_anchor_scene_from_disk(state_root: Path, session_id: str) -> dict | None: + session_file_candidates = [ + state_root / "webui-state" / "sessions" / f"{session_id}.json", + state_root / "sessions" / f"{session_id}.json", + ] + for session_file in session_file_candidates: + if not session_file.exists(): + continue + try: + raw = json.loads(session_file.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + records = raw.get("anchor_activity_scenes") + if not isinstance(records, dict): + continue + candidates = [] + for record in records.values(): + if not isinstance(record, dict): + continue + scene = record.get("scene") + if not isinstance(scene, dict): + continue + idx = record.get("message_index") + try: + message_index = int(idx) + except (TypeError, ValueError): + message_index = -1 + candidates.append((message_index, scene)) + if not candidates: + continue + _, latest = max(candidates, key=lambda item: item[0]) + return latest + return None + + +def _safe_request_post_data(request_or_route_request) -> str: + raw = getattr(request_or_route_request, "post_data", None) + if callable(raw): + raw = raw() + if raw is None: + return "" + if isinstance(raw, bytes): + return raw.decode("utf-8") + return str(raw) + + def _free_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(("127.0.0.1", 0)) @@ -238,7 +287,8 @@ def _start_webui_server(repo_root: Path, env: dict, artifact_dir: Path): class DeterministicGateway: """A localhost-only Gateway Runs server with test-controlled phase gates.""" - def __init__(self) -> None: + def __init__(self, scenario: str) -> None: + self.scenario = scenario self.activity_ready = threading.Event() self.release_settle = threading.Event() self.final_prefix_ready = threading.Event() @@ -302,6 +352,11 @@ class DeterministicGateway: "event": "reasoning.available", "text": REASONING_TEXT, }) + if owner.scenario == "terminal-error": + self._event("message.delta", { + "event": "message.delta", + "delta": TERMINAL_PROCESS_TEXT, + }) self._event("tool.started", { "event": "tool.started", "tool": TOOL_NAME, @@ -319,21 +374,30 @@ class DeterministicGateway: owner.activity_ready.set() if not owner.release_settle.wait(timeout=30): return - self._event("message.delta", { - "event": "message.delta", - "delta": FINAL_PREFIX, - }) - owner.final_prefix_ready.set() - if not owner.release_terminal.wait(timeout=30): - return - self._event("message.delta", { - "event": "message.delta", - "delta": FINAL_SUFFIX, - }) - self._event("run.completed", { - "event": "run.completed", - "usage": {"input_tokens": 12, "output_tokens": 5}, - }) + if owner.scenario == "terminal-error": + owner.final_prefix_ready.set() + if not owner.release_terminal.wait(timeout=30): + return + self._event("run.failed", { + "event": "run.failed", + "error": TERMINAL_ERROR_TEXT, + }) + else: + self._event("message.delta", { + "event": "message.delta", + "delta": FINAL_PREFIX, + }) + owner.final_prefix_ready.set() + if not owner.release_terminal.wait(timeout=30): + return + self._event("message.delta", { + "event": "message.delta", + "delta": FINAL_SUFFIX, + }) + self._event("run.completed", { + "event": "run.completed", + "usage": {"input_tokens": 12, "output_tokens": 5}, + }) self.wfile.write(b"data: [DONE]\n\n") self.wfile.flush() except (BrokenPipeError, ConnectionResetError): @@ -417,6 +481,9 @@ def _capture_anchor_scene_requests(page): def _activity_snapshot(page) -> dict: return page.evaluate( """() => { + const messages = (typeof S !== 'undefined' && Array.isArray(S.messages)) ? S.messages : []; + const assistants = messages.filter((message) => message && message.role === 'assistant'); + const lastAssistant = assistants.length ? assistants[assistants.length - 1] : null; const turn = document.querySelector('#liveAssistantTurn') || Array.from(document.querySelectorAll('.assistant-turn')).pop() || null; const groups = turn ? Array.from(turn.querySelectorAll('[data-anchor-scene-owner="1"]')) : []; @@ -424,7 +491,9 @@ def _activity_snapshot(page) -> dict: const visibleFinal = turn ? Array.from(turn.querySelectorAll('.assistant-segment .msg-body')) .filter(el => { const segment = el.closest('.assistant-segment'); + const role = segment && segment.getAttribute('data-anchor-row-role'); return segment && !segment.hidden && + role !== 'prose' && !segment.classList.contains('assistant-segment-worklog-source') && getComputedStyle(segment).display !== 'none'; }) @@ -450,11 +519,20 @@ def _activity_snapshot(page) -> dict: rows: rows.map(row => ({ role: row.getAttribute('data-anchor-row-role'), source: row.getAttribute('data-anchor-source-event-type'), + status: row.getAttribute('data-anchor-row-status'), tool: row.getAttribute('data-tool-name'), text: row.innerText.trim(), classes: row.className, })), visibleFinal, + assistantMessage: lastAssistant ? { + turnDuration: lastAssistant._turnDuration, + hasError: Boolean(lastAssistant._error), + anchorTerminalState: lastAssistant._anchor_activity_scene + && (lastAssistant._anchor_activity_scene.terminal_state + || (lastAssistant._anchor_activity_scene.lifecycle + && lastAssistant._anchor_activity_scene.lifecycle.terminal_state)) || null, + } : null, transcript: (document.querySelector('#msgInner') || {}).innerText || '', }; }""" @@ -485,31 +563,121 @@ def _expand_settled_worklog(page) -> None: ) +def _terminal_rows(snapshot: dict) -> list[dict]: + return [row for row in snapshot["rows"] if row["role"] == "terminal"] + + +def _process_rows(snapshot: dict) -> list[dict]: + return [ + row for row in snapshot["rows"] + if row["role"] == "prose" and _is_terminal_process_row_text(row.get("text") or "") + ] + + +def _is_terminal_process_row_text(text: str) -> bool: + text = " ".join(text.split()) + if not text: + return False + return ( + TERMINAL_PROCESS_TEXT.startswith(text) + or text.startswith(TERMINAL_PROCESS_TEXT) + ) + + +def _tool_rows(snapshot: dict) -> list[dict]: + return [row for row in snapshot["rows"] if row["role"] == "tool"] + + +def _is_terminal_row_error(row: dict) -> bool: + status = (row.get("status") or "").strip() + if status in {"error", "failed"}: + return True + if status and status not in {"", "done"}: + return False + row_text = (row.get("text") or "").lower() + row_classes = (row.get("classes") or "") + return ( + "error" in row_text + or "warning" in row_classes + or "agent-activity-status-error" in row_classes + ) + + +def _assert_no_running_tool_rows(rows: list[dict]) -> None: + running = [ + row + for row in rows + if ( + row.get("status") == "running" + or ("tool-card-running" in row.get("classes", "")) + ) + ] + assert not running, rows + + def _assert_live_activity(snapshot: dict) -> None: assert snapshot["live"], snapshot assert snapshot["groupCount"] == 1, snapshot roles = [row["role"] for row in snapshot["rows"]] assert roles.count("thinking") == 1, snapshot assert roles.count("tool") == 1, snapshot - tool_rows = [row for row in snapshot["rows"] if row["role"] == "tool"] + tool_rows = _tool_rows(snapshot) assert len(tool_rows) == 1 and tool_rows[0]["tool"] == TOOL_NAME, snapshot - assert "tool-card-running" not in tool_rows[0]["classes"], snapshot + _assert_no_running_tool_rows(tool_rows) assert any(REASONING_TEXT in row["text"] for row in snapshot["rows"]), snapshot assert all(FINAL_TEXT not in text for text in snapshot["visibleFinal"]), snapshot assert any(character.isdigit() for character in snapshot["summary"][0]["label"]), snapshot -def _assert_settled(snapshot: dict) -> None: +def _assert_settled(snapshot: dict, scenario: str) -> None: assert not snapshot["live"], snapshot assert snapshot["groupCount"] == 1, snapshot roles = [row["role"] for row in snapshot["rows"]] assert "thinking" in roles and "tool" in roles, snapshot - tool_rows = [row for row in snapshot["rows"] if row["role"] == "tool"] + tool_rows = _tool_rows(snapshot) assert len(tool_rows) == 1 and tool_rows[0]["tool"] == TOOL_NAME, snapshot - assert "tool-card-running" not in tool_rows[0]["classes"], snapshot + _assert_no_running_tool_rows(tool_rows) assert any(REASONING_TEXT in row["text"] for row in snapshot["rows"]), snapshot - assert sum(FINAL_TEXT in text for text in snapshot["visibleFinal"]) == 1, snapshot - assert snapshot["transcript"].count(FINAL_TEXT) == 1, snapshot + if scenario == "terminal-error": + assert "terminal" in roles, snapshot + terminal_rows = _terminal_rows(snapshot) + assert terminal_rows, snapshot + assert all(_is_terminal_row_error(row) for row in terminal_rows), snapshot + assert snapshot["assistantMessage"] is not None, snapshot + turn_duration = snapshot["assistantMessage"].get("turnDuration") + assert isinstance(turn_duration, (int, float)) and turn_duration > 0, snapshot + assert snapshot["assistantMessage"]["hasError"] is True, snapshot + assert snapshot["assistantMessage"]["anchorTerminalState"] in {"error", "failed"}, snapshot + assert all(FINAL_TEXT not in text for text in snapshot["visibleFinal"]), snapshot + assert sum(TERMINAL_ERROR_TEXT in text for text in snapshot["visibleFinal"]) == 1, snapshot + assert TERMINAL_ERROR_TEXT in snapshot["transcript"], snapshot + else: + assert sum(FINAL_TEXT in text for text in snapshot["visibleFinal"]) == 1, snapshot + assert snapshot["transcript"].count(FINAL_TEXT) == 1, snapshot + + +def _parse_json_payload(raw: str | bytes | None) -> dict: + if raw is None: + return {} + if isinstance(raw, bytes): + raw = raw.decode("utf-8") + if not raw: + return {} + try: + payload = json.loads(raw) + except (TypeError, json.JSONDecodeError): + return {} + if not isinstance(payload, dict): + return {} + return payload + + +def _assert_process_row_present(snapshot: dict) -> list[dict]: + rows = _process_rows(snapshot) + assert len(rows) == 1, snapshot + assert all(_is_terminal_process_row_text(row["text"]) for row in rows), snapshot + assert all(TERMINAL_PROCESS_TEXT not in text for text in snapshot["visibleFinal"]), snapshot + return rows def _semantic_activity(snapshot: dict) -> list[dict]: @@ -542,7 +710,24 @@ def main() -> int: tempfile.mkdtemp(prefix="hermes-lifecycle-artifacts-") ) artifact_dir.mkdir(parents=True, exist_ok=True) - gateway = DeterministicGateway() + scenario = SCENARIO + if scenario not in {"normal", "terminal-error"}: + raise ValueError( + f"Unsupported LIFECYCLE_SCENARIO {scenario!r}; " + "expected 'normal' or 'terminal-error'" + ) + if TEST_BITE not in {"", "drop-anchor-persistence", "drop-terminal-anchor-row"}: + raise ValueError( + f"Unsupported LIFECYCLE_TEST_BITE {TEST_BITE!r}; " + "expected one of '', 'drop-anchor-persistence', 'drop-terminal-anchor-row'" + ) + if TEST_BITE == "drop-terminal-anchor-row" and scenario != "terminal-error": + raise ValueError( + "drop-terminal-anchor-row is only valid for " + "LIFECYCLE_SCENARIO=terminal-error" + ) + + gateway = DeterministicGateway(scenario) gateway.start() agent_dir = state_dir / "no-agent" @@ -598,15 +783,43 @@ def main() -> int: context = browser.new_context(base_url=base_url) page = context.new_page() anchor_scene_requests = _capture_anchor_scene_requests(page) - if TEST_BITE == "drop-anchor-persistence": - page.route( - "**/api/session/anchor-scene", - lambda route: route.fulfill( - status=200, - content_type="application/json", - body='{"ok":true}', - ), - ) + if TEST_BITE: + def _route_anchor_scene(route): + if TEST_BITE == "drop-anchor-persistence": + route.fulfill( + status=200, + content_type="application/json", + body='{"ok":true}', + ) + return + if TEST_BITE == "drop-terminal-anchor-row": + raw_payload = _safe_request_post_data(route.request) + payload = _parse_json_payload(raw_payload) + if isinstance(payload, dict): + scene = payload.get("scene") + if isinstance(scene, dict): + rows = scene.get("activity_rows") + if isinstance(rows, list): + kept = [ + row for row in rows + if not (isinstance(row, dict) and row.get("role") == "terminal") + ] + if len(kept) != len(rows): + mutated = dict(payload) + updated_scene = dict(scene) + updated_scene["activity_rows"] = kept + mutated["scene"] = updated_scene + response = route.fetch(post_data=json.dumps(mutated)) + route.fulfill(response=response) + return + response = route.fetch() + route.fulfill(response=response) + return + response = route.fetch() + route.fulfill(response=response) + return + + page.route("**/api/session/anchor-scene", _route_anchor_scene) errors = _capture_page_errors(page) page.goto("/", wait_until="domcontentloaded") page.wait_for_selector("#msg", state="visible", timeout=15000) @@ -631,54 +844,153 @@ def main() -> int: ) live_snapshot = _activity_snapshot(page) _assert_live_activity(live_snapshot) - print("OK live activity: one Anchor worklog with reasoning + completed tool") + if scenario == "terminal-error": + _assert_process_row_present(live_snapshot) + print("OK live activity: terminal-error run keeps reasoning + completed tool") + else: + print("OK live activity: one Anchor worklog with reasoning + completed tool") _wait_for_live_anchor_projection(page) gateway.release_settle.set() if not gateway.final_prefix_ready.wait(timeout=10): raise AssertionError("mock Gateway did not emit the final-answer prefix") - page.wait_for_function( - """text => { - const turn = document.querySelector('#liveAssistantTurn'); - return turn && (turn.innerText || '').includes(text); - }""", - arg=FINAL_ACK_TEXT, - timeout=10000, - ) - gateway.release_terminal.set() - page.wait_for_function( - """text => typeof S !== 'undefined' && S.busy === false && !S.activeStreamId && - !document.querySelector('#liveAssistantTurn') && - (document.querySelector('#msgInner') || {}).innerText?.includes(text)""", - arg=FINAL_TEXT, - timeout=15000, - ) + if scenario == "normal": + page.wait_for_function( + """text => { + const turn = document.querySelector('#liveAssistantTurn'); + return Boolean(turn) && turn.innerText.includes(text); + }""", + arg=FINAL_ACK_TEXT, + timeout=10000, + ) + gateway.release_terminal.set() + page.wait_for_function( + """text => typeof S !== 'undefined' && S.busy === false && !S.activeStreamId && + ((document.querySelector('#msgInner') || {}).innerText || '').includes(text)""", + arg=FINAL_TEXT, + timeout=15000, + ) + else: + gateway.release_terminal.set() + page.wait_for_function( + """text => typeof S !== 'undefined' && S.busy === false && !S.activeStreamId && + !document.querySelector('#liveAssistantTurn') && + ((document.querySelector('#msgInner') || {}).innerText||'').includes(text)""", + arg=TERMINAL_ERROR_TEXT, + timeout=15000, + ) session_id = page.evaluate("S.session && S.session.session_id") assert session_id, "active session id missing after settlement" - if not TEST_BITE: + if TEST_BITE == "drop-terminal-anchor-row": scene = _wait_for_persisted_scene( base_url, session_id, anchor_scene_requests=anchor_scene_requests, ) assert scene.get("version") == "activity_scene_v1", scene + scene_rows = scene.get("activity_rows") or [] + scene_roles = [ + row.get("role") for row in scene_rows + if isinstance(row, dict) + ] + assert any( + isinstance(row, dict) and row.get("role") == "prose" + for row in scene_rows + ), scene + assert any( + isinstance(row, dict) and row.get("role") == "thinking" + for row in scene_rows + ), scene + assert any( + isinstance(row, dict) and row.get("role") == "tool" + for row in scene_rows + ), scene + assert all( + not ( + isinstance(row, dict) + and row.get("role") == "terminal" + ) + for row in scene_rows + ), scene + print( + "OK persisted scene via API: roles=%s terminal_present=%s" + % (sorted(set(scene_roles)), any(role == "terminal" for role in scene_roles)) + ) + persisted_scene = _latest_anchor_scene_from_disk(state_dir, session_id) + assert persisted_scene is not None, { + "session_id": session_id, + "state_dir": str(state_dir / "webui-state"), + } + persisted_rows = persisted_scene.get("activity_rows") or [] + persisted_roles = [ + row.get("role") for row in persisted_rows + if isinstance(row, dict) + ] + assert any( + isinstance(row, dict) and row.get("role") == "thinking" + for row in persisted_rows + ), { + "persisted_rows": persisted_rows, + } + assert any( + isinstance(row, dict) and row.get("role") == "prose" + for row in persisted_rows + ), { + "persisted_rows": persisted_rows, + } + assert any( + isinstance(row, dict) and row.get("role") == "tool" + for row in persisted_rows + ), { + "persisted_rows": persisted_rows, + } + assert all( + not ( + isinstance(row, dict) + and row.get("role") == "terminal" + ) + for row in persisted_rows + ), { + "persisted_rows": persisted_rows, + } + print( + "OK persisted scene on disk: roles=%s terminal_present=%s" + % (sorted(set(persisted_roles)), "terminal" in persisted_roles) + ) + elif not TEST_BITE: + scene = _wait_for_persisted_scene( + base_url, + session_id, + anchor_scene_requests=anchor_scene_requests, + ) + assert scene.get("version") == "activity_scene_v1", scene + if scenario == "terminal-error": + scene_rows = scene.get("activity_rows") or [] + assert any( + isinstance(row, dict) and row.get("role") == "terminal" for row in scene_rows + ), scene _expand_settled_worklog(page) page.wait_for_selector( '.assistant-turn [data-anchor-settled-scene-owner="1"] [data-anchor-scene-row="1"]', timeout=10000, ) settled_snapshot = _activity_snapshot(page) - _assert_settled(settled_snapshot) + _assert_settled(settled_snapshot, scenario) + if scenario == "terminal-error": + _assert_process_row_present(settled_snapshot) assert _semantic_activity(settled_snapshot) == _semantic_activity(live_snapshot), { "live": _semantic_activity(live_snapshot), "settled": _semantic_activity(settled_snapshot), } - print("OK settled: final prose and the same semantic activity coexist without duplication") + if scenario == "terminal-error": + print("OK settled: terminal row and same activity survived terminal settlement") + else: + print("OK settled: final prose and the same semantic activity coexist without duplication") page.reload(wait_until="domcontentloaded") page.wait_for_function( "text => (document.querySelector('#msgInner') || {}).innerText?.includes(text)", - arg=FINAL_TEXT, + arg=TERMINAL_ERROR_TEXT if scenario == "terminal-error" else FINAL_TEXT, timeout=15000, ) _expand_settled_worklog(page) @@ -687,11 +999,34 @@ def main() -> int: timeout=2000 if TEST_BITE else 10000, ) reloaded_snapshot = _activity_snapshot(page) - _assert_settled(reloaded_snapshot) + _assert_settled(reloaded_snapshot, scenario) + if scenario == "terminal-error": + _assert_process_row_present(reloaded_snapshot) assert _semantic_activity(reloaded_snapshot) == _semantic_activity(settled_snapshot), { "settled": _semantic_activity(settled_snapshot), "reloaded": _semantic_activity(reloaded_snapshot), } + if scenario == "terminal-error": + settled_terminal = _terminal_rows(settled_snapshot) + reloaded_terminal = _terminal_rows(reloaded_snapshot) + settled_process = _process_rows(settled_snapshot) + reloaded_process = _process_rows(reloaded_snapshot) + assert len(settled_process) == len(reloaded_process) == 1, { + "settled_process": settled_process, + "reloaded_process": reloaded_process, + } + assert settled_process[0]["text"] == reloaded_process[0]["text"], { + "settled_process": settled_process, + "reloaded_process": reloaded_process, + } + assert len(settled_terminal) == len(reloaded_terminal) == 1, { + "settled_terminal": settled_terminal, + "reloaded_terminal": reloaded_terminal, + } + assert settled_terminal[0]["text"] == reloaded_terminal[0]["text"], { + "settled_terminal": settled_terminal[0], + "reloaded_terminal": reloaded_terminal[0], + } print("OK hard reload: transcript-backed Anchor scene preserves settled parity") assert gateway.request_body and gateway.request_body.get("input") == PROMPT, gateway.request_body @@ -710,7 +1045,7 @@ def main() -> int: page.screenshot(path=str(artifact_dir / "failure.png"), full_page=True) (artifact_dir / "snapshot.json").write_text( json.dumps({ - "scenario": "normal-live-to-final", + "scenario": scenario, "test_bite": TEST_BITE or None, "browser_errors": errors, "anchor_scene_requests": anchor_scene_requests,