Merge remote-tracking branch 'origin/master' into paperclip/pr-6222
This commit is contained in:
commit
2e9c2b1dc8
3 changed files with 421 additions and 68 deletions
7
.github/workflows/conversation-lifecycle.yml
vendored
7
.github/workflows/conversation-lifecycle.yml
vendored
|
|
@ -31,6 +31,10 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
scenario: [normal, terminal-error]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
|
|
@ -53,12 +57,13 @@ jobs:
|
|||
- name: Run conversation lifecycle gate
|
||||
env:
|
||||
LIFECYCLE_ARTIFACT_DIR: lifecycle-artifacts
|
||||
LIFECYCLE_SCENARIO: ${{ matrix.scenario }}
|
||||
run: python tests/browser_conversation_lifecycle.py
|
||||
|
||||
- name: Upload failure artifacts
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: conversation-lifecycle-failure
|
||||
name: conversation-lifecycle-failure-${{ matrix.scenario }}
|
||||
path: lifecycle-artifacts
|
||||
if-no-files-found: ignore
|
||||
|
|
|
|||
31
TESTING.md
31
TESTING.md
|
|
@ -93,18 +93,24 @@ that breaks the page for everyone).
|
|||
|
||||
## Public conversation lifecycle gate
|
||||
|
||||
`tests/browser_conversation_lifecycle.py` adds the first public, deterministic
|
||||
chat golden path. It drives the real composer and real WebUI server in Chromium,
|
||||
while a localhost-only fixture supplies reasoning, tool, and final-answer events
|
||||
through the existing Hermes Gateway Runs API. The gate asserts semantic activity
|
||||
during live streaming, after settlement, and after a hard reload, including
|
||||
`tests/browser_conversation_lifecycle.py` adds a public deterministic
|
||||
multi-row lifecycle gate. It drives the real composer and real WebUI server in Chromium,
|
||||
while a localhost-only fixture supplies reasoning, tool, process, and final/error
|
||||
events through the existing Hermes Gateway Runs API. The gate now covers both
|
||||
normal and terminal-error proof-matrix rows, asserting semantic activity during
|
||||
live streaming, after settlement, and after hard reload, including
|
||||
transcript-backed `activity_scene_v1` persistence and zero unexpected browser
|
||||
errors. It uses isolated temporary state and no provider credentials.
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt playwright
|
||||
python -m playwright install --with-deps chromium
|
||||
|
||||
# Normal-path deterministic conversation lifecycle gate.
|
||||
python tests/browser_conversation_lifecycle.py
|
||||
|
||||
# Terminal-error lifecycle gate (new row in the proof matrix).
|
||||
LIFECYCLE_SCENARIO=terminal-error python tests/browser_conversation_lifecycle.py
|
||||
```
|
||||
|
||||
To certify that the gate catches its target failure, the test owns an opt-in
|
||||
|
|
@ -114,12 +120,19 @@ must fail at the hard-reload boundary:
|
|||
```bash
|
||||
LIFECYCLE_TEST_BITE=drop-anchor-persistence \
|
||||
python tests/browser_conversation_lifecycle.py
|
||||
|
||||
# Terminal-state-specific mutation bite: remove terminal row from persisted scene
|
||||
# so hard reload cannot recover terminal status.
|
||||
LIFECYCLE_SCENARIO=terminal-error \
|
||||
LIFECYCLE_TEST_BITE=drop-terminal-anchor-row \
|
||||
python tests/browser_conversation_lifecycle.py
|
||||
```
|
||||
|
||||
The dedicated `Conversation lifecycle (informational)` workflow runs this test
|
||||
without blocking merges while the public matrix is being established. The
|
||||
maintainer's private QA harness remains broader; later public slices will add
|
||||
session switching, reconnect/replay, cancellation, compression, and recovery.
|
||||
The dedicated `Conversation lifecycle (informational)` workflow runs both current
|
||||
proof rows (`normal` and `terminal-error`) and stays non-blocking while the public
|
||||
matrix expands to additional behavior rows. The maintainer's private QA harness
|
||||
remains broader; later public slices will add session switching, reconnect/replay,
|
||||
cancellation, compression, and recovery.
|
||||
|
||||
|
||||
`tests/test_static_js_runtime_lint.py` runs this automatically when eslint is present
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Public browser gate for the normal conversation lifecycle.
|
||||
"""Public browser gate for the normal + terminal-error lifecycle.
|
||||
|
||||
This test boots the real WebUI server with isolated state, drives the real chat
|
||||
composer in Chromium, and supplies deterministic runtime events through the
|
||||
|
|
@ -28,9 +28,12 @@ from urllib.parse import urlsplit
|
|||
PROMPT = "Exercise the public conversation lifecycle gate."
|
||||
REASONING_TEXT = "Checking the persistent assistant turn."
|
||||
FINAL_TEXT = "Lifecycle gate final answer."
|
||||
FINAL_ACK_TEXT = "Lifecycle"
|
||||
FINAL_PREFIX = "Lifecycle gate "
|
||||
FINAL_SUFFIX = "final answer."
|
||||
FINAL_ACK_TEXT = "Lifecycle"
|
||||
TERMINAL_PROCESS_TEXT = "Lifecycle terminal process check"
|
||||
TERMINAL_ERROR_TEXT = "Lifecycle gate encountered a terminal-side error."
|
||||
SCENARIO = os.environ.get("LIFECYCLE_SCENARIO", "normal").strip() or "normal"
|
||||
TOOL_NAME = "read_file"
|
||||
TOOL_ID = "lifecycle-tool-1"
|
||||
TEST_BITE = os.environ.get("LIFECYCLE_TEST_BITE", "").strip()
|
||||
|
|
@ -39,6 +42,52 @@ ANCHOR_SCENE_PERSIST_TIMEOUT = 60.0
|
|||
ANCHOR_SCENE_PROJECTION_TIMEOUT = 10_000
|
||||
|
||||
|
||||
def _latest_anchor_scene_from_disk(state_root: Path, session_id: str) -> dict | None:
|
||||
session_file_candidates = [
|
||||
state_root / "webui-state" / "sessions" / f"{session_id}.json",
|
||||
state_root / "sessions" / f"{session_id}.json",
|
||||
]
|
||||
for session_file in session_file_candidates:
|
||||
if not session_file.exists():
|
||||
continue
|
||||
try:
|
||||
raw = json.loads(session_file.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
records = raw.get("anchor_activity_scenes")
|
||||
if not isinstance(records, dict):
|
||||
continue
|
||||
candidates = []
|
||||
for record in records.values():
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
scene = record.get("scene")
|
||||
if not isinstance(scene, dict):
|
||||
continue
|
||||
idx = record.get("message_index")
|
||||
try:
|
||||
message_index = int(idx)
|
||||
except (TypeError, ValueError):
|
||||
message_index = -1
|
||||
candidates.append((message_index, scene))
|
||||
if not candidates:
|
||||
continue
|
||||
_, latest = max(candidates, key=lambda item: item[0])
|
||||
return latest
|
||||
return None
|
||||
|
||||
|
||||
def _safe_request_post_data(request_or_route_request) -> str:
|
||||
raw = getattr(request_or_route_request, "post_data", None)
|
||||
if callable(raw):
|
||||
raw = raw()
|
||||
if raw is None:
|
||||
return ""
|
||||
if isinstance(raw, bytes):
|
||||
return raw.decode("utf-8")
|
||||
return str(raw)
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
|
|
@ -238,7 +287,8 @@ def _start_webui_server(repo_root: Path, env: dict, artifact_dir: Path):
|
|||
class DeterministicGateway:
|
||||
"""A localhost-only Gateway Runs server with test-controlled phase gates."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, scenario: str) -> None:
|
||||
self.scenario = scenario
|
||||
self.activity_ready = threading.Event()
|
||||
self.release_settle = threading.Event()
|
||||
self.final_prefix_ready = threading.Event()
|
||||
|
|
@ -302,6 +352,11 @@ class DeterministicGateway:
|
|||
"event": "reasoning.available",
|
||||
"text": REASONING_TEXT,
|
||||
})
|
||||
if owner.scenario == "terminal-error":
|
||||
self._event("message.delta", {
|
||||
"event": "message.delta",
|
||||
"delta": TERMINAL_PROCESS_TEXT,
|
||||
})
|
||||
self._event("tool.started", {
|
||||
"event": "tool.started",
|
||||
"tool": TOOL_NAME,
|
||||
|
|
@ -319,21 +374,30 @@ class DeterministicGateway:
|
|||
owner.activity_ready.set()
|
||||
if not owner.release_settle.wait(timeout=30):
|
||||
return
|
||||
self._event("message.delta", {
|
||||
"event": "message.delta",
|
||||
"delta": FINAL_PREFIX,
|
||||
})
|
||||
owner.final_prefix_ready.set()
|
||||
if not owner.release_terminal.wait(timeout=30):
|
||||
return
|
||||
self._event("message.delta", {
|
||||
"event": "message.delta",
|
||||
"delta": FINAL_SUFFIX,
|
||||
})
|
||||
self._event("run.completed", {
|
||||
"event": "run.completed",
|
||||
"usage": {"input_tokens": 12, "output_tokens": 5},
|
||||
})
|
||||
if owner.scenario == "terminal-error":
|
||||
owner.final_prefix_ready.set()
|
||||
if not owner.release_terminal.wait(timeout=30):
|
||||
return
|
||||
self._event("run.failed", {
|
||||
"event": "run.failed",
|
||||
"error": TERMINAL_ERROR_TEXT,
|
||||
})
|
||||
else:
|
||||
self._event("message.delta", {
|
||||
"event": "message.delta",
|
||||
"delta": FINAL_PREFIX,
|
||||
})
|
||||
owner.final_prefix_ready.set()
|
||||
if not owner.release_terminal.wait(timeout=30):
|
||||
return
|
||||
self._event("message.delta", {
|
||||
"event": "message.delta",
|
||||
"delta": FINAL_SUFFIX,
|
||||
})
|
||||
self._event("run.completed", {
|
||||
"event": "run.completed",
|
||||
"usage": {"input_tokens": 12, "output_tokens": 5},
|
||||
})
|
||||
self.wfile.write(b"data: [DONE]\n\n")
|
||||
self.wfile.flush()
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
|
|
@ -417,6 +481,9 @@ def _capture_anchor_scene_requests(page):
|
|||
def _activity_snapshot(page) -> dict:
|
||||
return page.evaluate(
|
||||
"""() => {
|
||||
const messages = (typeof S !== 'undefined' && Array.isArray(S.messages)) ? S.messages : [];
|
||||
const assistants = messages.filter((message) => message && message.role === 'assistant');
|
||||
const lastAssistant = assistants.length ? assistants[assistants.length - 1] : null;
|
||||
const turn = document.querySelector('#liveAssistantTurn') ||
|
||||
Array.from(document.querySelectorAll('.assistant-turn')).pop() || null;
|
||||
const groups = turn ? Array.from(turn.querySelectorAll('[data-anchor-scene-owner="1"]')) : [];
|
||||
|
|
@ -424,7 +491,9 @@ def _activity_snapshot(page) -> dict:
|
|||
const visibleFinal = turn ? Array.from(turn.querySelectorAll('.assistant-segment .msg-body'))
|
||||
.filter(el => {
|
||||
const segment = el.closest('.assistant-segment');
|
||||
const role = segment && segment.getAttribute('data-anchor-row-role');
|
||||
return segment && !segment.hidden &&
|
||||
role !== 'prose' &&
|
||||
!segment.classList.contains('assistant-segment-worklog-source') &&
|
||||
getComputedStyle(segment).display !== 'none';
|
||||
})
|
||||
|
|
@ -450,11 +519,20 @@ def _activity_snapshot(page) -> dict:
|
|||
rows: rows.map(row => ({
|
||||
role: row.getAttribute('data-anchor-row-role'),
|
||||
source: row.getAttribute('data-anchor-source-event-type'),
|
||||
status: row.getAttribute('data-anchor-row-status'),
|
||||
tool: row.getAttribute('data-tool-name'),
|
||||
text: row.innerText.trim(),
|
||||
classes: row.className,
|
||||
})),
|
||||
visibleFinal,
|
||||
assistantMessage: lastAssistant ? {
|
||||
turnDuration: lastAssistant._turnDuration,
|
||||
hasError: Boolean(lastAssistant._error),
|
||||
anchorTerminalState: lastAssistant._anchor_activity_scene
|
||||
&& (lastAssistant._anchor_activity_scene.terminal_state
|
||||
|| (lastAssistant._anchor_activity_scene.lifecycle
|
||||
&& lastAssistant._anchor_activity_scene.lifecycle.terminal_state)) || null,
|
||||
} : null,
|
||||
transcript: (document.querySelector('#msgInner') || {}).innerText || '',
|
||||
};
|
||||
}"""
|
||||
|
|
@ -485,31 +563,121 @@ def _expand_settled_worklog(page) -> None:
|
|||
)
|
||||
|
||||
|
||||
def _terminal_rows(snapshot: dict) -> list[dict]:
|
||||
return [row for row in snapshot["rows"] if row["role"] == "terminal"]
|
||||
|
||||
|
||||
def _process_rows(snapshot: dict) -> list[dict]:
|
||||
return [
|
||||
row for row in snapshot["rows"]
|
||||
if row["role"] == "prose" and _is_terminal_process_row_text(row.get("text") or "")
|
||||
]
|
||||
|
||||
|
||||
def _is_terminal_process_row_text(text: str) -> bool:
|
||||
text = " ".join(text.split())
|
||||
if not text:
|
||||
return False
|
||||
return (
|
||||
TERMINAL_PROCESS_TEXT.startswith(text)
|
||||
or text.startswith(TERMINAL_PROCESS_TEXT)
|
||||
)
|
||||
|
||||
|
||||
def _tool_rows(snapshot: dict) -> list[dict]:
|
||||
return [row for row in snapshot["rows"] if row["role"] == "tool"]
|
||||
|
||||
|
||||
def _is_terminal_row_error(row: dict) -> bool:
|
||||
status = (row.get("status") or "").strip()
|
||||
if status in {"error", "failed"}:
|
||||
return True
|
||||
if status and status not in {"", "done"}:
|
||||
return False
|
||||
row_text = (row.get("text") or "").lower()
|
||||
row_classes = (row.get("classes") or "")
|
||||
return (
|
||||
"error" in row_text
|
||||
or "warning" in row_classes
|
||||
or "agent-activity-status-error" in row_classes
|
||||
)
|
||||
|
||||
|
||||
def _assert_no_running_tool_rows(rows: list[dict]) -> None:
|
||||
running = [
|
||||
row
|
||||
for row in rows
|
||||
if (
|
||||
row.get("status") == "running"
|
||||
or ("tool-card-running" in row.get("classes", ""))
|
||||
)
|
||||
]
|
||||
assert not running, rows
|
||||
|
||||
|
||||
def _assert_live_activity(snapshot: dict) -> None:
|
||||
assert snapshot["live"], snapshot
|
||||
assert snapshot["groupCount"] == 1, snapshot
|
||||
roles = [row["role"] for row in snapshot["rows"]]
|
||||
assert roles.count("thinking") == 1, snapshot
|
||||
assert roles.count("tool") == 1, snapshot
|
||||
tool_rows = [row for row in snapshot["rows"] if row["role"] == "tool"]
|
||||
tool_rows = _tool_rows(snapshot)
|
||||
assert len(tool_rows) == 1 and tool_rows[0]["tool"] == TOOL_NAME, snapshot
|
||||
assert "tool-card-running" not in tool_rows[0]["classes"], snapshot
|
||||
_assert_no_running_tool_rows(tool_rows)
|
||||
assert any(REASONING_TEXT in row["text"] for row in snapshot["rows"]), snapshot
|
||||
assert all(FINAL_TEXT not in text for text in snapshot["visibleFinal"]), snapshot
|
||||
assert any(character.isdigit() for character in snapshot["summary"][0]["label"]), snapshot
|
||||
|
||||
|
||||
def _assert_settled(snapshot: dict) -> None:
|
||||
def _assert_settled(snapshot: dict, scenario: str) -> None:
|
||||
assert not snapshot["live"], snapshot
|
||||
assert snapshot["groupCount"] == 1, snapshot
|
||||
roles = [row["role"] for row in snapshot["rows"]]
|
||||
assert "thinking" in roles and "tool" in roles, snapshot
|
||||
tool_rows = [row for row in snapshot["rows"] if row["role"] == "tool"]
|
||||
tool_rows = _tool_rows(snapshot)
|
||||
assert len(tool_rows) == 1 and tool_rows[0]["tool"] == TOOL_NAME, snapshot
|
||||
assert "tool-card-running" not in tool_rows[0]["classes"], snapshot
|
||||
_assert_no_running_tool_rows(tool_rows)
|
||||
assert any(REASONING_TEXT in row["text"] for row in snapshot["rows"]), snapshot
|
||||
assert sum(FINAL_TEXT in text for text in snapshot["visibleFinal"]) == 1, snapshot
|
||||
assert snapshot["transcript"].count(FINAL_TEXT) == 1, snapshot
|
||||
if scenario == "terminal-error":
|
||||
assert "terminal" in roles, snapshot
|
||||
terminal_rows = _terminal_rows(snapshot)
|
||||
assert terminal_rows, snapshot
|
||||
assert all(_is_terminal_row_error(row) for row in terminal_rows), snapshot
|
||||
assert snapshot["assistantMessage"] is not None, snapshot
|
||||
turn_duration = snapshot["assistantMessage"].get("turnDuration")
|
||||
assert isinstance(turn_duration, (int, float)) and turn_duration > 0, snapshot
|
||||
assert snapshot["assistantMessage"]["hasError"] is True, snapshot
|
||||
assert snapshot["assistantMessage"]["anchorTerminalState"] in {"error", "failed"}, snapshot
|
||||
assert all(FINAL_TEXT not in text for text in snapshot["visibleFinal"]), snapshot
|
||||
assert sum(TERMINAL_ERROR_TEXT in text for text in snapshot["visibleFinal"]) == 1, snapshot
|
||||
assert TERMINAL_ERROR_TEXT in snapshot["transcript"], snapshot
|
||||
else:
|
||||
assert sum(FINAL_TEXT in text for text in snapshot["visibleFinal"]) == 1, snapshot
|
||||
assert snapshot["transcript"].count(FINAL_TEXT) == 1, snapshot
|
||||
|
||||
|
||||
def _parse_json_payload(raw: str | bytes | None) -> dict:
|
||||
if raw is None:
|
||||
return {}
|
||||
if isinstance(raw, bytes):
|
||||
raw = raw.decode("utf-8")
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return {}
|
||||
if not isinstance(payload, dict):
|
||||
return {}
|
||||
return payload
|
||||
|
||||
|
||||
def _assert_process_row_present(snapshot: dict) -> list[dict]:
|
||||
rows = _process_rows(snapshot)
|
||||
assert len(rows) == 1, snapshot
|
||||
assert all(_is_terminal_process_row_text(row["text"]) for row in rows), snapshot
|
||||
assert all(TERMINAL_PROCESS_TEXT not in text for text in snapshot["visibleFinal"]), snapshot
|
||||
return rows
|
||||
|
||||
|
||||
def _semantic_activity(snapshot: dict) -> list[dict]:
|
||||
|
|
@ -542,7 +710,24 @@ def main() -> int:
|
|||
tempfile.mkdtemp(prefix="hermes-lifecycle-artifacts-")
|
||||
)
|
||||
artifact_dir.mkdir(parents=True, exist_ok=True)
|
||||
gateway = DeterministicGateway()
|
||||
scenario = SCENARIO
|
||||
if scenario not in {"normal", "terminal-error"}:
|
||||
raise ValueError(
|
||||
f"Unsupported LIFECYCLE_SCENARIO {scenario!r}; "
|
||||
"expected 'normal' or 'terminal-error'"
|
||||
)
|
||||
if TEST_BITE not in {"", "drop-anchor-persistence", "drop-terminal-anchor-row"}:
|
||||
raise ValueError(
|
||||
f"Unsupported LIFECYCLE_TEST_BITE {TEST_BITE!r}; "
|
||||
"expected one of '', 'drop-anchor-persistence', 'drop-terminal-anchor-row'"
|
||||
)
|
||||
if TEST_BITE == "drop-terminal-anchor-row" and scenario != "terminal-error":
|
||||
raise ValueError(
|
||||
"drop-terminal-anchor-row is only valid for "
|
||||
"LIFECYCLE_SCENARIO=terminal-error"
|
||||
)
|
||||
|
||||
gateway = DeterministicGateway(scenario)
|
||||
gateway.start()
|
||||
|
||||
agent_dir = state_dir / "no-agent"
|
||||
|
|
@ -598,15 +783,43 @@ def main() -> int:
|
|||
context = browser.new_context(base_url=base_url)
|
||||
page = context.new_page()
|
||||
anchor_scene_requests = _capture_anchor_scene_requests(page)
|
||||
if TEST_BITE == "drop-anchor-persistence":
|
||||
page.route(
|
||||
"**/api/session/anchor-scene",
|
||||
lambda route: route.fulfill(
|
||||
status=200,
|
||||
content_type="application/json",
|
||||
body='{"ok":true}',
|
||||
),
|
||||
)
|
||||
if TEST_BITE:
|
||||
def _route_anchor_scene(route):
|
||||
if TEST_BITE == "drop-anchor-persistence":
|
||||
route.fulfill(
|
||||
status=200,
|
||||
content_type="application/json",
|
||||
body='{"ok":true}',
|
||||
)
|
||||
return
|
||||
if TEST_BITE == "drop-terminal-anchor-row":
|
||||
raw_payload = _safe_request_post_data(route.request)
|
||||
payload = _parse_json_payload(raw_payload)
|
||||
if isinstance(payload, dict):
|
||||
scene = payload.get("scene")
|
||||
if isinstance(scene, dict):
|
||||
rows = scene.get("activity_rows")
|
||||
if isinstance(rows, list):
|
||||
kept = [
|
||||
row for row in rows
|
||||
if not (isinstance(row, dict) and row.get("role") == "terminal")
|
||||
]
|
||||
if len(kept) != len(rows):
|
||||
mutated = dict(payload)
|
||||
updated_scene = dict(scene)
|
||||
updated_scene["activity_rows"] = kept
|
||||
mutated["scene"] = updated_scene
|
||||
response = route.fetch(post_data=json.dumps(mutated))
|
||||
route.fulfill(response=response)
|
||||
return
|
||||
response = route.fetch()
|
||||
route.fulfill(response=response)
|
||||
return
|
||||
response = route.fetch()
|
||||
route.fulfill(response=response)
|
||||
return
|
||||
|
||||
page.route("**/api/session/anchor-scene", _route_anchor_scene)
|
||||
errors = _capture_page_errors(page)
|
||||
page.goto("/", wait_until="domcontentloaded")
|
||||
page.wait_for_selector("#msg", state="visible", timeout=15000)
|
||||
|
|
@ -631,54 +844,153 @@ def main() -> int:
|
|||
)
|
||||
live_snapshot = _activity_snapshot(page)
|
||||
_assert_live_activity(live_snapshot)
|
||||
print("OK live activity: one Anchor worklog with reasoning + completed tool")
|
||||
if scenario == "terminal-error":
|
||||
_assert_process_row_present(live_snapshot)
|
||||
print("OK live activity: terminal-error run keeps reasoning + completed tool")
|
||||
else:
|
||||
print("OK live activity: one Anchor worklog with reasoning + completed tool")
|
||||
_wait_for_live_anchor_projection(page)
|
||||
|
||||
gateway.release_settle.set()
|
||||
if not gateway.final_prefix_ready.wait(timeout=10):
|
||||
raise AssertionError("mock Gateway did not emit the final-answer prefix")
|
||||
page.wait_for_function(
|
||||
"""text => {
|
||||
const turn = document.querySelector('#liveAssistantTurn');
|
||||
return turn && (turn.innerText || '').includes(text);
|
||||
}""",
|
||||
arg=FINAL_ACK_TEXT,
|
||||
timeout=10000,
|
||||
)
|
||||
gateway.release_terminal.set()
|
||||
page.wait_for_function(
|
||||
"""text => typeof S !== 'undefined' && S.busy === false && !S.activeStreamId &&
|
||||
!document.querySelector('#liveAssistantTurn') &&
|
||||
(document.querySelector('#msgInner') || {}).innerText?.includes(text)""",
|
||||
arg=FINAL_TEXT,
|
||||
timeout=15000,
|
||||
)
|
||||
if scenario == "normal":
|
||||
page.wait_for_function(
|
||||
"""text => {
|
||||
const turn = document.querySelector('#liveAssistantTurn');
|
||||
return Boolean(turn) && turn.innerText.includes(text);
|
||||
}""",
|
||||
arg=FINAL_ACK_TEXT,
|
||||
timeout=10000,
|
||||
)
|
||||
gateway.release_terminal.set()
|
||||
page.wait_for_function(
|
||||
"""text => typeof S !== 'undefined' && S.busy === false && !S.activeStreamId &&
|
||||
((document.querySelector('#msgInner') || {}).innerText || '').includes(text)""",
|
||||
arg=FINAL_TEXT,
|
||||
timeout=15000,
|
||||
)
|
||||
else:
|
||||
gateway.release_terminal.set()
|
||||
page.wait_for_function(
|
||||
"""text => typeof S !== 'undefined' && S.busy === false && !S.activeStreamId &&
|
||||
!document.querySelector('#liveAssistantTurn') &&
|
||||
((document.querySelector('#msgInner') || {}).innerText||'').includes(text)""",
|
||||
arg=TERMINAL_ERROR_TEXT,
|
||||
timeout=15000,
|
||||
)
|
||||
session_id = page.evaluate("S.session && S.session.session_id")
|
||||
assert session_id, "active session id missing after settlement"
|
||||
if not TEST_BITE:
|
||||
if TEST_BITE == "drop-terminal-anchor-row":
|
||||
scene = _wait_for_persisted_scene(
|
||||
base_url,
|
||||
session_id,
|
||||
anchor_scene_requests=anchor_scene_requests,
|
||||
)
|
||||
assert scene.get("version") == "activity_scene_v1", scene
|
||||
scene_rows = scene.get("activity_rows") or []
|
||||
scene_roles = [
|
||||
row.get("role") for row in scene_rows
|
||||
if isinstance(row, dict)
|
||||
]
|
||||
assert any(
|
||||
isinstance(row, dict) and row.get("role") == "prose"
|
||||
for row in scene_rows
|
||||
), scene
|
||||
assert any(
|
||||
isinstance(row, dict) and row.get("role") == "thinking"
|
||||
for row in scene_rows
|
||||
), scene
|
||||
assert any(
|
||||
isinstance(row, dict) and row.get("role") == "tool"
|
||||
for row in scene_rows
|
||||
), scene
|
||||
assert all(
|
||||
not (
|
||||
isinstance(row, dict)
|
||||
and row.get("role") == "terminal"
|
||||
)
|
||||
for row in scene_rows
|
||||
), scene
|
||||
print(
|
||||
"OK persisted scene via API: roles=%s terminal_present=%s"
|
||||
% (sorted(set(scene_roles)), any(role == "terminal" for role in scene_roles))
|
||||
)
|
||||
persisted_scene = _latest_anchor_scene_from_disk(state_dir, session_id)
|
||||
assert persisted_scene is not None, {
|
||||
"session_id": session_id,
|
||||
"state_dir": str(state_dir / "webui-state"),
|
||||
}
|
||||
persisted_rows = persisted_scene.get("activity_rows") or []
|
||||
persisted_roles = [
|
||||
row.get("role") for row in persisted_rows
|
||||
if isinstance(row, dict)
|
||||
]
|
||||
assert any(
|
||||
isinstance(row, dict) and row.get("role") == "thinking"
|
||||
for row in persisted_rows
|
||||
), {
|
||||
"persisted_rows": persisted_rows,
|
||||
}
|
||||
assert any(
|
||||
isinstance(row, dict) and row.get("role") == "prose"
|
||||
for row in persisted_rows
|
||||
), {
|
||||
"persisted_rows": persisted_rows,
|
||||
}
|
||||
assert any(
|
||||
isinstance(row, dict) and row.get("role") == "tool"
|
||||
for row in persisted_rows
|
||||
), {
|
||||
"persisted_rows": persisted_rows,
|
||||
}
|
||||
assert all(
|
||||
not (
|
||||
isinstance(row, dict)
|
||||
and row.get("role") == "terminal"
|
||||
)
|
||||
for row in persisted_rows
|
||||
), {
|
||||
"persisted_rows": persisted_rows,
|
||||
}
|
||||
print(
|
||||
"OK persisted scene on disk: roles=%s terminal_present=%s"
|
||||
% (sorted(set(persisted_roles)), "terminal" in persisted_roles)
|
||||
)
|
||||
elif not TEST_BITE:
|
||||
scene = _wait_for_persisted_scene(
|
||||
base_url,
|
||||
session_id,
|
||||
anchor_scene_requests=anchor_scene_requests,
|
||||
)
|
||||
assert scene.get("version") == "activity_scene_v1", scene
|
||||
if scenario == "terminal-error":
|
||||
scene_rows = scene.get("activity_rows") or []
|
||||
assert any(
|
||||
isinstance(row, dict) and row.get("role") == "terminal" for row in scene_rows
|
||||
), scene
|
||||
_expand_settled_worklog(page)
|
||||
page.wait_for_selector(
|
||||
'.assistant-turn [data-anchor-settled-scene-owner="1"] [data-anchor-scene-row="1"]',
|
||||
timeout=10000,
|
||||
)
|
||||
settled_snapshot = _activity_snapshot(page)
|
||||
_assert_settled(settled_snapshot)
|
||||
_assert_settled(settled_snapshot, scenario)
|
||||
if scenario == "terminal-error":
|
||||
_assert_process_row_present(settled_snapshot)
|
||||
assert _semantic_activity(settled_snapshot) == _semantic_activity(live_snapshot), {
|
||||
"live": _semantic_activity(live_snapshot),
|
||||
"settled": _semantic_activity(settled_snapshot),
|
||||
}
|
||||
print("OK settled: final prose and the same semantic activity coexist without duplication")
|
||||
if scenario == "terminal-error":
|
||||
print("OK settled: terminal row and same activity survived terminal settlement")
|
||||
else:
|
||||
print("OK settled: final prose and the same semantic activity coexist without duplication")
|
||||
|
||||
page.reload(wait_until="domcontentloaded")
|
||||
page.wait_for_function(
|
||||
"text => (document.querySelector('#msgInner') || {}).innerText?.includes(text)",
|
||||
arg=FINAL_TEXT,
|
||||
arg=TERMINAL_ERROR_TEXT if scenario == "terminal-error" else FINAL_TEXT,
|
||||
timeout=15000,
|
||||
)
|
||||
_expand_settled_worklog(page)
|
||||
|
|
@ -687,11 +999,34 @@ def main() -> int:
|
|||
timeout=2000 if TEST_BITE else 10000,
|
||||
)
|
||||
reloaded_snapshot = _activity_snapshot(page)
|
||||
_assert_settled(reloaded_snapshot)
|
||||
_assert_settled(reloaded_snapshot, scenario)
|
||||
if scenario == "terminal-error":
|
||||
_assert_process_row_present(reloaded_snapshot)
|
||||
assert _semantic_activity(reloaded_snapshot) == _semantic_activity(settled_snapshot), {
|
||||
"settled": _semantic_activity(settled_snapshot),
|
||||
"reloaded": _semantic_activity(reloaded_snapshot),
|
||||
}
|
||||
if scenario == "terminal-error":
|
||||
settled_terminal = _terminal_rows(settled_snapshot)
|
||||
reloaded_terminal = _terminal_rows(reloaded_snapshot)
|
||||
settled_process = _process_rows(settled_snapshot)
|
||||
reloaded_process = _process_rows(reloaded_snapshot)
|
||||
assert len(settled_process) == len(reloaded_process) == 1, {
|
||||
"settled_process": settled_process,
|
||||
"reloaded_process": reloaded_process,
|
||||
}
|
||||
assert settled_process[0]["text"] == reloaded_process[0]["text"], {
|
||||
"settled_process": settled_process,
|
||||
"reloaded_process": reloaded_process,
|
||||
}
|
||||
assert len(settled_terminal) == len(reloaded_terminal) == 1, {
|
||||
"settled_terminal": settled_terminal,
|
||||
"reloaded_terminal": reloaded_terminal,
|
||||
}
|
||||
assert settled_terminal[0]["text"] == reloaded_terminal[0]["text"], {
|
||||
"settled_terminal": settled_terminal[0],
|
||||
"reloaded_terminal": reloaded_terminal[0],
|
||||
}
|
||||
print("OK hard reload: transcript-backed Anchor scene preserves settled parity")
|
||||
|
||||
assert gateway.request_body and gateway.request_body.get("input") == PROMPT, gateway.request_body
|
||||
|
|
@ -710,7 +1045,7 @@ def main() -> int:
|
|||
page.screenshot(path=str(artifact_dir / "failure.png"), full_page=True)
|
||||
(artifact_dir / "snapshot.json").write_text(
|
||||
json.dumps({
|
||||
"scenario": "normal-live-to-final",
|
||||
"scenario": scenario,
|
||||
"test_bite": TEST_BITE or None,
|
||||
"browser_errors": errors,
|
||||
"anchor_scene_requests": anchor_scene_requests,
|
||||
|
|
|
|||
Loading…
Reference in a new issue