feat: stamp per-turn used model in footer stats (#6068)
Record resolved_model on assistant messages as _usedModel, expose used_model in the usage SSE payload, and render model chips in transparent and settled turn footers while suppressing duplicates when gateway routing already shows used_model.
This commit is contained in:
parent
1dd5cc43ce
commit
30aef85b18
5 changed files with 88 additions and 4 deletions
|
|
@ -7989,6 +7989,7 @@ _SESSION_MESSAGE_DISPLAY_METADATA_KEYS = (
|
||||||
"_turnTps",
|
"_turnTps",
|
||||||
"_turnUsage",
|
"_turnUsage",
|
||||||
"_firstTokenMs",
|
"_firstTokenMs",
|
||||||
|
"_usedModel",
|
||||||
"_gatewayRouting",
|
"_gatewayRouting",
|
||||||
"_statusCard",
|
"_statusCard",
|
||||||
"_anchor_stream_id",
|
"_anchor_stream_id",
|
||||||
|
|
|
||||||
|
|
@ -9936,6 +9936,9 @@ def _run_agent_streaming(
|
||||||
_ttft_ms = meter().get_ttft_ms(stream_id)
|
_ttft_ms = meter().get_ttft_ms(stream_id)
|
||||||
if _ttft_ms is not None:
|
if _ttft_ms is not None:
|
||||||
_dm['_firstTokenMs'] = _ttft_ms
|
_dm['_firstTokenMs'] = _ttft_ms
|
||||||
|
_used_model = resolved_model or model
|
||||||
|
if _used_model:
|
||||||
|
_dm['_usedModel'] = _used_model
|
||||||
break
|
break
|
||||||
# Persist context window data on the session so the context-ring
|
# Persist context window data on the session so the context-ring
|
||||||
# indicator survives a page reload (#1318). Must run BEFORE
|
# indicator survives a page reload (#1318). Must run BEFORE
|
||||||
|
|
@ -10323,6 +10326,9 @@ def _run_agent_streaming(
|
||||||
_ttft_ms = meter().get_ttft_ms(stream_id)
|
_ttft_ms = meter().get_ttft_ms(stream_id)
|
||||||
if _ttft_ms is not None:
|
if _ttft_ms is not None:
|
||||||
usage['ttft_ms'] = _ttft_ms
|
usage['ttft_ms'] = _ttft_ms
|
||||||
|
_used_model = resolved_model or model
|
||||||
|
if _used_model:
|
||||||
|
usage['used_model'] = _used_model
|
||||||
# Include context window data from the agent's compressor for the UI indicator.
|
# Include context window data from the agent's compressor for the UI indicator.
|
||||||
# The session-level persistence happens above (before s.save()) so the values
|
# The session-level persistence happens above (before s.save()) so the values
|
||||||
# survive a page reload; this block only populates the live SSE usage payload.
|
# survive a page reload; this block only populates the live SSE usage payload.
|
||||||
|
|
|
||||||
|
|
@ -4411,6 +4411,7 @@ body.resizing .sidebar{transition:none!important;}
|
||||||
.transparent-turn-footer .lf-time{color:var(--text);}
|
.transparent-turn-footer .lf-time{color:var(--text);}
|
||||||
.transparent-turn-footer .lf-sep{opacity:.4;}
|
.transparent-turn-footer .lf-sep{opacity:.4;}
|
||||||
.transparent-turn-footer .lf-status{color:var(--muted);opacity:.85;}
|
.transparent-turn-footer .lf-status{color:var(--muted);opacity:.85;}
|
||||||
|
.transparent-turn-footer .lf-model{color:var(--muted);opacity:.85;}
|
||||||
.transparent-turn-footer .lf-ttft{color:var(--accent);opacity:.9;}
|
.transparent-turn-footer .lf-ttft{color:var(--accent);opacity:.9;}
|
||||||
.transparent-turn-footer .lf-tokens{font-variant-numeric:tabular-nums;}
|
.transparent-turn-footer .lf-tokens{font-variant-numeric:tabular-nums;}
|
||||||
|
|
||||||
|
|
@ -6110,6 +6111,7 @@ main.main > #mainPlugin{display:none;}
|
||||||
.msg-usage-inline,
|
.msg-usage-inline,
|
||||||
.msg-duration-inline,
|
.msg-duration-inline,
|
||||||
.msg-gateway-inline,
|
.msg-gateway-inline,
|
||||||
|
.msg-used-model-inline,
|
||||||
.gateway-failover-inline,
|
.gateway-failover-inline,
|
||||||
.msg-model-warning-inline {
|
.msg-model-warning-inline {
|
||||||
font-size: 11px;
|
font-size: 11px;
|
||||||
|
|
|
||||||
28
static/ui.js
28
static/ui.js
|
|
@ -7030,6 +7030,14 @@ function _formatGatewayModelLabel(modelId,labelText,routing){
|
||||||
const via=_gatewayRoutingLabel(routing);
|
const via=_gatewayRoutingLabel(routing);
|
||||||
return via?`${base} ${via}`:base;
|
return via?`${base} ${via}`:base;
|
||||||
}
|
}
|
||||||
|
function _usedModelTurnChipLabel(msg){
|
||||||
|
if(!msg)return'';
|
||||||
|
const routing=msg._gatewayRouting||null;
|
||||||
|
if(routing&&String(routing.used_model||'').trim())return'';
|
||||||
|
const usedModel=String(msg._usedModel||'').trim();
|
||||||
|
if(!usedModel)return'';
|
||||||
|
return _compactComposerModelChipLabel(usedModel,getModelLabel(usedModel));
|
||||||
|
}
|
||||||
function _gatewayRoutingFailoverText(routing){
|
function _gatewayRoutingFailoverText(routing){
|
||||||
if(!routing||!routing.has_failover)return'';
|
if(!routing||!routing.has_failover)return'';
|
||||||
const attempts=Array.isArray(routing.routing)?routing.routing:[];
|
const attempts=Array.isArray(routing.routing)?routing.routing:[];
|
||||||
|
|
@ -11754,9 +11762,10 @@ function _applyTransparentRowFading(turn){
|
||||||
// Mirrors the live run-status line for settled turns in transparent
|
// Mirrors the live run-status line for settled turns in transparent
|
||||||
// mode. Shows duration, first-token time, token usage, and final status.
|
// mode. Shows duration, first-token time, token usage, and final status.
|
||||||
// Only rendered for turns that have transparent event rows.
|
// Only rendered for turns that have transparent event rows.
|
||||||
function _transparentTurnFooterHtml(durationText, ttftText, tokensText, statusText){
|
function _transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText){
|
||||||
const parts=[];
|
const parts=[];
|
||||||
if(durationText) parts.push(`<span class="lf-time">${esc(durationText)}</span>`);
|
if(durationText) parts.push(`<span class="lf-time">${esc(durationText)}</span>`);
|
||||||
|
if(modelText) parts.push(`<span class="lf-model">${esc(modelText)}</span>`);
|
||||||
if(ttftText) parts.push(`<span class="lf-ttft" title="${esc(t('first_token_time')||'Time to first token')}">TTFT ${esc(ttftText)}</span>`);
|
if(ttftText) parts.push(`<span class="lf-ttft" title="${esc(t('first_token_time')||'Time to first token')}">TTFT ${esc(ttftText)}</span>`);
|
||||||
if(tokensText) parts.push(`<span class="lf-tokens">${esc(tokensText)}</span>`);
|
if(tokensText) parts.push(`<span class="lf-tokens">${esc(tokensText)}</span>`);
|
||||||
if(statusText) parts.push(`<span class="lf-status">${esc(statusText)}</span>`);
|
if(statusText) parts.push(`<span class="lf-status">${esc(statusText)}</span>`);
|
||||||
|
|
@ -11775,10 +11784,11 @@ function _renderTransparentTurnFooter(turn, opts){
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const durationText=opts&&opts.durationText||'';
|
const durationText=opts&&opts.durationText||'';
|
||||||
|
const modelText=opts&&opts.modelText||'';
|
||||||
const ttftText=opts&&opts.ttftText||'';
|
const ttftText=opts&&opts.ttftText||'';
|
||||||
const tokensText=opts&&opts.tokensText||'';
|
const tokensText=opts&&opts.tokensText||'';
|
||||||
const statusText=opts&&opts.statusText||(t('done')||'Done');
|
const statusText=opts&&opts.statusText||(t('done')||'Done');
|
||||||
const html=_transparentTurnFooterHtml(durationText, ttftText, tokensText, statusText);
|
const html=_transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText);
|
||||||
let footer=turn.querySelector('.transparent-turn-footer');
|
let footer=turn.querySelector('.transparent-turn-footer');
|
||||||
if(!html){
|
if(!html){
|
||||||
if(footer) footer.remove();
|
if(footer) footer.remove();
|
||||||
|
|
@ -16595,12 +16605,13 @@ function renderMessages(options){
|
||||||
// Worklog above the final answer.
|
// Worklog above the final answer.
|
||||||
const compactWorklogForMessage=isCompactWorklogMode()&&(toolCallAssistantIdxs.has(mi)||assistantThinking.has(mi));
|
const compactWorklogForMessage=isCompactWorklogMode()&&(toolCallAssistantIdxs.has(mi)||assistantThinking.has(mi));
|
||||||
const durationText=compactWorklogForMessage?'':_formatTurnDuration(msg._turnDuration);
|
const durationText=compactWorklogForMessage?'':_formatTurnDuration(msg._turnDuration);
|
||||||
if(!hasTurnUsage&&!durationText&&!gatewayText&&!failoverText&&!modelWarningText) continue;
|
const usedModelText=_usedModelTurnChipLabel(msg);
|
||||||
|
if(!hasTurnUsage&&!durationText&&!gatewayText&&!failoverText&&!modelWarningText&&!usedModelText) continue;
|
||||||
const seg=assistantSegments.get(mi);
|
const seg=assistantSegments.get(mi);
|
||||||
const row=seg?seg.closest('.assistant-turn'):null;
|
const row=seg?seg.closest('.assistant-turn'):null;
|
||||||
const footerRows=row?row.querySelectorAll('.msg-foot'):[];
|
const footerRows=row?row.querySelectorAll('.msg-foot'):[];
|
||||||
const targetFoot=footerRows.length?footerRows[footerRows.length-1]:null;
|
const targetFoot=footerRows.length?footerRows[footerRows.length-1]:null;
|
||||||
if(!targetFoot||targetFoot.querySelector('.msg-usage-inline,.msg-duration-inline,.msg-gateway-inline,.gateway-failover-inline,.msg-model-warning-inline')) continue;
|
if(!targetFoot||targetFoot.querySelector('.msg-usage-inline,.msg-duration-inline,.msg-gateway-inline,.gateway-failover-inline,.msg-model-warning-inline,.msg-used-model-inline')) continue;
|
||||||
const fragments=[];
|
const fragments=[];
|
||||||
if(modelWarningText){
|
if(modelWarningText){
|
||||||
const warning=document.createElement('span');
|
const warning=document.createElement('span');
|
||||||
|
|
@ -16620,6 +16631,12 @@ function renderMessages(options){
|
||||||
gateway.textContent=gatewayText;
|
gateway.textContent=gatewayText;
|
||||||
fragments.push(gateway);
|
fragments.push(gateway);
|
||||||
}
|
}
|
||||||
|
if(usedModelText){
|
||||||
|
const usedModel=document.createElement('span');
|
||||||
|
usedModel.className='msg-used-model-inline';
|
||||||
|
usedModel.textContent=usedModelText;
|
||||||
|
fragments.push(usedModel);
|
||||||
|
}
|
||||||
if(durationText){
|
if(durationText){
|
||||||
const duration=document.createElement('span');
|
const duration=document.createElement('span');
|
||||||
duration.className='msg-duration-inline';
|
duration.className='msg-duration-inline';
|
||||||
|
|
@ -16676,6 +16693,7 @@ function renderMessages(options){
|
||||||
// Find the corresponding message to read duration/usage.
|
// Find the corresponding message to read duration/usage.
|
||||||
const seg=turn.querySelector('.assistant-segment');
|
const seg=turn.querySelector('.assistant-segment');
|
||||||
let durationText='';
|
let durationText='';
|
||||||
|
let modelText='';
|
||||||
let ttftText='';
|
let ttftText='';
|
||||||
let tokensText='';
|
let tokensText='';
|
||||||
if(seg){
|
if(seg){
|
||||||
|
|
@ -16683,6 +16701,7 @@ function renderMessages(options){
|
||||||
if(mi!=null){
|
if(mi!=null){
|
||||||
const msg=S.messages[Number(mi)]||{};
|
const msg=S.messages[Number(mi)]||{};
|
||||||
if(msg._turnDuration!=null) durationText=_formatTurnDuration(msg._turnDuration);
|
if(msg._turnDuration!=null) durationText=_formatTurnDuration(msg._turnDuration);
|
||||||
|
modelText=_usedModelTurnChipLabel(msg);
|
||||||
if(msg._firstTokenMs!=null) ttftText=_formatFirstToken(msg._firstTokenMs);
|
if(msg._firstTokenMs!=null) ttftText=_formatFirstToken(msg._firstTokenMs);
|
||||||
if(msg._turnUsage){
|
if(msg._turnUsage){
|
||||||
const inTok=msg._turnUsage.input_tokens||0;
|
const inTok=msg._turnUsage.input_tokens||0;
|
||||||
|
|
@ -16693,6 +16712,7 @@ function renderMessages(options){
|
||||||
}
|
}
|
||||||
_renderTransparentTurnFooter(turn,{
|
_renderTransparentTurnFooter(turn,{
|
||||||
durationText,
|
durationText,
|
||||||
|
modelText,
|
||||||
ttftText,
|
ttftText,
|
||||||
tokensText,
|
tokensText,
|
||||||
statusText: t('done')||'Done',
|
statusText: t('done')||'Done',
|
||||||
|
|
|
||||||
55
tests/test_issue6068_used_model_footer.py
Normal file
55
tests/test_issue6068_used_model_footer.py
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
"""Regression coverage for #6068 per-turn used-model footer instrumentation."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from api.models import Session
|
||||||
|
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[1]
|
||||||
|
STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8")
|
||||||
|
MODELS_PY = (REPO / "api" / "models.py").read_text(encoding="utf-8")
|
||||||
|
UI_JS = (REPO / "static" / "ui.js").read_text(encoding="utf-8")
|
||||||
|
STYLE_CSS = (REPO / "static" / "style.css").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_streaming_stamps_used_model_on_assistant_message_and_usage_payload():
|
||||||
|
assert "_dm['_usedModel'] = _used_model" in STREAMING_PY
|
||||||
|
assert "_used_model = resolved_model or model" in STREAMING_PY
|
||||||
|
assert "usage['used_model'] = _used_model" in STREAMING_PY
|
||||||
|
|
||||||
|
|
||||||
|
def test_models_allowlist_round_trips_used_model_across_save_reload():
|
||||||
|
assert '"_usedModel"' in MODELS_PY
|
||||||
|
assert "_usedModel" in MODELS_PY.split("_SESSION_MESSAGE_DISPLAY_METADATA_KEYS")[1].split(")")[0]
|
||||||
|
|
||||||
|
session = Session(session_id="6068usedmodel", title="Used model")
|
||||||
|
session.messages = [
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"content": "done",
|
||||||
|
"_firstTokenMs": 250,
|
||||||
|
"_usedModel": "gpt-5-mini",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
session.save()
|
||||||
|
|
||||||
|
reloaded = Session.load("6068usedmodel")
|
||||||
|
assert reloaded.messages[-1]["_usedModel"] == "gpt-5-mini"
|
||||||
|
assert reloaded.messages[-1]["_firstTokenMs"] == 250
|
||||||
|
|
||||||
|
|
||||||
|
def test_settled_footer_renders_used_model_chip_and_suppresses_gateway_duplicate():
|
||||||
|
assert "function _usedModelTurnChipLabel" in UI_JS
|
||||||
|
assert "msg-used-model-inline" in UI_JS
|
||||||
|
assert "_usedModelTurnChipLabel(msg)" in UI_JS
|
||||||
|
assert "routing.used_model" in UI_JS
|
||||||
|
assert "msg._usedModel" in UI_JS
|
||||||
|
assert "_compactComposerModelChipLabel(usedModel,getModelLabel(usedModel))" in UI_JS
|
||||||
|
assert ".msg-used-model-inline" in STYLE_CSS
|
||||||
|
|
||||||
|
|
||||||
|
def test_transparent_turn_footer_includes_model_between_duration_and_ttft():
|
||||||
|
assert "function _transparentTurnFooterHtml(durationText, modelText, ttftText, tokensText, statusText)" in UI_JS
|
||||||
|
assert 'class="lf-model"' in UI_JS
|
||||||
|
assert "modelText=_usedModelTurnChipLabel(msg)" in UI_JS
|
||||||
|
assert ".transparent-turn-footer .lf-model" in STYLE_CSS
|
||||||
Loading…
Reference in a new issue