fix: send image uploads as native multimodal inputs
This commit is contained in:
yzp12138
parent
24b1e6f3fc
commit
f35d7786e5
5 changed files with 105 additions and 8 deletions
|
|
@ -2872,7 +2872,7 @@ def _handle_chat_start(handler, body):
|
|||
msg = str(body.get("message", "")).strip()
|
||||
if not msg:
|
||||
return bad(handler, "message is required")
|
||||
attachments = [str(a) for a in (body.get("attachments") or [])][:20]
|
||||
attachments = _normalize_chat_attachments(body.get("attachments") or [])[:20]
|
||||
try:
|
||||
workspace = str(resolve_trusted_workspace(body.get("workspace") or s.workspace))
|
||||
except ValueError as e:
|
||||
|
|
@ -2922,6 +2922,36 @@ def _handle_chat_start(handler, body):
|
|||
return j(handler, response)
|
||||
|
||||
|
||||
def _normalize_chat_attachments(raw_attachments):
|
||||
"""Normalize attachment payloads from the browser.
|
||||
|
||||
Older clients send a list of filenames. Newer clients send upload result
|
||||
objects containing name/path/mime/size so image attachments can be supplied
|
||||
to Hermes as native multimodal inputs for the current turn.
|
||||
"""
|
||||
normalized = []
|
||||
if not isinstance(raw_attachments, list):
|
||||
return normalized
|
||||
for item in raw_attachments:
|
||||
if isinstance(item, dict):
|
||||
name = str(item.get("name") or item.get("filename") or "").strip()
|
||||
path = str(item.get("path") or "").strip()
|
||||
mime = str(item.get("mime") or "").strip()
|
||||
att = {"name": name or path, "path": path, "mime": mime}
|
||||
size = item.get("size")
|
||||
if isinstance(size, int):
|
||||
att["size"] = size
|
||||
is_image = item.get("is_image")
|
||||
if isinstance(is_image, bool):
|
||||
att["is_image"] = is_image
|
||||
normalized.append(att)
|
||||
else:
|
||||
value = str(item).strip()
|
||||
if value:
|
||||
normalized.append({"name": value, "path": "", "mime": ""})
|
||||
return normalized
|
||||
|
||||
|
||||
def _handle_chat_sync(handler, body):
|
||||
"""Fallback synchronous chat endpoint (POST /api/chat). Not used by frontend."""
|
||||
s = get_session(body["session_id"])
|
||||
|
|
|
|||
|
|
@ -2,9 +2,11 @@
|
|||
Hermes Web UI -- SSE streaming engine and agent thread runner.
|
||||
Includes Sprint 10 cancel support via CANCEL_FLAGS.
|
||||
"""
|
||||
import base64
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
|
|
@ -63,6 +65,60 @@ from api.workspace import set_last_workspace
|
|||
# metadata added by the webui and must be stripped before the API call.
|
||||
_API_SAFE_MSG_KEYS = {'role', 'content', 'tool_calls', 'tool_call_id', 'name', 'refusal'}
|
||||
|
||||
_NATIVE_IMAGE_MAX_BYTES = 20 * 1024 * 1024
|
||||
|
||||
|
||||
def _attachment_name(att) -> str:
|
||||
if isinstance(att, dict):
|
||||
return str(att.get('name') or att.get('filename') or att.get('path') or '').strip()
|
||||
return str(att or '').strip()
|
||||
|
||||
|
||||
def _build_native_multimodal_message(workspace_ctx: str, msg_text: str, attachments, workspace: str):
|
||||
"""Build native multimodal content parts for current-turn image uploads.
|
||||
|
||||
WebUI uploads files into the active workspace. For image files, pass the
|
||||
bytes to Hermes as OpenAI-style image_url data URLs so vision-capable main
|
||||
models can consume them in the same request. Non-image files intentionally
|
||||
stay as text path attachments so the agent can inspect them with file tools.
|
||||
"""
|
||||
if not attachments:
|
||||
return workspace_ctx + msg_text
|
||||
|
||||
parts = [{'type': 'text', 'text': workspace_ctx + msg_text}]
|
||||
workspace_root = Path(workspace).expanduser().resolve()
|
||||
image_count = 0
|
||||
|
||||
for att in attachments or []:
|
||||
if not isinstance(att, dict):
|
||||
continue
|
||||
raw_path = str(att.get('path') or '').strip()
|
||||
if not raw_path:
|
||||
continue
|
||||
try:
|
||||
path = Path(raw_path).expanduser().resolve()
|
||||
# Uploads should live inside the selected workspace. Do not read
|
||||
# arbitrary paths from client-provided attachment metadata.
|
||||
path.relative_to(workspace_root)
|
||||
if not path.is_file():
|
||||
continue
|
||||
size = path.stat().st_size
|
||||
if size <= 0 or size > _NATIVE_IMAGE_MAX_BYTES:
|
||||
continue
|
||||
mime = str(att.get('mime') or '').strip() or (mimetypes.guess_type(path.name)[0] or '')
|
||||
if not mime.startswith('image/'):
|
||||
continue
|
||||
data = base64.b64encode(path.read_bytes()).decode('ascii')
|
||||
except Exception:
|
||||
continue
|
||||
parts.append({
|
||||
'type': 'image_url',
|
||||
'image_url': {'url': f'data:{mime};base64,{data}'},
|
||||
})
|
||||
image_count += 1
|
||||
|
||||
return parts if image_count else workspace_ctx + msg_text
|
||||
|
||||
|
||||
def _strip_thinking_markup(text: str) -> str:
|
||||
"""Remove common reasoning/thinking wrappers from model text."""
|
||||
|
|
@ -1677,8 +1733,9 @@ def _run_agent_streaming(session_id, msg_text, model, workspace, stream_id, atta
|
|||
)
|
||||
_ckpt_thread.start()
|
||||
|
||||
user_message = _build_native_multimodal_message(workspace_ctx, msg_text, attachments, workspace)
|
||||
result = agent.run_conversation(
|
||||
user_message=workspace_ctx + msg_text,
|
||||
user_message=user_message,
|
||||
system_message=workspace_system_msg,
|
||||
conversation_history=_sanitize_messages_for_api(s.messages),
|
||||
task_id=session_id,
|
||||
|
|
@ -1902,13 +1959,14 @@ def _run_agent_streaming(session_id, msg_text, model, workspace, stream_id, atta
|
|||
# Only tag a user message whose content relates to this turn's text
|
||||
# (msg_text is the full message including the [Attached files: ...] suffix)
|
||||
if attachments:
|
||||
display_attachments = [_attachment_name(a) for a in attachments if _attachment_name(a)]
|
||||
for m in reversed(s.messages):
|
||||
if m.get('role') == 'user':
|
||||
content = str(m.get('content', ''))
|
||||
# Match if content is part of the sent message or vice-versa
|
||||
base_text = msg_text.split('\n\n[Attached files:')[0].strip() if '\n\n[Attached files:' in msg_text else msg_text
|
||||
if base_text[:60] in content or content[:60] in msg_text:
|
||||
m['attachments'] = attachments
|
||||
m['attachments'] = display_attachments
|
||||
break
|
||||
# Persist reasoning trace in the session so it survives reload.
|
||||
# Must run BEFORE s.save() — otherwise the mutation lives only in
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
Hermes Web UI -- File upload: multipart parser and upload handler.
|
||||
"""
|
||||
import mimetypes
|
||||
import re as _re
|
||||
import email.parser
|
||||
import tempfile
|
||||
|
|
@ -80,7 +81,14 @@ def handle_upload(handler):
|
|||
safe_name = _sanitize_upload_name(filename)
|
||||
dest = safe_resolve_ws(workspace, safe_name)
|
||||
dest.write_bytes(file_bytes)
|
||||
return j(handler, {'filename': safe_name, 'path': str(dest), 'size': dest.stat().st_size})
|
||||
mime = mimetypes.guess_type(safe_name)[0] or 'application/octet-stream'
|
||||
return j(handler, {
|
||||
'filename': safe_name,
|
||||
'path': str(dest),
|
||||
'size': dest.stat().st_size,
|
||||
'mime': mime,
|
||||
'is_image': mime.startswith('image/'),
|
||||
})
|
||||
except ValueError as e:
|
||||
return j(handler, {'error': str(e)}, status=400)
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ async function send(){
|
|||
catch(e){if(!text){setComposerStatus(`Upload error: ${e.message}`);return;}}
|
||||
|
||||
const uploadedNames=uploaded.map(u=>u.name||u);
|
||||
const uploadedPaths=uploaded.map(u=>u.path||u.name||u);
|
||||
const uploadedPaths=uploaded.map(u=>u&&u.is_image?(u.name||u.filename||u):(u.path||u.name||u));
|
||||
let msgText=text;
|
||||
if(uploaded.length&&!msgText)msgText=`I've uploaded ${uploaded.length} file(s): ${uploadedPaths.join(', ')}`;
|
||||
else if(uploaded.length)msgText=`${text}\n\n[Attached files: ${uploadedPaths.join(', ')}]`;
|
||||
|
|
@ -151,7 +151,7 @@ async function send(){
|
|||
const startData=await api('/api/chat/start',{method:'POST',body:JSON.stringify({
|
||||
session_id:activeSid,message:msgText,
|
||||
model:S.session.model||$('modelSelect').value,workspace:S.session.workspace,
|
||||
attachments:uploaded.length?uploadedNames:undefined
|
||||
attachments:uploaded.length?uploaded:undefined
|
||||
})});
|
||||
if(startData.effective_model && S.session){
|
||||
S.session.model=startData.effective_model;
|
||||
|
|
|
|||
|
|
@ -2453,7 +2453,8 @@ function renderMessages(){
|
|||
if(m.attachments&&m.attachments.length){
|
||||
const _attachSid=(S.session&&S.session.session_id)||'';
|
||||
filesHtml=`<div class="msg-files">${m.attachments.map(f=>{
|
||||
const fname=f.split('/').pop()||f;
|
||||
const fLabel=typeof f==='string'?f:(f&&(f.name||f.filename||f.path))||'';
|
||||
const fname=String(fLabel).split('/').pop()||String(fLabel);
|
||||
if(_IMAGE_EXTS.test(fname)){
|
||||
// Use api/file/raw which resolves filename relative to the session workspace.
|
||||
// api/media expects a full absolute path which we don't store on the client side.
|
||||
|
|
@ -3485,7 +3486,7 @@ async function uploadPendingFiles(){
|
|||
if(!res.ok){const err=await res.text();throw new Error(err);}
|
||||
const data=await res.json();
|
||||
if(data.error)throw new Error(data.error);
|
||||
names.push({name: data.filename, path: data.path});
|
||||
names.push({name: data.filename, path: data.path, mime: data.mime, size: data.size, is_image: !!data.is_image});
|
||||
}catch(e){failures++;setStatus(`\u274c ${t('upload_failed')}${f.name} \u2014 ${e.message}`);}
|
||||
bar.style.width=`${Math.round((i+1)/total*100)}%`;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue