diff --git a/api/routes.py b/api/routes.py index 6adfb5c1..46b2321b 100644 --- a/api/routes.py +++ b/api/routes.py @@ -2872,7 +2872,7 @@ def _handle_chat_start(handler, body): msg = str(body.get("message", "")).strip() if not msg: return bad(handler, "message is required") - attachments = [str(a) for a in (body.get("attachments") or [])][:20] + attachments = _normalize_chat_attachments(body.get("attachments") or [])[:20] try: workspace = str(resolve_trusted_workspace(body.get("workspace") or s.workspace)) except ValueError as e: @@ -2922,6 +2922,36 @@ def _handle_chat_start(handler, body): return j(handler, response) +def _normalize_chat_attachments(raw_attachments): + """Normalize attachment payloads from the browser. + + Older clients send a list of filenames. Newer clients send upload result + objects containing name/path/mime/size so image attachments can be supplied + to Hermes as native multimodal inputs for the current turn. + """ + normalized = [] + if not isinstance(raw_attachments, list): + return normalized + for item in raw_attachments: + if isinstance(item, dict): + name = str(item.get("name") or item.get("filename") or "").strip() + path = str(item.get("path") or "").strip() + mime = str(item.get("mime") or "").strip() + att = {"name": name or path, "path": path, "mime": mime} + size = item.get("size") + if isinstance(size, int): + att["size"] = size + is_image = item.get("is_image") + if isinstance(is_image, bool): + att["is_image"] = is_image + normalized.append(att) + else: + value = str(item).strip() + if value: + normalized.append({"name": value, "path": "", "mime": ""}) + return normalized + + def _handle_chat_sync(handler, body): """Fallback synchronous chat endpoint (POST /api/chat). Not used by frontend.""" s = get_session(body["session_id"]) diff --git a/api/streaming.py b/api/streaming.py index a2466085..dd60cb96 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -2,9 +2,11 @@ Hermes Web UI -- SSE streaming engine and agent thread runner. Includes Sprint 10 cancel support via CANCEL_FLAGS. """ +import base64 import contextlib import json import logging +import mimetypes import os import queue import re @@ -63,6 +65,60 @@ from api.workspace import set_last_workspace # metadata added by the webui and must be stripped before the API call. _API_SAFE_MSG_KEYS = {'role', 'content', 'tool_calls', 'tool_call_id', 'name', 'refusal'} +_NATIVE_IMAGE_MAX_BYTES = 20 * 1024 * 1024 + + +def _attachment_name(att) -> str: + if isinstance(att, dict): + return str(att.get('name') or att.get('filename') or att.get('path') or '').strip() + return str(att or '').strip() + + +def _build_native_multimodal_message(workspace_ctx: str, msg_text: str, attachments, workspace: str): + """Build native multimodal content parts for current-turn image uploads. + + WebUI uploads files into the active workspace. For image files, pass the + bytes to Hermes as OpenAI-style image_url data URLs so vision-capable main + models can consume them in the same request. Non-image files intentionally + stay as text path attachments so the agent can inspect them with file tools. + """ + if not attachments: + return workspace_ctx + msg_text + + parts = [{'type': 'text', 'text': workspace_ctx + msg_text}] + workspace_root = Path(workspace).expanduser().resolve() + image_count = 0 + + for att in attachments or []: + if not isinstance(att, dict): + continue + raw_path = str(att.get('path') or '').strip() + if not raw_path: + continue + try: + path = Path(raw_path).expanduser().resolve() + # Uploads should live inside the selected workspace. Do not read + # arbitrary paths from client-provided attachment metadata. + path.relative_to(workspace_root) + if not path.is_file(): + continue + size = path.stat().st_size + if size <= 0 or size > _NATIVE_IMAGE_MAX_BYTES: + continue + mime = str(att.get('mime') or '').strip() or (mimetypes.guess_type(path.name)[0] or '') + if not mime.startswith('image/'): + continue + data = base64.b64encode(path.read_bytes()).decode('ascii') + except Exception: + continue + parts.append({ + 'type': 'image_url', + 'image_url': {'url': f'data:{mime};base64,{data}'}, + }) + image_count += 1 + + return parts if image_count else workspace_ctx + msg_text + def _strip_thinking_markup(text: str) -> str: """Remove common reasoning/thinking wrappers from model text.""" @@ -1677,8 +1733,9 @@ def _run_agent_streaming(session_id, msg_text, model, workspace, stream_id, atta ) _ckpt_thread.start() + user_message = _build_native_multimodal_message(workspace_ctx, msg_text, attachments, workspace) result = agent.run_conversation( - user_message=workspace_ctx + msg_text, + user_message=user_message, system_message=workspace_system_msg, conversation_history=_sanitize_messages_for_api(s.messages), task_id=session_id, @@ -1902,13 +1959,14 @@ def _run_agent_streaming(session_id, msg_text, model, workspace, stream_id, atta # Only tag a user message whose content relates to this turn's text # (msg_text is the full message including the [Attached files: ...] suffix) if attachments: + display_attachments = [_attachment_name(a) for a in attachments if _attachment_name(a)] for m in reversed(s.messages): if m.get('role') == 'user': content = str(m.get('content', '')) # Match if content is part of the sent message or vice-versa base_text = msg_text.split('\n\n[Attached files:')[0].strip() if '\n\n[Attached files:' in msg_text else msg_text if base_text[:60] in content or content[:60] in msg_text: - m['attachments'] = attachments + m['attachments'] = display_attachments break # Persist reasoning trace in the session so it survives reload. # Must run BEFORE s.save() — otherwise the mutation lives only in diff --git a/api/upload.py b/api/upload.py index ec1dab38..059d8920 100644 --- a/api/upload.py +++ b/api/upload.py @@ -1,6 +1,7 @@ """ Hermes Web UI -- File upload: multipart parser and upload handler. """ +import mimetypes import re as _re import email.parser import tempfile @@ -80,7 +81,14 @@ def handle_upload(handler): safe_name = _sanitize_upload_name(filename) dest = safe_resolve_ws(workspace, safe_name) dest.write_bytes(file_bytes) - return j(handler, {'filename': safe_name, 'path': str(dest), 'size': dest.stat().st_size}) + mime = mimetypes.guess_type(safe_name)[0] or 'application/octet-stream' + return j(handler, { + 'filename': safe_name, + 'path': str(dest), + 'size': dest.stat().st_size, + 'mime': mime, + 'is_image': mime.startswith('image/'), + }) except ValueError as e: return j(handler, {'error': str(e)}, status=400) except Exception: diff --git a/static/messages.js b/static/messages.js index 2c15270b..6076b88e 100644 --- a/static/messages.js +++ b/static/messages.js @@ -108,7 +108,7 @@ async function send(){ catch(e){if(!text){setComposerStatus(`Upload error: ${e.message}`);return;}} const uploadedNames=uploaded.map(u=>u.name||u); - const uploadedPaths=uploaded.map(u=>u.path||u.name||u); + const uploadedPaths=uploaded.map(u=>u&&u.is_image?(u.name||u.filename||u):(u.path||u.name||u)); let msgText=text; if(uploaded.length&&!msgText)msgText=`I've uploaded ${uploaded.length} file(s): ${uploadedPaths.join(', ')}`; else if(uploaded.length)msgText=`${text}\n\n[Attached files: ${uploadedPaths.join(', ')}]`; @@ -151,7 +151,7 @@ async function send(){ const startData=await api('/api/chat/start',{method:'POST',body:JSON.stringify({ session_id:activeSid,message:msgText, model:S.session.model||$('modelSelect').value,workspace:S.session.workspace, - attachments:uploaded.length?uploadedNames:undefined + attachments:uploaded.length?uploaded:undefined })}); if(startData.effective_model && S.session){ S.session.model=startData.effective_model; diff --git a/static/ui.js b/static/ui.js index 2d891e0e..2d80af14 100644 --- a/static/ui.js +++ b/static/ui.js @@ -2453,7 +2453,8 @@ function renderMessages(){ if(m.attachments&&m.attachments.length){ const _attachSid=(S.session&&S.session.session_id)||''; filesHtml=`
${m.attachments.map(f=>{ - const fname=f.split('/').pop()||f; + const fLabel=typeof f==='string'?f:(f&&(f.name||f.filename||f.path))||''; + const fname=String(fLabel).split('/').pop()||String(fLabel); if(_IMAGE_EXTS.test(fname)){ // Use api/file/raw which resolves filename relative to the session workspace. // api/media expects a full absolute path which we don't store on the client side. @@ -3485,7 +3486,7 @@ async function uploadPendingFiles(){ if(!res.ok){const err=await res.text();throw new Error(err);} const data=await res.json(); if(data.error)throw new Error(data.error); - names.push({name: data.filename, path: data.path}); + names.push({name: data.filename, path: data.path, mime: data.mime, size: data.size, is_image: !!data.is_image}); }catch(e){failures++;setStatus(`\u274c ${t('upload_failed')}${f.name} \u2014 ${e.message}`);} bar.style.width=`${Math.round((i+1)/total*100)}%`; }