Merge pull request #4352 from nesquena/stage-3510

Release v0.51.475 — ElevenLabs TTS engine (#3510)
This commit is contained in:
nesquena-hermes 2026-06-16 23:08:13 -07:00 committed by GitHub
commit 502f806d5b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 358 additions and 4 deletions

View file

@ -3,6 +3,12 @@
## [Unreleased]
## [v0.51.475] — 2026-06-17 — Release QJ (ElevenLabs TTS engine)
### Added
- **ElevenLabs is now selectable as a TTS engine in the WebUI (#3510).** Alongside Browser speech synthesis and Edge TTS, you can pick ElevenLabs for voice mode and message read-aloud. It uses the same credentials and voice you've already configured for the Hermes agent — `ELEVENLABS_API_KEY` (env, falling back to `~/.hermes/.env`) and `tts.elevenlabs.voice_id` / `model` from `config.yaml` — so there's nothing new to set up if the agent already has it (ElevenLabs is a built-in Hermes TTS provider). The endpoint validates the configured `voice_id` as a safe path segment, enforces the existing 5000-character cap and rate limit, and returns a clear error when no key is configured. Thanks @linux-fertxo.
## [v0.51.474] — 2026-06-17 — Release QI (reconcile WebUI session source flags)
### Fixed

View file

@ -11277,6 +11277,7 @@ def _handle_tts(handler, parsed):
voice = "zh-CN-XiaoxiaoNeural"
rate_str = ""
pitch_str = ""
engine = "edge" # "edge" | "elevenlabs" | "browser" (browser is client-side only)
if handler.command != "POST":
from api.helpers import bad as _bad
@ -11288,6 +11289,7 @@ def _handle_tts(handler, parsed):
voice = data.get("voice") or voice
rate_str = _normalize_tts_prosody(data.get("rate"), unit="%")
pitch_str = _normalize_tts_prosody(data.get("pitch"), unit="Hz")
engine = (data.get("engine") or "edge").strip().lower()
except Exception:
from api.helpers import bad as _bad
return _bad(handler, "invalid request body", 400)
@ -11360,6 +11362,90 @@ def _handle_tts(handler, parsed):
from api.helpers import bad as _bad
return _bad(handler, "rate limit exceeded — please wait", 429)
# ── ElevenLabs TTS ──────────────────────────────────────────────────
if engine == "elevenlabs":
api_key = os.getenv("ELEVENLABS_API_KEY", "").strip()
if not api_key:
# Fall back to reading from Hermes .env file
try:
from api.onboarding import _load_env_file
from api.profiles import get_active_hermes_home
api_key = _load_env_file(get_active_hermes_home() / ".env").get("ELEVENLABS_API_KEY", "")
except Exception:
pass
if not api_key:
from api.helpers import bad as _bad
return _bad(handler, "ELEVENLABS_API_KEY not configured", 503)
# Resolve voice_id from Hermes config.yaml → env fallback
voice_id = "pNInz6obpgDQGcFmaJgB" # Adam (same default as hermes-agent config.yaml)
model_id = "eleven_multilingual_v2"
try:
from api.config import get_config
tts_cfg = (get_config() or {}).get("tts", {})
if isinstance(tts_cfg, dict):
el_cfg = tts_cfg.get("elevenlabs", {})
if isinstance(el_cfg, dict):
voice_id = el_cfg.get("voice_id", voice_id)
model_id = el_cfg.get("model", model_id) or el_cfg.get("model_id", model_id)
# ^ treat empty string as "not set" — fall through to default
except Exception:
pass # fall back to defaults
# Validate voice_id is a safe path segment (no traversal)
# fullmatch (not match) so a trailing newline can't slip past the `$`
# anchor — defense-in-depth on the config-derived voice_id before it
# goes into the request URL (#3510 review).
if not re.fullmatch(r'[A-Za-z0-9_-]+', voice_id):
from api.helpers import bad as _bad
return _bad(handler, "invalid voice_id in config", 400)
url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream?output_format=mp3_44100_128"
req_body = json.dumps({
"text": text,
"model_id": model_id,
"voice_settings": {"stability": 0.5, "similarity_boost": 0.75},
}).encode("utf-8")
from urllib.request import Request, urlopen as _urlopen
req = Request(url, data=req_body, headers={
"xi-api-key": api_key,
"Content-Type": "application/json",
"Accept": "audio/mpeg",
})
# Buffer the full response before sending first byte.
# The streaming endpoint is designed for chunked delivery, but urllib's
# chunked-read path adds per-chunk overhead that dominates short TTS
# payloads. With the 5000-char cap enforced above the buffer is bounded
# (a few MB of mp3 worst case), so full-buffer-then-send is faster in
# practice and simpler to reason about.
audio_data = b""
try:
with _urlopen(req, timeout=30) as resp:
while True:
chunk = resp.read(65536)
if not chunk:
break
audio_data += chunk
except Exception:
logger.exception("ElevenLabs TTS generation failed")
from api.helpers import bad as _bad
return _bad(handler, "ElevenLabs TTS generation failed", 500)
handler.send_response(200)
handler.send_header("Content-Type", "audio/mpeg")
handler.send_header("Cache-Control", "no-store")
handler.send_header("Content-Length", str(len(audio_data)))
handler.end_headers()
try:
handler.wfile.write(audio_data)
except (BrokenPipeError, ConnectionResetError):
pass
return True
# ── Edge TTS ────────────────────────────────────────────────────────
allowed = {
"zh-CN-XiaoxiaoNeural", "zh-CN-XiaoyiNeural", "zh-CN-YunxiNeural",
"zh-CN-YunjianNeural", "zh-CN-YunyangNeural",

View file

@ -1034,6 +1034,46 @@ window._micPendingSend=window._micPendingSend||false;
}
if(!clean){ _startListening(); return; }
const engine=localStorage.getItem("hermes-tts-engine")||"browser";
if(engine==="elevenlabs"){
_ttsSpeaking=true;
fetch(new URL('api/tts', document.baseURI || location.href).href, {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({text: clean, engine: 'elevenlabs'})
})
.then(r => {
if(!r.ok) throw new Error('TTS request failed: ' + r.status);
return r.blob();
})
.then(blob => {
const url = URL.createObjectURL(blob);
const audio = new Audio(url);
_playingEdgeAudio=audio;
audio.onended = () => {
_ttsSpeaking=false;
if(_playingEdgeAudio===audio) _playingEdgeAudio=null;
URL.revokeObjectURL(url);
if(_voiceModeActive) setTimeout(()=>_startListening(),500);
};
audio.onerror = () => {
_ttsSpeaking=false;
if(_playingEdgeAudio===audio) _playingEdgeAudio=null;
URL.revokeObjectURL(url);
if(_voiceModeActive) setTimeout(()=>_startListening(),1000);
};
audio.play().catch(e => {
_ttsSpeaking=false;
if(_playingEdgeAudio===audio) _playingEdgeAudio=null;
URL.revokeObjectURL(url);
if(_voiceModeActive) setTimeout(()=>_startListening(),1000);
});
})
.catch(() => {
_ttsSpeaking=false;
if(_voiceModeActive) setTimeout(()=>_startListening(),1000);
});
return;
}
if(engine==="edge"){
const voice=localStorage.getItem("hermes-tts-voice")||"zh-CN-XiaoxiaoNeural";
const savedRate=parseFloat(localStorage.getItem("hermes-tts-rate"));

View file

@ -1149,7 +1149,7 @@
<div style="font-size:11px;color:var(--muted);margin-top:4px" data-i18n="settings_desc_raw_audio">Record and send the original audio file to the agent instead of converting it to text first. The agent can then transcribe it or process the raw audio (emotion, background noise, custom STT). Like Telegram's voice message behavior.</div>
</div>
<div class="settings-field">
<label for="settingsTtsEngine" data-i18n="settings_label_tts_engine">TTS Engine</label><select id="settingsTtsEngine" style="width:100%;padding:8px;background:var(--code-bg);color:var(--text);border:1px solid var(--border2);border-radius:6px"><option value="browser">Browser speech synthesis</option><option value="edge">Edge TTS (server)</option></select><div style="font-size:11px;color:var(--muted);margin-top:4px" data-i18n="settings_desc_tts_engine">Choose speech engine. Edge TTS uses Microsoft neural voices via the server.</div></div><div class="settings-field"><label for="settingsTtsVoice" data-i18n="settings_label_tts_voice">Voice</label>
<label for="settingsTtsEngine" data-i18n="settings_label_tts_engine">TTS Engine</label><select id="settingsTtsEngine" style="width:100%;padding:8px;background:var(--code-bg);color:var(--text);border:1px solid var(--border2);border-radius:6px"><option value="browser">Browser speech synthesis</option><option value="edge">Edge TTS (server)</option><option value="elevenlabs">ElevenLabs (server)</option></select><div style="font-size:11px;color:var(--muted);margin-top:4px" data-i18n="settings_desc_tts_engine">Choose speech engine. Edge TTS uses Microsoft neural voices via the server.</div></div><div class="settings-field"><label for="settingsTtsVoice" data-i18n="settings_label_tts_voice">Voice</label>
<select id="settingsTtsVoice" style="width:100%;padding:8px;background:var(--code-bg);color:var(--text);border:1px solid var(--border2);border-radius:6px">
<option value="">Default system voice</option>
</select>

View file

@ -7102,7 +7102,9 @@ async function loadSettingsPanel(){
if(!ttsVoiceSel) return;
const engine=localStorage.getItem('hermes-tts-engine')||'browser';
const current=localStorage.getItem('hermes-tts-voice')||'';
if(engine==='edge'){
if(engine==='elevenlabs'){
ttsVoiceSel.innerHTML='<option value="">Hermy — ElevenLabs (server-configured)</option>';
} else if(engine==='edge'){
const edgeVoices=[
{value:'zh-CN-XiaoxiaoNeural',label:'Xiaoxiao (Chinese, Female)'},
{value:'zh-CN-XiaoyiNeural',label:'Xiaoyi (Chinese, Female)'},

View file

@ -5923,6 +5923,10 @@ function speakMessage(btn){
if(!clean) return;
const engine=localStorage.getItem('hermes-tts-engine')||'browser';
if(engine==='elevenlabs'){
_playElevenLabsTts(clean, btn);
return;
}
if(engine==='edge'){
_playEdgeTtsChunked(clean, btn);
return;
@ -5944,13 +5948,88 @@ function speakMessage(btn){
speechSynthesis.speak(utter);
}
function _playElevenLabsTts(text, btn){
if(btn) btn.dataset.speaking='1';
_ttsSpeaking=true;
const _fail=function(msg){
_ttsSpeaking=false;_playingEdgeAudio=null;
if(btn)btn.dataset.speaking='0';
if(msg&&typeof showToast==='function') showToast(msg,4000,'error');
};
fetch(new URL('api/tts', document.baseURI || location.href).href, {
method:'POST',
headers:{'Content-Type':'application/json'},
body:JSON.stringify({text:text, engine:'elevenlabs'})
})
.then(function(r){
if(!r.ok){
return r.json().catch(function(){return {};}).then(function(j){
throw new Error((j&&j.error)||('TTS request failed: '+r.status));
});
}
return r.arrayBuffer();
})
.then(function(buf){
return _playAudioBuf(buf, btn, 'ElevenLabs TTS');
})
.catch(function(e){ _fail((e&&e.message)||'ElevenLabs TTS failed'); });
}
// ── Shared AudioContext for TTS playback (no blob URLs needed) ──
let _ttsAudioCtx=null;
function _getTtsAudioCtx(){
if(!_ttsAudioCtx){
const C=window.AudioContext||window.webkitAudioContext;
if(!C) return null;
_ttsAudioCtx=new C();
}
if(_ttsAudioCtx.state==='suspended') _ttsAudioCtx.resume();
return _ttsAudioCtx;
}
function _playAudioBuf(arrayBuffer, btn, label){
const ctx=_getTtsAudioCtx();
if(!ctx){
if(btn)btn.dataset.speaking='0';
_ttsSpeaking=false;
showToast(label+': Web Audio API not available');
return;
}
return new Promise(function(resolve){
ctx.decodeAudioData(arrayBuffer.slice(0), function(audioBuffer){
const src=ctx.createBufferSource();
src.buffer=audioBuffer;
src.connect(ctx.destination);
_playingEdgeAudio=src;
const _cleanup=function(){
_ttsSpeaking=false;_playingEdgeAudio=null;
if(btn)btn.dataset.speaking='0';
try{src.stop();src.disconnect();}catch(_){}
resolve();
};
src.onended=_cleanup;
src.start(0);
}, function(e){
_ttsSpeaking=false;
if(btn)btn.dataset.speaking='0';
showToast(label+' error: '+(e&&e.message||e));
resolve(); // prevent permanently pending Promise on decode failure
});
});
}
function stopTTS(){
if('speechSynthesis' in window){
speechSynthesis.cancel();
}
// Stop Edge TTS audio
// Stop Web Audio API playback (AudioBufferSourceNode)
if(_playingEdgeAudio){
try{ _playingEdgeAudio.pause(); _playingEdgeAudio.currentTime=0; }catch(_){}
try{
if(typeof _playingEdgeAudio.stop==='function'){
_playingEdgeAudio.stop(); _playingEdgeAudio.disconnect();
}else{
_playingEdgeAudio.pause(); _playingEdgeAudio.currentTime=0;
}
}catch(_){}
_playingEdgeAudio=null;
}
_ttsSpeaking=false;
@ -5975,6 +6054,10 @@ function autoReadLastAssistant(){
if(!text.trim()) return;
const clean=_stripForTTS(text);
if(!clean) return;
if(engine==='elevenlabs'){
_playElevenLabsTts(clean, null);
return;
}
if(engine==='edge'){
_playEdgeTtsChunked(clean, null);
return;

View file

@ -0,0 +1,137 @@
"""Coverage for the ElevenLabs TTS engine on the /api/ttt endpoint (#3510).
Exercises the engine routing + guard rails of _handle_tts's elevenlabs branch
in-process via a fake handler. The happy path mocks urllib.urlopen so no real
ElevenLabs network call is made; the rejection paths (missing key, bad config
voice_id) bail before any network call.
"""
import io
import json
import pytest
import api.routes as routes
class _FakeHandler:
def __init__(self, body: bytes, command: str = "POST", headers=None, client="9.9.9.9"):
self.command = command
self.rfile = io.BytesIO(body)
self.wfile = io.BytesIO()
self.headers = headers or {}
self.headers.setdefault("Content-Length", str(len(body)))
self.client_address = (client, 12345)
self.status = None
self.sent_headers = {}
def send_response(self, status):
self.status = status
def send_header(self, key, value):
self.sent_headers[key] = value
def end_headers(self):
pass
def payload(self):
try:
return json.loads(self.wfile.getvalue().decode("utf-8"))
except Exception:
return None
def _post(body_dict, **kw):
return _FakeHandler(json.dumps(body_dict).encode(), **kw)
@pytest.fixture(autouse=True)
def _fresh(monkeypatch):
# Auth + limiter sit before the engine branch; pin them off/clean so these
# assertions are deterministic regardless of suite order.
import api.auth as _auth
monkeypatch.setattr(_auth, "is_auth_enabled", lambda: False)
monkeypatch.setattr(routes, "is_auth_enabled", lambda: False, raising=False)
monkeypatch.delenv("HERMES_WEBUI_TRUST_FORWARDED_FOR", raising=False)
if hasattr(routes._handle_tts, "_tts_limiter"):
del routes._handle_tts._tts_limiter
yield
if hasattr(routes._handle_tts, "_tts_limiter"):
del routes._handle_tts._tts_limiter
def test_elevenlabs_missing_key_returns_503(monkeypatch, tmp_path):
"""No ELEVENLABS_API_KEY (env or .env) → 503, no network call."""
monkeypatch.delenv("ELEVENLABS_API_KEY", raising=False)
# Point the .env fallback at an empty home so no key is found.
import api.profiles as profiles
monkeypatch.setattr(profiles, "get_active_hermes_home", lambda: tmp_path)
h = _post({"text": "hello", "engine": "elevenlabs"}, client="9.9.9.1")
routes._handle_tts(h, None)
assert h.status == 503
assert "ELEVENLABS_API_KEY" in (h.payload() or {}).get("error", "")
def test_elevenlabs_rejects_traversal_voice_id_in_config(monkeypatch, tmp_path):
"""A config voice_id that isn't a safe path segment → 400 before any call."""
monkeypatch.setenv("ELEVENLABS_API_KEY", "sk-test")
monkeypatch.setattr(routes, "get_config", lambda: {"tts": {"elevenlabs": {"voice_id": "../../etc/passwd"}}}, raising=False)
# If get_config isn't a routes-level name, patch api.config.
import api.config as _cfg
monkeypatch.setattr(_cfg, "get_config", lambda: {"tts": {"elevenlabs": {"voice_id": "../../etc/passwd"}}})
h = _post({"text": "hello", "engine": "elevenlabs"}, client="9.9.9.2")
routes._handle_tts(h, None)
assert h.status == 400
assert "voice_id" in (h.payload() or {}).get("error", "")
def test_elevenlabs_happy_path_streams_mp3(monkeypatch):
"""With a key + valid config, the branch calls ElevenLabs and returns audio/mpeg."""
monkeypatch.setenv("ELEVENLABS_API_KEY", "sk-test")
import api.config as _cfg
monkeypatch.setattr(_cfg, "get_config", lambda: {"tts": {"elevenlabs": {"voice_id": "pNInz6obpgDQGcFmaJgB", "model": "eleven_multilingual_v2"}}})
captured = {}
class _Resp:
def __init__(self):
self._chunks = [b"ID3fakeaudio", b""]
self._i = 0
def read(self, n=-1):
c = self._chunks[self._i] if self._i < len(self._chunks) else b""
self._i += 1
return c
def __enter__(self):
return self
def __exit__(self, *a):
return False
def _fake_urlopen(req, timeout=30):
captured["url"] = req.full_url
captured["xi_api_key"] = req.get_header("Xi-api-key")
captured["body"] = json.loads(req.data.decode("utf-8"))
return _Resp()
# The branch does `from urllib.request import ... urlopen as _urlopen` at
# call time, so patching the source module is what takes effect.
import urllib.request as _ur
monkeypatch.setattr(_ur, "urlopen", _fake_urlopen)
h = _post({"text": "hello world", "engine": "elevenlabs"}, client="9.9.9.3")
routes._handle_tts(h, None)
assert h.status == 200
assert h.sent_headers.get("Content-Type") == "audio/mpeg"
assert h.wfile.getvalue() == b"ID3fakeaudio"
# Confirmed it hit the configured voice + sent the key header + capped text.
assert "pNInz6obpgDQGcFmaJgB" in captured["url"]
assert captured["xi_api_key"] == "sk-test"
assert captured["body"]["text"] == "hello world"
def test_elevenlabs_overlong_text_rejected_before_engine(monkeypatch):
"""The shared 5000-char cap applies to the elevenlabs engine too (no call)."""
monkeypatch.setenv("ELEVENLABS_API_KEY", "sk-test")
h = _post({"text": "x" * 5001, "engine": "elevenlabs"}, client="9.9.9.4")
routes._handle_tts(h, None)
assert h.status == 400
assert "too long" in (h.payload() or {}).get("error", "")