From ec476c78234edae60cc0068e911efddddefa9cf6 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Tue, 18 Aug 2026 15:30:22 +0200 Subject: [PATCH] v6.10: Auto-segment long text to fix mid-sentence truncation A single generate call was capped at --max-seq-len codec tokens, so text long enough to need more audio than that got cut off wherever generation happened to be. Long input is now split into paragraph/sentence-sized segments and generated back-to-back, so --max-seq-len can be sized for VRAM budget instead of the longest expected input. Co-Authored-By: Claude Sonnet 5 --- README.md | 6 ++ patches/openai_server.patch | 180 ++++++++++++++++++++++++++---------- 2 files changed, 137 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 0fdbc50..4c7c055 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,12 @@ The first request after container startup can be slower because CUDA graph captu ## Changelog +### v6.10 — 2026-08-18 +**Fix: Long Text No Longer Truncated Mid-Sentence** +- **Root Cause:** `/v1/audio/speech` sent the entire request text as one generate call, capped at `--max-seq-len` codec tokens. Text long enough to need more audio than that cap allowed got cut off wherever generation happened to be — mid-sentence, mid-word — with no error surfaced to the client. +- **Automatic Segmentation:** Long input is now split into paragraph-sized segments (falling back to sentence boundaries for a paragraph that alone exceeds the safe budget) and generated back-to-back, stitched into one continuous stream or file. Applies to both the streaming (wav/pcm) and non-streaming (mp3/zip) paths. +- **Why This Beats Raising `--max-seq-len`:** The CUDA graph static cache sized by `--max-seq-len` is reserved VRAM for the life of the container, whether or not any request needs it. Segmenting text keeps that cache small — `--max-seq-len` can be set to fit your VRAM budget instead of your longest expected input — while still supporting arbitrarily long text. + ### v6.9 — 2026-07-29 **Feature: Per-Line Emotional Direction on Cloned Voices** - **Request-Level `instruct` for VoiceClone:** The clone server previously read `instruct` only from `voices.json`, so a character could have one fixed delivery for an entire book. `SpeechRequest` now accepts `instruct`, applied to both the streaming and non-streaming paths. diff --git a/patches/openai_server.patch b/patches/openai_server.patch index 8c6440f..9e8905b 100644 --- a/patches/openai_server.patch +++ b/patches/openai_server.patch @@ -1,8 +1,8 @@ diff --git a/examples/openai_server.py b/examples/openai_server.py -index 2199e14..a65f81e 100644 +index 2199e14..05151ce 100644 --- a/examples/openai_server.py +++ b/examples/openai_server.py -@@ -36,6 +36,7 @@ API usage: +@@ -36,11 +36,13 @@ API usage: """ import argparse import asyncio @@ -10,7 +10,13 @@ index 2199e14..a65f81e 100644 import io import json import logging -@@ -66,9 +67,66 @@ app = FastAPI(title="faster-qwen3-tts OpenAI-compatible API") + import os + import queue ++import re + import struct + import sys + import threading +@@ -66,9 +68,67 @@ app = FastAPI(title="faster-qwen3-tts OpenAI-compatible API") tts_model = None voices: dict = {} @@ -18,6 +24,7 @@ index 2199e14..a65f81e 100644 +last_voices_mtime: float = 0.0 default_voice: Optional[str] = None SAMPLE_RATE = 24000 # updated once the model loads ++MAX_SEQ_LEN = 4096 # CUDA graph static cache size, updated from --max-seq-len _model_lock = threading.Lock() # prevent concurrent GPU inference +aligner_model = None + @@ -77,7 +84,7 @@ index 2199e14..a65f81e 100644 # --------------------------------------------------------------------------- # Request / response models -@@ -79,8 +137,9 @@ class SpeechRequest(BaseModel): +@@ -79,8 +139,60 @@ class SpeechRequest(BaseModel): model: str = "tts-1" input: str voice: str = "alloy" @@ -86,10 +93,61 @@ index 2199e14..a65f81e 100644 + response_format: str = "wav" # wav | pcm | mp3 | zip + speed: float = 1.0 # scales audio tempo + instruct: Optional[str] = None # per-line delivery direction, see _merge_instruct ++ ++ ++# --------------------------------------------------------------------------- ++# Text chunking for long input ++# ++# A single generate call is capped at MAX_SEQ_LEN codec tokens (the CUDA ++# graph's static cache size). Rather than raising that cache to cover the ++# longest text anyone might send — which burns VRAM permanently for every ++# request — long input is split into paragraph/sentence-sized segments that ++# each fit comfortably under the cap, generated back-to-back, and stitched ++# into one continuous stream/file. ++# --------------------------------------------------------------------------- ++ ++_CODEC_HZ = 12 # talker codec frame rate (generated audio tokens/sec) ++_CHARS_PER_SEC = 15.0 # conservative estimate of spoken characters/sec ++_SEQ_LEN_SAFETY = 0.6 # stay well under the hard cap to leave room for slower speech ++_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+") ++ ++ ++def _max_chars_per_segment() -> int: ++ budget_seconds = (MAX_SEQ_LEN * _SEQ_LEN_SAFETY) / _CODEC_HZ ++ return max(200, int(budget_seconds * _CHARS_PER_SEC)) ++ ++ ++def _split_text_for_generation(text: str, max_chars: int) -> list: ++ """Split text into segments that each fit under max_chars. ++ ++ Splits on paragraph breaks first (kept as separate segments so natural ++ pauses land there), then falls back to sentence boundaries for any ++ paragraph that alone exceeds the budget. ++ """ ++ paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()] ++ if not paragraphs: ++ return [text] ++ ++ segments = [] ++ for para in paragraphs: ++ if len(para) <= max_chars: ++ segments.append(para) ++ continue ++ buf = "" ++ for sent in _SENTENCE_SPLIT_RE.split(para): ++ candidate = f"{buf} {sent}".strip() if buf else sent ++ if len(candidate) > max_chars and buf: ++ segments.append(buf) ++ buf = sent ++ else: ++ buf = candidate ++ if buf: ++ segments.append(buf) ++ return segments or [text] # --------------------------------------------------------------------------- -@@ -145,6 +204,21 @@ def _to_mp3_bytes(pcm: np.ndarray, sample_rate: int) -> bytes: +@@ -145,6 +257,21 @@ def _to_mp3_bytes(pcm: np.ndarray, sample_rate: int) -> bytes: def resolve_voice(voice_name: str) -> dict: """Return voice config dict or fall back to default, else raise 400.""" @@ -111,7 +169,7 @@ index 2199e14..a65f81e 100644 if voice_name in voices: return voices[voice_name] if default_voice and default_voice in voices: -@@ -168,7 +242,47 @@ def resolve_voice(voice_name: str) -> dict: +@@ -168,7 +295,47 @@ def resolve_voice(voice_name: str) -> dict: # --------------------------------------------------------------------------- @@ -160,7 +218,7 @@ index 2199e14..a65f81e 100644 """ Run generate_voice_clone_streaming in a background thread and yield raw PCM bytes for each chunk as they arrive. -@@ -177,21 +291,63 @@ async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, No +@@ -177,21 +344,65 @@ async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, No _DONE = object() def producer(): @@ -191,29 +249,37 @@ index 2199e14..a65f81e 100644 + try: with _model_lock: -+ _seed_rng(voice_cfg.get("seed", _voice_seed(voice_name))) - for chunk, _sr, _timing in tts_model.generate_voice_clone_streaming( - text=text, - language=voice_cfg.get("language", "Auto"), +- for chunk, _sr, _timing in tts_model.generate_voice_clone_streaming( +- text=text, +- language=voice_cfg.get("language", "Auto"), - ref_audio=voice_cfg["ref_audio"], -+ ref_audio=voice_cfg.get("ref_audio"), - ref_text=voice_cfg.get("ref_text", ""), - chunk_size=voice_cfg.get("chunk_size", 12), +- ref_text=voice_cfg.get("ref_text", ""), +- chunk_size=voice_cfg.get("chunk_size", 12), - non_streaming_mode=False, -+ instruct=_merge_instruct(voice_cfg, instruct), -+ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, voice_name, tts_model), -+ non_streaming_mode=True, -+ temperature=voice_cfg.get("temperature", 0.8), -+ top_k=voice_cfg.get("top_k", 50), -+ top_p=voice_cfg.get("top_p", 0.9), - ): +- ): - q.put(chunk) -+ raw = _to_pcm16(chunk) -+ if process: -+ process.stdin.write(raw) -+ process.stdin.flush() -+ else: -+ q.put(raw) ++ _seed_rng(voice_cfg.get("seed", _voice_seed(voice_name))) ++ segments = _split_text_for_generation(text, _max_chars_per_segment()) ++ for seg in segments: ++ for chunk, _sr, _timing in tts_model.generate_voice_clone_streaming( ++ text=seg, ++ language=voice_cfg.get("language", "Auto"), ++ ref_audio=voice_cfg.get("ref_audio"), ++ ref_text=voice_cfg.get("ref_text", ""), ++ chunk_size=voice_cfg.get("chunk_size", 12), ++ instruct=_merge_instruct(voice_cfg, instruct), ++ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, voice_name, tts_model), ++ non_streaming_mode=True, ++ temperature=voice_cfg.get("temperature", 0.8), ++ top_k=voice_cfg.get("top_k", 50), ++ top_p=voice_cfg.get("top_p", 0.9), ++ ): ++ raw = _to_pcm16(chunk) ++ if process: ++ process.stdin.write(raw) ++ process.stdin.flush() ++ else: ++ q.put(raw) except Exception as exc: q.put(exc) finally: @@ -228,7 +294,7 @@ index 2199e14..a65f81e 100644 thread = threading.Thread(target=producer, daemon=True) thread.start() -@@ -203,7 +359,7 @@ async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, No +@@ -203,7 +414,7 @@ async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, No break if isinstance(item, Exception): raise item @@ -237,7 +303,7 @@ index 2199e14..a65f81e 100644 # --------------------------------------------------------------------------- -@@ -220,7 +376,8 @@ async def health(): +@@ -220,7 +431,8 @@ async def health(): async def create_speech(req: SpeechRequest): if tts_model is None: raise HTTPException(status_code=503, detail="Model not loaded") @@ -247,7 +313,7 @@ index 2199e14..a65f81e 100644 raise HTTPException(status_code=400, detail="'input' text is empty") voice_cfg = resolve_voice(req.voice) -@@ -230,36 +387,77 @@ async def create_speech(req: SpeechRequest): +@@ -230,36 +442,84 @@ async def create_speech(req: SpeechRequest): "wav": "audio/wav", "pcm": "audio/pcm", "mp3": "audio/mpeg", @@ -269,23 +335,38 @@ index 2199e14..a65f81e 100644 def _generate(): with _model_lock: -+ _seed_rng(voice_cfg.get("seed", _voice_seed(req.voice))) - return tts_model.generate_voice_clone( - text=req.input, - language=voice_cfg.get("language", "Auto"), +- return tts_model.generate_voice_clone( +- text=req.input, +- language=voice_cfg.get("language", "Auto"), - ref_audio=voice_cfg["ref_audio"], -+ ref_audio=voice_cfg.get("ref_audio"), - ref_text=voice_cfg.get("ref_text", ""), -+ instruct=_merge_instruct(voice_cfg, req.instruct), -+ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, req.voice, tts_model), -+ non_streaming_mode=True, -+ temperature=voice_cfg.get("temperature", 0.8), -+ top_k=voice_cfg.get("top_k", 50), -+ top_p=voice_cfg.get("top_p", 0.9), - ) - - audio_arrays, sr = await loop.run_in_executor(None, _generate) - audio = audio_arrays[0] if audio_arrays else np.zeros(1, dtype=np.float32) +- ref_text=voice_cfg.get("ref_text", ""), +- ) +- +- audio_arrays, sr = await loop.run_in_executor(None, _generate) +- audio = audio_arrays[0] if audio_arrays else np.zeros(1, dtype=np.float32) ++ _seed_rng(voice_cfg.get("seed", _voice_seed(req.voice))) ++ segments = _split_text_for_generation(req.input, _max_chars_per_segment()) ++ pieces = [] ++ sr = SAMPLE_RATE ++ for seg in segments: ++ audio_arrays, sr = tts_model.generate_voice_clone( ++ text=seg, ++ language=voice_cfg.get("language", "Auto"), ++ ref_audio=voice_cfg.get("ref_audio"), ++ ref_text=voice_cfg.get("ref_text", ""), ++ instruct=_merge_instruct(voice_cfg, req.instruct), ++ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, req.voice, tts_model), ++ non_streaming_mode=True, ++ temperature=voice_cfg.get("temperature", 0.8), ++ top_k=voice_cfg.get("top_k", 50), ++ top_p=voice_cfg.get("top_p", 0.9), ++ ) ++ if audio_arrays: ++ pieces.append(audio_arrays[0]) ++ return pieces, sr ++ ++ pieces, sr = await loop.run_in_executor(None, _generate) ++ audio = np.concatenate(pieces) if pieces else np.zeros(1, dtype=np.float32) + + if req.speed != 1.0: + import subprocess @@ -330,7 +411,7 @@ index 2199e14..a65f81e 100644 yield raw_chunk return StreamingResponse(audio_stream(), media_type=content_type) -@@ -306,16 +504,20 @@ def _parse_args(): +@@ -306,16 +566,21 @@ def _parse_args(): p.add_argument("--host", default="0.0.0.0", help="Bind host (default: 0.0.0.0)") p.add_argument("--port", type=int, default=8000, help="Bind port (default: 8000)") p.add_argument("--device", default="cuda", help="Torch device (default: cuda)") @@ -340,9 +421,10 @@ index 2199e14..a65f81e 100644 def main(): - global tts_model, voices, default_voice, SAMPLE_RATE -+ global tts_model, voices, voices_file_path, last_voices_mtime, default_voice, SAMPLE_RATE ++ global tts_model, voices, voices_file_path, last_voices_mtime, default_voice, SAMPLE_RATE, MAX_SEQ_LEN args = _parse_args() ++ MAX_SEQ_LEN = args.max_seq_len # Build voice registry if args.voices: @@ -352,7 +434,7 @@ index 2199e14..a65f81e 100644 with open(args.voices) as f: voices = json.load(f) default_voice = next(iter(voices)) -@@ -344,6 +546,7 @@ def main(): +@@ -344,6 +609,7 @@ def main(): args.model, device=args.device, dtype=torch.bfloat16,