feat(conversation): thinking on/off switch
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
e8c0df027e
commit
8ce8e791b2
@ -6,6 +6,9 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi
|
||||
---
|
||||
|
||||
## [Unreleased]
|
||||
### Added
|
||||
- Conversation Playground: a Thinking on/off switch next to the Language Model (default off, setting `conversation_thinking`). Off sends `chat_template_kwargs.enable_thinking=false` (plus `think:false` for Ollama) with each reply request and never touches the system prompt; the empty-reply error now says to turn Thinking off.
|
||||
|
||||
### Changed
|
||||
- Playwright specs now import `test` from `tests/_guard.js`, whose auto fixture aborts and fails any unmocked non-GET request to `/api/**` so a spec can no longer modify live settings. See `architecture/qa.md`.
|
||||
|
||||
|
||||
19
Manual.md
19
Manual.md
@ -842,3 +842,22 @@ not yet built.
|
||||
Ingested magazines are stored server-side, so any device on the network sees the
|
||||
same library and reading progress. Playback needs the server; there is no
|
||||
offline cache yet.
|
||||
|
||||
|
||||
## Conversation Playground: Thinking switch
|
||||
|
||||
**Purpose:** choose whether a reasoning model (e.g. Qwen3) thinks before replying. Off (default) gives faster answers; on gives better reasoning but is slower and some models then return an empty reply.
|
||||
|
||||
**Prerequisites:** an OpenAI-compatible Language Model endpoint selected in Engines & Prompt.
|
||||
|
||||
**Usage:** open Conversation, find the "Thinking on/off" switch next to the Language Model row, toggle it with click, tap, or Space. The state is saved on the server and restored on reload.
|
||||
|
||||
**Expected result:** with Thinking off, replies start sooner. The request asks the backend to disable thinking; your system prompt is never modified. Any `<think>` text is removed from the spoken reply.
|
||||
|
||||
**Errors:** "LLM returned empty response" means the model spent its answer on thinking: turn Thinking off or pick a non-thinking model. Backends that reject the disable parameter are retried without it.
|
||||
|
||||
**Mobile / accessibility:** 44 px touch target, role=switch with visible state text (on/off), keyboard operable, help text linked via aria-describedby; German and English labels.
|
||||
|
||||
**Offline:** works with a local LLM; no network beyond your configured endpoint.
|
||||
|
||||
**Privacy:** the setting is stored in the local app settings only; nothing extra is sent anywhere.
|
||||
|
||||
@ -45,7 +45,7 @@ _SETTINGS_KEYS = {
|
||||
# LLM so page images never accidentally go to a text-only model.
|
||||
"magazine_layout_url", "magazine_layout_model",
|
||||
"vision_ocr_url", "vision_ocr_model", "vision_ocr_enabled",
|
||||
"inline_tags_enabled",
|
||||
"inline_tags_enabled", "conversation_thinking",
|
||||
"vision_ocr_fallback_url", "vision_ocr_fallback_model",
|
||||
# OPDS catalogues to search for magazine issues: [{"name": ..., "url": ...}], no credentials.
|
||||
"magazine_opds_catalogs",
|
||||
@ -348,6 +348,7 @@ def _load_settings() -> dict:
|
||||
"vision_ocr_model": _VISION_OCR_MODEL_DEFAULT,
|
||||
"vision_ocr_enabled": True,
|
||||
"inline_tags_enabled": False,
|
||||
"conversation_thinking": False,
|
||||
"export_provenance_default": True,
|
||||
"vision_ocr_fallback_url": _VISION_OCR_FALLBACK_DEFAULT,
|
||||
"vision_ocr_fallback_model": _VISION_OCR_FALLBACK_MODEL_DEFAULT,
|
||||
|
||||
@ -373,6 +373,21 @@ def _reasoning_text(obj: dict) -> str:
|
||||
return obj.get("reasoning_content") or obj.get("reasoning") or ""
|
||||
|
||||
|
||||
def _thinking_off_params(settings: dict, llm_url: str = "") -> dict:
|
||||
"""Extra request-body keys that switch reasoning off, or {} when the Thinking switch is on.
|
||||
|
||||
Same mechanism the sheet-extraction retry uses (chat_template_kwargs for vLLM/llama-swap);
|
||||
Ollama's OpenAI-compatible endpoint additionally takes ``think: false``. Backends that reject
|
||||
the keys are retried without them by the caller.
|
||||
"""
|
||||
if settings.get("conversation_thinking"):
|
||||
return {}
|
||||
params: dict = {"chat_template_kwargs": {"enable_thinking": False}}
|
||||
if ":11434" in (llm_url or ""):
|
||||
params["think"] = False
|
||||
return params
|
||||
|
||||
|
||||
def _rewrite_with_persona_sync(text: str, persona: str, llm_url: str, model: str = "") -> str:
|
||||
"""Inline synchronous persona rewrite; raises RuntimeError on failure."""
|
||||
settings = _load_settings()
|
||||
@ -391,15 +406,18 @@ def _rewrite_with_persona_sync(text: str, persona: str, llm_url: str, model: str
|
||||
}
|
||||
if model:
|
||||
payload["model"] = model
|
||||
resp = requests.post(
|
||||
f"{llm_url.rstrip('/')}/chat/completions",
|
||||
json=payload,
|
||||
headers={"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"},
|
||||
timeout=60,
|
||||
)
|
||||
payload.update(_thinking_off_params(settings, llm_url))
|
||||
url = f"{llm_url.rstrip('/')}/chat/completions"
|
||||
hdrs = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"}
|
||||
resp = requests.post(url, json=payload, headers=hdrs, timeout=60)
|
||||
if resp.status_code in (400, 422) and "chat_template_kwargs" in payload:
|
||||
for k in ("chat_template_kwargs", "think"):
|
||||
payload.pop(k, None)
|
||||
resp = requests.post(url, json=payload, headers=hdrs, timeout=60)
|
||||
resp.raise_for_status()
|
||||
_msg = resp.json()["choices"][0]["message"]
|
||||
result = (_msg.get("content") or _reasoning_text(_msg) or "").strip()
|
||||
result = re.sub(r"<think>.*?</think>", "", result, flags=re.DOTALL).strip() or result
|
||||
if result.startswith('"') and result.endswith('"'):
|
||||
result = result[1:-1].strip()
|
||||
return result
|
||||
@ -6213,16 +6231,22 @@ async def conversation_turn(
|
||||
eff_llm_model = llm_model or settings.get("llm_model") or ""
|
||||
if eff_llm_model:
|
||||
llm_payload["model"] = eff_llm_model
|
||||
llm_payload.update(_thinking_off_params(settings, eff_llm_url))
|
||||
|
||||
_loop = asyncio.get_event_loop()
|
||||
_token_q: asyncio.Queue[tuple[str, str] | None] = asyncio.Queue()
|
||||
|
||||
def _llm_thread() -> None:
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{eff_llm_url}/chat/completions", json=llm_payload,
|
||||
headers={"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"}, stream=True, timeout=120,
|
||||
)
|
||||
_hdrs = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"}
|
||||
resp = requests.post(f"{eff_llm_url}/chat/completions", json=llm_payload,
|
||||
headers=_hdrs, stream=True, timeout=120)
|
||||
if resp.status_code in (400, 422) and "chat_template_kwargs" in llm_payload:
|
||||
# backend rejects the thinking switch: ask again without it
|
||||
llm_payload.pop("chat_template_kwargs", None)
|
||||
llm_payload.pop("think", None)
|
||||
resp = requests.post(f"{eff_llm_url}/chat/completions", json=llm_payload,
|
||||
headers=_hdrs, stream=True, timeout=120)
|
||||
resp.raise_for_status()
|
||||
for raw_line in resp.iter_lines():
|
||||
if not raw_line:
|
||||
@ -6403,8 +6427,8 @@ async def conversation_turn(
|
||||
if not llm_text.strip():
|
||||
yield sse({"type": "error", "stage": "llm",
|
||||
"message": "LLM returned empty response. "
|
||||
"If using a Qwen3 model, try adding /no-think to your system prompt "
|
||||
"or pick a non-thinking model in the Language Model dropdown."})
|
||||
"If using a thinking model (e.g. Qwen3), turn Thinking off next to the "
|
||||
"Language Model, or pick a non-thinking model in the dropdown."})
|
||||
return
|
||||
|
||||
# 3. Stream audio chunks in order — each chunk's TTS ran concurrently
|
||||
|
||||
@ -203,6 +203,19 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
if (_appSettings) _appSettings.conv_llm_url = llmUrlInp.value;
|
||||
});
|
||||
|
||||
// Thinking switch (server setting conversation_thinking; default off = faster answers)
|
||||
const thinkingSw = $('conv-thinking-switch');
|
||||
const thinkingState = $('conv-thinking-state');
|
||||
const showThinking = () => {
|
||||
if (thinkingState && thinkingSw) thinkingState.textContent = window.t ? window.t(thinkingSw.checked ? 'on' : 'off') : (thinkingSw.checked ? 'on' : 'off');
|
||||
};
|
||||
if (thinkingSw) {
|
||||
thinkingSw.checked = !!(_appSettings && _appSettings.conversation_thinking);
|
||||
showThinking();
|
||||
thinkingSw.addEventListener('thinking-sync', showThinking);
|
||||
thinkingSw.addEventListener('change', () => { showThinking(); _patchSettings({ conversation_thinking: thinkingSw.checked }); });
|
||||
}
|
||||
|
||||
let mediaRecorder = null;
|
||||
let recChunks = [];
|
||||
let recTimerInterval = null;
|
||||
|
||||
@ -85,6 +85,10 @@ const I18N_DICT = {
|
||||
'No candidate could be generated.': 'Es konnte kein Kandidat erzeugt werden.',
|
||||
'Speech-to-text unavailable: scored by level and audio problems only.': 'Sprache-zu-Text nicht verfügbar: Wertung nur nach Pegel und Audioproblemen.',
|
||||
'Auto-pick failed': 'Automatische Wahl fehlgeschlagen',
|
||||
// Conversation thinking switch
|
||||
'Thinking': 'Denken',
|
||||
'on': 'an', 'off': 'aus',
|
||||
'Off = faster answers. On = better reasoning but slower, and some models may return an empty reply.': 'Aus = schnellere Antworten. An = besseres Schlussfolgern, aber langsamer; manche Modelle liefern eine leere Antwort.',
|
||||
// Inline text tags
|
||||
'Inline text tags': 'Inline-Texttags',
|
||||
'Off by default. When on, tags shape the speech in Try It Out, Read Aloud, the speech API and audiobooks; streaming and magazine audio only remove them.': 'Standardmäßig aus. Wenn aktiv, steuern Tags die Sprache in „Ausprobieren“, „Vorlesen“, der Sprach-API und Hörbüchern; Streaming und Magazin-Audio entfernen sie nur.',
|
||||
|
||||
@ -585,6 +585,8 @@ async function loadSettings() {
|
||||
inlineTags.dataset.wired = '1';
|
||||
inlineTags.addEventListener('change', () => _patchSettings({ inline_tags_enabled: inlineTags.checked }));
|
||||
}
|
||||
const convThinking = $('conv-thinking-switch');
|
||||
if (convThinking) { convThinking.checked = !!s.conversation_thinking; convThinking.dispatchEvent(new Event('thinking-sync')); }
|
||||
const extKeyReq = $('s-external-api-key-required');
|
||||
if (extKeyReq) extKeyReq.checked = !!s.external_api_key_required;
|
||||
if (extKeyReq && !extKeyReq.dataset.wired) {
|
||||
|
||||
@ -22,6 +22,11 @@
|
||||
<input id="conv-llm-url" class="conv-url-inp" type="text" placeholder="http://localhost:11434/v1" spellcheck="false">
|
||||
<button class="btn-secondary" id="conv-llm-fetch-btn" title="Fetch models"><span class="mdi mdi-refresh"></span></button>
|
||||
<select id="conv-llm-model-select" aria-label="Language model"><option value="">— fetch models —</option></select>
|
||||
<label class="conv-thinking" title="Off = faster answers. On = better reasoning but slower, and some models may return an empty reply.">
|
||||
<input type="checkbox" role="switch" id="conv-thinking-switch" aria-describedby="conv-thinking-help">
|
||||
<span>Thinking</span> <span id="conv-thinking-state" class="conv-thinking-state">off</span>
|
||||
</label>
|
||||
<span id="conv-thinking-help" class="sr-only">Off = faster answers. On = better reasoning but slower, and some models may return an empty reply.</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="engine-setup-col" style="flex:1.5; border-right:none; padding-right:0">
|
||||
|
||||
@ -6031,3 +6031,8 @@ html[data-mag-night] #toast.success { color: var(--green); } html[data-mag-night
|
||||
#open-sources-card .getvoices-toolbar { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }
|
||||
#open-sources-card button, #open-sources-card input, #open-sources-card select { min-height: 44px; }
|
||||
#open-sources-card .voice-source-actions a { display: inline-flex; align-items: center; min-height: 44px; }
|
||||
|
||||
.conv-thinking { display:inline-flex; align-items:center; gap:6px; min-height:44px; min-width:44px; padding:0 8px; cursor:pointer; white-space:nowrap; }
|
||||
.conv-thinking input[type="checkbox"] { width:20px; height:20px; accent-color:var(--accent); cursor:pointer; }
|
||||
.conv-thinking:focus-within { outline:2px solid var(--accent); outline-offset:2px; }
|
||||
.conv-thinking-state { font-weight:600; }
|
||||
|
||||
51
tests/conversation-thinking.spec.js
Normal file
51
tests/conversation-thinking.spec.js
Normal file
@ -0,0 +1,51 @@
|
||||
// @ts-check
|
||||
// Conversation Playground "Thinking" switch (all /api routes mocked).
|
||||
const { test, expect } = require('./_guard');
|
||||
|
||||
async function boot(page, settings, lang) {
|
||||
/** @type {any[]} */ const patches = [];
|
||||
await page.route('**/api/**', route => route.request().method() === 'GET' ? route.fulfill({ json: {} }) : route.fulfill({ json: { ok: true } }));
|
||||
await page.route('**/api/settings', route => {
|
||||
if (route.request().method() === 'GET') return route.fulfill({ json: settings });
|
||||
patches.push(route.request().postDataJSON());
|
||||
return route.fulfill({ json: { ok: true } });
|
||||
});
|
||||
await page.route('**/static/dist/main.min.js*', r => r.abort());
|
||||
if (lang) await page.addInitScript(l => { try { localStorage.setItem('app-lang', l); } catch (e) {} }, lang);
|
||||
await page.goto('/', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForFunction(() => typeof window.navTo === 'function', null, { timeout: 15_000 });
|
||||
await page.waitForSelector('#app-skeleton', { state: 'detached', timeout: 15_000 });
|
||||
await page.evaluate(() => window.navTo('s-conversation'));
|
||||
return patches;
|
||||
}
|
||||
|
||||
test('switch is labelled, keyboard operable, 44px, saves the setting', async ({ page }) => {
|
||||
const patches = await boot(page, { conversation_thinking: false });
|
||||
const sw = page.getByRole('switch', { name: /Thinking/ });
|
||||
await expect(sw).not.toBeChecked();
|
||||
await expect(page.locator('#conv-thinking-state')).toHaveText('off');
|
||||
const box = await page.locator('label.conv-thinking').boundingBox();
|
||||
expect(box && box.height).toBeGreaterThanOrEqual(44);
|
||||
await sw.focus();
|
||||
await page.keyboard.press('Space');
|
||||
await expect(sw).toBeChecked();
|
||||
await expect(page.locator('#conv-thinking-state')).toHaveText('on');
|
||||
await expect.poll(() => patches.some(p => p && p.conversation_thinking === true)).toBe(true);
|
||||
});
|
||||
|
||||
test('restores saved state', async ({ page }) => {
|
||||
await boot(page, { conversation_thinking: true });
|
||||
await expect(page.getByRole('switch', { name: /Thinking/ })).toBeChecked();
|
||||
});
|
||||
|
||||
test('help text is linked to the switch (request body is covered by the unit tests; the server reads the setting)', async ({ page }) => {
|
||||
await boot(page, { conversation_thinking: false });
|
||||
await expect(page.locator('#conv-thinking-switch')).toHaveAttribute('aria-describedby', 'conv-thinking-help');
|
||||
await expect(page.locator('#conv-thinking-help')).toContainText('faster answers');
|
||||
});
|
||||
|
||||
test('German labels', async ({ page }) => {
|
||||
await boot(page, { conversation_thinking: false }, 'de');
|
||||
await expect(page.locator('label.conv-thinking')).toContainText('Denken');
|
||||
await expect(page.locator('#conv-thinking-state')).toHaveText('aus');
|
||||
});
|
||||
91
tests/test_conversation_thinking.py
Normal file
91
tests/test_conversation_thinking.py
Normal file
@ -0,0 +1,91 @@
|
||||
"""Thinking switch: the LLM request body carries the thinking-off parameter only when off."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-think-test-config-"))
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import routes.conversation as conversation
|
||||
|
||||
|
||||
class _Resp:
|
||||
status_code = 200
|
||||
|
||||
def __init__(self, lines=(), body=None):
|
||||
self._lines, self._body = lines, body or {}
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def iter_lines(self):
|
||||
return iter(self._lines)
|
||||
|
||||
def json(self):
|
||||
return self._body
|
||||
|
||||
|
||||
class ThinkingParams(unittest.TestCase):
|
||||
def test_off_by_default_openai_compatible(self):
|
||||
self.assertEqual(conversation._thinking_off_params({}, "http://h:17861/v1"),
|
||||
{"chat_template_kwargs": {"enable_thinking": False}})
|
||||
|
||||
def test_off_ollama_adds_think_false(self):
|
||||
p = conversation._thinking_off_params({"conversation_thinking": False}, "http://h:11434/v1")
|
||||
self.assertIs(p["think"], False)
|
||||
self.assertIn("chat_template_kwargs", p)
|
||||
|
||||
def test_on_sends_nothing(self):
|
||||
for url in ("http://h:17861/v1", "http://h:11434/v1"):
|
||||
self.assertEqual(conversation._thinking_off_params({"conversation_thinking": True}, url), {})
|
||||
|
||||
|
||||
class TurnRequest(unittest.TestCase):
|
||||
def _post_turn(self, setting, url="http://h:17861/v1"):
|
||||
app = FastAPI()
|
||||
app.include_router(conversation.router)
|
||||
sent = []
|
||||
|
||||
def fake_post(u, json=None, **kw):
|
||||
sent.append(json)
|
||||
return _Resp(lines=[b"data: [DONE]"]) # empty reply -> error event, no TTS
|
||||
|
||||
settings = {"llm_url": url, "conversation_thinking": setting}
|
||||
with mock.patch.object(conversation, "_load_settings", return_value=settings), \
|
||||
mock.patch.object(conversation.requests, "post", side_effect=fake_post):
|
||||
r = TestClient(app).post("/api/conversation/turn", data={"text": "Please tell me a short story about the sea"})
|
||||
return sent, r.text
|
||||
|
||||
def test_stream_off_sends_param(self):
|
||||
sent, body = self._post_turn(False)
|
||||
self.assertEqual(sent[0]["chat_template_kwargs"], {"enable_thinking": False})
|
||||
self.assertIn("turn Thinking off", body)
|
||||
|
||||
def test_stream_on_omits_param(self):
|
||||
sent, _ = self._post_turn(True)
|
||||
self.assertNotIn("chat_template_kwargs", sent[0])
|
||||
self.assertNotIn("think", sent[0])
|
||||
|
||||
def test_stream_ollama_off(self):
|
||||
sent, _ = self._post_turn(False, "http://h:11434/v1")
|
||||
self.assertIs(sent[0]["think"], False)
|
||||
|
||||
def test_persona_rewrite_off_on_and_strips_think(self):
|
||||
body = {"choices": [{"message": {"content": "<think>x</think>Aye"}}]}
|
||||
for setting, expect in ((False, True), (True, False)):
|
||||
sent = []
|
||||
with mock.patch.object(conversation, "_load_settings", return_value={"conversation_thinking": setting}), \
|
||||
mock.patch.object(conversation.requests, "post",
|
||||
side_effect=lambda u, json=None, **k: (sent.append(json), _Resp(body=body))[1]):
|
||||
out = conversation._rewrite_with_persona_sync("hi", "pirate", "http://h/v1")
|
||||
self.assertEqual(out, "Aye")
|
||||
self.assertEqual("chat_template_kwargs" in sent[0], expect)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue
Block a user