feat: dedicated wakeword models for cancel and send actions (v2.03.30)
WakewordActionListener opens a second Wyoming connection during active wakeword recording, listening for the configured cancel/send models. When either fires it immediately calls cancel_dictation() or finish_dictation() without any silence timer or Whisper pass. Settings UI adds Cancel model and Send model pickers to the wakeword config card, populated from the same server model list as the trigger model. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
a77b15ccaf
commit
5aafb572ed
@ -9,6 +9,18 @@ The version is defined in [`blitztext/__init__.py`](blitztext/__init__.py).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.03.30] - 2026-06-10
|
||||
|
||||
### Added
|
||||
- **Dedicated wakeword models for cancel and send.** Two new optional fields in
|
||||
the wakeword settings card — "Cancel model" and "Send model" — let you assign
|
||||
a specific wakeword model (e.g. a custom "stop" or "send it" ONNX model) to
|
||||
each action. When configured, a `WakewordActionListener` opens a second
|
||||
Wyoming connection during recording and fires the action the instant the model
|
||||
triggers — no Whisper pass, no silence timer. The Whisper-based cancel watcher
|
||||
from v2.03.29 remains as a fallback when no cancel wakeword model is set.
|
||||
Model dropdowns are populated from the same server fetch as the trigger model.
|
||||
|
||||
## [2.03.29] - 2026-06-10
|
||||
|
||||
### Changed
|
||||
|
||||
@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host
|
||||
(not in a container) so it can type into any application via xdotool.
|
||||
"""
|
||||
|
||||
__version__ = "2.03.29"
|
||||
__version__ = "2.03.30"
|
||||
|
||||
@ -110,6 +110,8 @@ class Config:
|
||||
wakeword_sound_detected: str = "" # WAV played when the wakeword fires (speak now)
|
||||
wakeword_sound_done: str = "" # WAV played when the command is captured
|
||||
wakeword_silence_seconds: float = 2.0 # auto-stop after this much trailing silence
|
||||
wakeword_cancel_model: str = "" # wakeword model that cancels an in-progress recording
|
||||
wakeword_send_model: str = "" # wakeword model that finishes + sends (Enter) a recording
|
||||
# Text-to-speech for the wakeword benchmark — its own OpenAI-compatible
|
||||
# endpoint (Kokoro, XTTS, OpenAI, …): base URL incl. /v1, an optional bearer
|
||||
# key env var, a model id, and the voices to cycle through.
|
||||
@ -239,6 +241,8 @@ def load(path: Path = CONFIG_PATH) -> Config:
|
||||
wakeword_sound_detected=ww.get("sound_detected", ""),
|
||||
wakeword_sound_done=ww.get("sound_done", ""),
|
||||
wakeword_silence_seconds=float(ww.get("silence_seconds", 2.0)),
|
||||
wakeword_cancel_model=ww.get("cancel_model", ""),
|
||||
wakeword_send_model=ww.get("send_model", ""),
|
||||
tts_url=tts.get("url", "").rstrip("/"),
|
||||
tts_api_key_env=tts.get("api_key_env", ""),
|
||||
tts_model=tts.get("model", ""),
|
||||
@ -384,6 +388,8 @@ def save(cfg: Config, path: Path = CONFIG_PATH) -> None:
|
||||
"sound_detected": cfg.wakeword_sound_detected,
|
||||
"sound_done": cfg.wakeword_sound_done,
|
||||
"silence_seconds": cfg.wakeword_silence_seconds,
|
||||
"cancel_model": cfg.wakeword_cancel_model,
|
||||
"send_model": cfg.wakeword_send_model,
|
||||
},
|
||||
"tts": {
|
||||
"url": cfg.tts_url,
|
||||
|
||||
@ -315,11 +315,31 @@ class Daemon:
|
||||
elif self.countdown_cb:
|
||||
self.countdown_cb(None, silence)
|
||||
|
||||
# Real-time cancel keyword watcher: accumulate PCM from the VAD meter
|
||||
# and run quick transcription checks so cancel fires immediately rather
|
||||
# than waiting for the full silence timeout + normal transcription pass.
|
||||
# Wakeword-based action listener: if cancel/send wakeword models are
|
||||
# configured, open a second Wyoming connection during recording so a
|
||||
# dedicated wakeword ("stop", "send it") fires instantly — no Whisper pass.
|
||||
self._action_listener = None
|
||||
if self.cfg.wakeword_enabled and (
|
||||
self.cfg.wakeword_cancel_model or self.cfg.wakeword_send_model):
|
||||
from .wakeword import WakewordActionListener
|
||||
cbs: dict = {}
|
||||
if self.cfg.wakeword_cancel_model:
|
||||
cbs[self.cfg.wakeword_cancel_model] = lambda: GLib.idle_add(
|
||||
self.cancel_dictation)
|
||||
if self.cfg.wakeword_send_model:
|
||||
cbs[self.cfg.wakeword_send_model] = lambda: GLib.idle_add(
|
||||
lambda: self.finish_dictation(send_enter=True))
|
||||
self._action_listener = WakewordActionListener(
|
||||
uri=self.cfg.wakeword_uri, model_callbacks=cbs, mic=self.cfg.mic)
|
||||
self._action_listener.start()
|
||||
|
||||
# Whisper-based cancel watcher: fallback when no cancel wakeword model is
|
||||
# set, or as belt-and-suspenders for the spoken cancel keyword list.
|
||||
on_chunk = None
|
||||
if self.cfg.cancel_keywords and getattr(self, "transcriber", None) is not None:
|
||||
use_whisper_watcher = (self.cfg.cancel_keywords
|
||||
and getattr(self, "transcriber", None) is not None
|
||||
and not self.cfg.wakeword_cancel_model)
|
||||
if use_whisper_watcher:
|
||||
self._cancel_watcher = _CancelWatcher(
|
||||
self.transcriber,
|
||||
self.cfg.cancel_keywords,
|
||||
@ -356,6 +376,9 @@ class Daemon:
|
||||
if getattr(self, "_cancel_watcher", None) is not None:
|
||||
self._cancel_watcher.stop()
|
||||
self._cancel_watcher = None
|
||||
if getattr(self, "_action_listener", None) is not None:
|
||||
self._action_listener.stop()
|
||||
self._action_listener = None
|
||||
|
||||
def _ov_meter_start(self) -> None:
|
||||
"""A level meter purely to drive the overlay waveform in streaming mode.
|
||||
|
||||
@ -1823,6 +1823,10 @@ notebook.bt-nb tab:checked label {
|
||||
self.ww_uri.connect("focus-out-event", self._on_ww_uri_leave)
|
||||
self.ww_model = _labeled(ww_cfg_card, "Model name", _model_combo("Search models…"), width=LW,
|
||||
tooltip="Which wake model to listen for (e.g. okay_computer, hey_jarvis). Press ⟳ on the URI field to load models from the server.")
|
||||
self.ww_cancel_model = _labeled(ww_cfg_card, "Cancel model", _model_combo("none — use text keywords"), width=LW,
|
||||
tooltip="Optional: a wakeword model that immediately cancels the recording when heard (e.g. a 'stop' model). Faster than the Whisper-based text cancel.")
|
||||
self.ww_send_model = _labeled(ww_cfg_card, "Send model", _model_combo("none — use text keywords"), width=LW,
|
||||
tooltip="Optional: a wakeword model that finishes recording and presses Enter (e.g. a 'send it' model).")
|
||||
self.ww_combo.connect("changed", self._ww_changed)
|
||||
|
||||
self.ww_mic_level = Gtk.LevelBar()
|
||||
@ -2052,6 +2056,10 @@ notebook.bt-nb tab:checked label {
|
||||
self.ww_name.set_text(e.name)
|
||||
self.ww_uri.set_text(e.uri)
|
||||
_fill_combo(self.ww_model, [], e.model)
|
||||
if hasattr(self, "ww_cancel_model"):
|
||||
_fill_combo(self.ww_cancel_model, [], self.cfg.wakeword_cancel_model)
|
||||
if hasattr(self, "ww_send_model"):
|
||||
_fill_combo(self.ww_send_model, [], self.cfg.wakeword_send_model)
|
||||
self._probe_dot(self.ww_dot, e.uri, 10400)
|
||||
self._ww_fetch_models()
|
||||
|
||||
@ -2151,6 +2159,14 @@ notebook.bt-nb tab:checked label {
|
||||
if models:
|
||||
cur = _combo_text(self.ww_model)
|
||||
_fill_combo(self.ww_model, models, cur or models[0])
|
||||
# Also populate cancel/send model pickers with the same list.
|
||||
# Preserve whatever the user already typed; empty stays empty.
|
||||
if hasattr(self, "ww_cancel_model"):
|
||||
cur_c = _combo_text(self.ww_cancel_model)
|
||||
_fill_combo(self.ww_cancel_model, models, cur_c)
|
||||
if hasattr(self, "ww_send_model"):
|
||||
cur_s = _combo_text(self.ww_send_model)
|
||||
_fill_combo(self.ww_send_model, models, cur_s)
|
||||
if hasattr(self, "ww_status"):
|
||||
self.ww_status.set_markup(
|
||||
f'<span foreground="#4a8" size="small">'
|
||||
@ -3083,6 +3099,8 @@ notebook.bt-nb tab:checked label {
|
||||
c.wakeword_sound_detected = self.ww_snd_detected.get_filename() or ""
|
||||
c.wakeword_sound_done = self.ww_snd_done.get_filename() or ""
|
||||
c.wakeword_silence_seconds = float(self.ww_silence.get_text())
|
||||
c.wakeword_cancel_model = _combo_text(self.ww_cancel_model) if hasattr(self, "ww_cancel_model") else ""
|
||||
c.wakeword_send_model = _combo_text(self.ww_send_model) if hasattr(self, "ww_send_model") else ""
|
||||
c.cancel_keywords = [k.strip() for k in self.cancel_keywords.get_text().split(",") if k.strip()]
|
||||
c.send_keywords = [k.strip() for k in self.send_keywords.get_text().split(",") if k.strip()]
|
||||
if hasattr(self, "ww_key_cancel"):
|
||||
|
||||
@ -158,11 +158,141 @@ class WakewordListener:
|
||||
def _handle_detection(self):
|
||||
if time.time() < self._cooldown_until:
|
||||
return
|
||||
|
||||
|
||||
if is_muted():
|
||||
logbuffer.log("[wakeword] Detected, but paused (resume via tray)")
|
||||
return
|
||||
|
||||
|
||||
logbuffer.log(f"[wakeword] Detected '{self.model}'!")
|
||||
self._cooldown_until = time.time() + 3.0 # 3s cooldown
|
||||
self.on_detect()
|
||||
|
||||
|
||||
class WakewordActionListener:
|
||||
"""Listens for multiple wakeword models simultaneously and calls per-model callbacks.
|
||||
|
||||
Used during active wakeword recording so that dedicated "cancel" and "send"
|
||||
wakeword phrases trigger :meth:`~blitztext.daemon.Daemon.cancel_dictation` or
|
||||
:meth:`~blitztext.daemon.Daemon.finish_dictation` immediately — much faster
|
||||
than waiting for Whisper to transcribe the whole clip.
|
||||
|
||||
``model_callbacks`` is a ``{model_name: callable}`` dict; only the models
|
||||
present in the dict are requested from the server. No cooldown is applied
|
||||
because the listener is torn down immediately after the first action fires.
|
||||
"""
|
||||
|
||||
def __init__(self, uri: str, model_callbacks: dict, mic: str):
|
||||
self.uri = uri
|
||||
self.model_callbacks = dict(model_callbacks) # {name: callable}
|
||||
self.mic = mic
|
||||
self._stop_event = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if not self.model_callbacks:
|
||||
return
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, daemon=True, name="WakewordActionListener")
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=2.0)
|
||||
self._thread = None
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self._stream()
|
||||
except Exception as e:
|
||||
logbuffer.log(f"[wakeword-action] Connection error: {e}", level="WARNING")
|
||||
time.sleep(2)
|
||||
|
||||
def _stream(self) -> None:
|
||||
parsed = urlparse(self.uri)
|
||||
host = parsed.hostname or "127.0.0.1"
|
||||
port = parsed.port or 10400
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(5.0)
|
||||
sock.connect((host, port))
|
||||
logbuffer.log(f"[wakeword-action] Connected — listening for {list(self.model_callbacks)}")
|
||||
|
||||
detect_msg = {"type": "detect", "data": {"names": list(self.model_callbacks)}}
|
||||
sock.sendall((json.dumps(detect_msg) + "\n").encode("utf-8"))
|
||||
|
||||
audio_start = {"type": "audio-start",
|
||||
"data": {"rate": 16000, "width": 2, "channels": 1}}
|
||||
sock.sendall((json.dumps(audio_start) + "\n").encode("utf-8"))
|
||||
|
||||
read_active = True
|
||||
|
||||
def read_loop() -> None:
|
||||
try:
|
||||
sock.settimeout(1.0)
|
||||
while read_active and not self._stop_event.is_set():
|
||||
try:
|
||||
line = b""
|
||||
while not line.endswith(b"\n"):
|
||||
byte = sock.recv(1)
|
||||
if not byte:
|
||||
return
|
||||
line += byte
|
||||
if not line:
|
||||
return
|
||||
msg = json.loads(line.decode("utf-8"))
|
||||
if msg.get("type") == "detection":
|
||||
name = msg.get("data", {}).get("name", "")
|
||||
cb = self.model_callbacks.get(name)
|
||||
if cb is None:
|
||||
# Try partial match — some servers omit the lang suffix
|
||||
for k, v in self.model_callbacks.items():
|
||||
if name.startswith(k) or k.startswith(name):
|
||||
cb = v
|
||||
break
|
||||
if cb:
|
||||
logbuffer.log(
|
||||
f"[wakeword-action] '{name}' detected — firing action")
|
||||
self._stop_event.set() # one-shot: stop after first fire
|
||||
cb()
|
||||
payload_len = msg.get("payload_length", 0)
|
||||
if payload_len > 0:
|
||||
remaining = payload_len
|
||||
while remaining > 0:
|
||||
chunk = sock.recv(min(remaining, 4096))
|
||||
if not chunk:
|
||||
break
|
||||
remaining -= len(chunk)
|
||||
except socket.timeout:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
reader = threading.Thread(target=read_loop, daemon=True)
|
||||
reader.start()
|
||||
|
||||
cmd = ["pw-record", "--rate=16000", "--channels=1", "--format=s16", "-"]
|
||||
if self.mic:
|
||||
cmd.extend(["--target", self.mic])
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
|
||||
try:
|
||||
while not self._stop_event.is_set() and proc.poll() is None:
|
||||
chunk = proc.stdout.read(3200)
|
||||
if not chunk:
|
||||
break
|
||||
header = {"type": "audio-chunk",
|
||||
"data": {"rate": 16000, "width": 2, "channels": 1},
|
||||
"payload_length": len(chunk)}
|
||||
sock.sendall((json.dumps(header) + "\n").encode("utf-8"))
|
||||
sock.sendall(chunk)
|
||||
finally:
|
||||
read_active = False
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=1.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
reader.join(timeout=1.0)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user