From 5484bc6a1b7f2d396ea87bba169cb3a3627c30b6 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Fri, 5 Jun 2026 17:16:05 +0200 Subject: [PATCH] Add Wayland, Tests, CI, and OpenWakeword integration - Wayland support: text delivery using wtype or ydotool (paste.py) - Testing: added Pytest coverage for routing, quality, and config logic - CI/CD: added test-linux job to .github/workflows/ci.yml - Feature: Hands-free dictation using an external wyoming-openwakeword server, respecting the /tmp/wake_muted toggle. --- .github/workflows/ci.yml | 33 ++++++++ README.md | 6 +- linux/blitztext/config.py | 20 +++++ linux/blitztext/daemon.py | 21 +++++ linux/blitztext/gtksettings.py | 9 +++ linux/blitztext/paste.py | 51 +++++++++--- linux/blitztext/quality.py | 3 - linux/blitztext/wakeword.py | 141 +++++++++++++++++++++++++++++++++ linux/tests/test_config.py | 70 ++++++++++++++++ linux/tests/test_quality.py | 31 ++++++++ linux/tests/test_routing.py | 53 +++++++++++++ workspace.code-workspace | 7 ++ 12 files changed, 429 insertions(+), 16 deletions(-) create mode 100644 linux/blitztext/wakeword.py create mode 100644 linux/tests/test_config.py create mode 100644 linux/tests/test_quality.py create mode 100644 linux/tests/test_routing.py create mode 100644 workspace.code-workspace diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a5dd74..847c7d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,3 +50,36 @@ jobs: - name: Build run: ./build.sh --debug + + test-linux: + name: Test Linux app + runs-on: ubuntu-24.04 + timeout-minutes: 15 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y python3-gi gir1.2-gtk-3.0 gir1.2-appindicator3-0.1 libgirepository1.0-dev libcairo2-dev python3-dev + cd linux + pip install -r requirements.txt + pip install pytest ruff PyGObject + + - name: Syntax check + run: | + cd linux + python -m py_compile blitztext/*.py + + - name: Run Pytest + run: | + cd linux + PYTHONPATH=. pytest tests diff --git a/README.md b/README.md index e43f4dc..3aa811a 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Stream: hotkey → mic PCM chunks → Riva/NIM WebSocket → live words typed ## Requirements -- **Linux desktop with an X11 session** (Wayland needs `ydotool`/`wtype` — not yet supported) +- **Linux desktop with an X11 or Wayland session** (Wayland uses `wtype` or `ydotool`) - **Python 3.11+** (for source installs) - **Host tools:** @@ -355,7 +355,7 @@ systemctl --user enable --now blitztext ## Current Limitations -- **X11 only.** Text delivery uses `xdotool`. Wayland support (`wtype`/`ydotool`) is planned but not implemented. +- **Wayland support** requires `wtype` or `ydotool`. Wayland security prevents global window focus manipulation, so text is delivered to whatever window is active when delivery occurs. - **No automated tests yet.** Contributions welcome (routing, quality gate, config parsing are all highly testable). - **Realtime streaming** requires a compatible Riva/NIM server. - **Local STT speed** depends on your hardware, Whisper model size, and CTranslate2 build (CPU `int8` by default). @@ -372,7 +372,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. - Add Linux screenshots - Improve error messages and first-run setup - Document known-good STT/LLM engine configurations -- Add Wayland support via `wtype` or `ydotool` +- Document known-good Wayland configurations for specific compositors **Quick development loop:** diff --git a/linux/blitztext/config.py b/linux/blitztext/config.py index e3d8cc7..186fac7 100644 --- a/linux/blitztext/config.py +++ b/linux/blitztext/config.py @@ -73,6 +73,10 @@ class Config: # llm engines (presets) for the rewrite step llm_engines: list[LLMEngine] = field(default_factory=list) llm_active: str = "" + # wakeword + wakeword_enabled: bool = False + wakeword_uri: str = "tcp://127.0.0.1:10400" + wakeword_model: str = "okay_computer" # workflows workflows: list[Workflow] = field(default_factory=list) @@ -127,6 +131,7 @@ def load(path: Path = CONFIG_PATH) -> Config: rt = data.get("routing", {}) inp = data.get("input", {}) q = data.get("quality", {}) + ww = data.get("wakeword", {}) cfg = Config( recorder=g.get("recorder", "auto"), @@ -158,6 +163,9 @@ def load(path: Path = CONFIG_PATH) -> Config: silence_rms=float(q.get("silence_rms", 150.0)), reject_hallucinations=bool(q.get("reject_hallucinations", True)), strip_trailing_punctuation=bool(q.get("strip_trailing_punctuation", False)), + wakeword_enabled=bool(ww.get("enabled", False)), + wakeword_uri=ww.get("uri", "tcp://127.0.0.1:10400"), + wakeword_model=ww.get("model", "okay_computer"), ) for entry in data.get("workflow", []): @@ -257,6 +265,11 @@ def save(cfg: Config, path: Path = CONFIG_PATH) -> None: "reject_hallucinations": cfg.reject_hallucinations, "strip_trailing_punctuation": cfg.strip_trailing_punctuation, }, + "wakeword": { + "enabled": cfg.wakeword_enabled, + "uri": cfg.wakeword_uri, + "model": cfg.wakeword_model, + }, "stt": {"active": cfg.stt_active}, "stt_engine": [ {k: v for k, v in { @@ -362,6 +375,13 @@ hotkey = "++" default = "Transcribe" # preset used when no keyword is recognised threshold = 0.82 # 0..1 fuzzy-match strictness (higher = stricter) +[wakeword] +# Hands-free dictation using an external wyoming-openwakeword server. +# Respects the /tmp/wake_muted file to disable listening. +enabled = false +uri = "tcp://127.0.0.1:10400" +model = "okay_computer" + # ---------------------------------------------------------------------------- # Speech-to-text engines (presets). The active one is used for transcription. # type = "local" -> in-process faster-whisper (uses [whisper] above) diff --git a/linux/blitztext/daemon.py b/linux/blitztext/daemon.py index 11cc23b..f60932f 100644 --- a/linux/blitztext/daemon.py +++ b/linux/blitztext/daemon.py @@ -45,6 +45,24 @@ class Daemon: self.recorder_name = detect_recorder(cfg.recorder) self.transcriber: Transcriber | None = None + self._wakeword_listener = None + + def _init_wakeword(self): + if self.cfg.wakeword_enabled: + from .wakeword import WakewordListener + self._wakeword_listener = WakewordListener( + uri=self.cfg.wakeword_uri, + model=self.cfg.wakeword_model, + mic=self.cfg.mic, + on_detect=self._on_wakeword, + ) + self._wakeword_listener.start() + + def _on_wakeword(self): + if not self.is_recording: + # Wakeword only starts dictation. You still stop it via keyboard, + # or we could make it toggle. A toggle is easiest. + self.toggle(self._route_workflow) # -- feedback ------------------------------------------------------------- def _notify(self, title: str, body: str = "", urgency: str = "normal") -> None: @@ -77,6 +95,7 @@ class Daemon: self._emit("loading", None, f"Using {engine.name}") log(f"Using remote STT '{engine.name}' — no local model to load") self._prepared = True + self._init_wakeword() log("Ready.") self._emit("idle", None, "Ready") @@ -376,6 +395,8 @@ class Daemon: scheme.stop_listener() self._scheme = None self.stop_hotkeys() + if self._wakeword_listener: + self._wakeword_listener.stop() def stop_hotkeys(self) -> None: if self._listener is not None: diff --git a/linux/blitztext/gtksettings.py b/linux/blitztext/gtksettings.py index 1601b87..c8e458c 100644 --- a/linux/blitztext/gtksettings.py +++ b/linux/blitztext/gtksettings.py @@ -689,6 +689,12 @@ class SettingsDialog: _labeled(page, "Reject hallucinations", self.q_halluc) self.q_strip = Gtk.Switch(); self.q_strip.set_active(self.cfg.strip_trailing_punctuation); self.q_strip.set_halign(Gtk.Align.START) _labeled(page, "Strip trailing punctuation", self.q_strip) + page.pack_start(Gtk.Separator(), False, False, 8) + page.pack_start(Gtk.Label(label="Hands-free (Wakeword)", xalign=0.0), False, False, 2) + self.ww_enabled = Gtk.Switch(); self.ww_enabled.set_active(self.cfg.wakeword_enabled); self.ww_enabled.set_halign(Gtk.Align.START) + _labeled(page, "Enable wakeword", self.ww_enabled) + self.ww_uri = _labeled(page, "Wyoming URI", _entry(self.cfg.wakeword_uri, placeholder="tcp://127.0.0.1:10400")) + self.ww_model = _labeled(page, "Model name", _entry(self.cfg.wakeword_model, placeholder="okay_computer")) # ===== General ========================================================== def _build_general(self, page: Gtk.Box) -> None: @@ -896,6 +902,9 @@ class SettingsDialog: c.silence_rms = float(self.q_rms.get_text()) c.reject_hallucinations = self.q_halluc.get_active() c.strip_trailing_punctuation = self.q_strip.get_active() + c.wakeword_enabled = self.ww_enabled.get_active() + c.wakeword_uri = self.ww_uri.get_text().strip() + c.wakeword_model = self.ww_model.get_text().strip() c.mic = self._selected_mic_name() c.output = self.gen_output.get_active_text() or "type" c.language = self.gen_lang.get_text().strip() diff --git a/linux/blitztext/paste.py b/linux/blitztext/paste.py index 6c8c9a3..b0e0490 100644 --- a/linux/blitztext/paste.py +++ b/linux/blitztext/paste.py @@ -10,12 +10,19 @@ delivery, so a brief focus change during processing doesn't misfire. from __future__ import annotations +import os import shutil import subprocess import time +def _is_wayland() -> bool: + return os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland" + + def active_window_id() -> str | None: + if _is_wayland(): + return None # Wayland compositors don't allow global window ID queries if not shutil.which("xdotool"): return None try: @@ -29,7 +36,7 @@ def active_window_id() -> str | None: def _focus(window_id: str | None) -> None: - if window_id: + if window_id and not _is_wayland() and shutil.which("xdotool"): subprocess.run(["xdotool", "windowactivate", "--sync", window_id], check=False) time.sleep(0.05) @@ -37,35 +44,59 @@ def _focus(window_id: str | None) -> None: def deliver(text: str, *, mode: str = "type", window_id: str | None = None, type_delay_ms: int = 4) -> None: if not text: return - if not shutil.which("xdotool"): + + wayland = _is_wayland() + if not wayland and not shutil.which("xdotool"): raise RuntimeError("xdotool not found; cannot type into the focused window.") + if wayland and not shutil.which("wtype") and not shutil.which("ydotool"): + raise RuntimeError("wtype or ydotool not found; cannot type in Wayland session.") _focus(window_id) # Give the user time to release the hotkey modifiers before we synthesize input. time.sleep(0.12) if mode == "paste" and _set_clipboard(text): - subprocess.run(["xdotool", "key", "--clearmodifiers", "ctrl+v"], check=False) + if wayland: + if shutil.which("wtype"): + subprocess.run(["wtype", "-M", "ctrl", "v", "-m", "ctrl"], check=False) + else: + subprocess.run(["ydotool", "key", "ctrl+v"], check=False) + else: + subprocess.run(["xdotool", "key", "--clearmodifiers", "ctrl+v"], check=False) return - subprocess.run( - ["xdotool", "type", "--clearmodifiers", "--delay", str(type_delay_ms), "--", text], - check=False, - ) + if wayland: + if shutil.which("wtype"): + subprocess.run(["wtype", "-d", str(type_delay_ms), "--", text], check=False) + else: + subprocess.run(["ydotool", "type", "-d", str(type_delay_ms), text], check=False) + else: + subprocess.run( + ["xdotool", "type", "--clearmodifiers", "--delay", str(type_delay_ms), "--", text], + check=False, + ) def press_enter(window_id: str | None = None) -> None: """Send Return to the focused/target window (auto-send after paste).""" - if not shutil.which("xdotool"): + wayland = _is_wayland() + if not wayland and not shutil.which("xdotool"): return _focus(window_id) time.sleep(0.08) - subprocess.run(["xdotool", "key", "--clearmodifiers", "Return"], check=False) + + if wayland: + if shutil.which("wtype"): + subprocess.run(["wtype", "-k", "Return"], check=False) + else: + subprocess.run(["ydotool", "key", "enter"], check=False) + else: + subprocess.run(["xdotool", "key", "--clearmodifiers", "Return"], check=False) def _set_clipboard(text: str) -> bool: """Best-effort clipboard set; returns False if no clipboard tool is available.""" - for argv in (["xclip", "-selection", "clipboard"], ["xsel", "--clipboard", "--input"], ["wl-copy"]): + for argv in (["wl-copy"], ["xclip", "-selection", "clipboard"], ["xsel", "--clipboard", "--input"]): if shutil.which(argv[0]): try: subprocess.run(argv, input=text.encode("utf-8"), check=True) diff --git a/linux/blitztext/quality.py b/linux/blitztext/quality.py index 8b6ed5d..03fc023 100644 --- a/linux/blitztext/quality.py +++ b/linux/blitztext/quality.py @@ -63,9 +63,6 @@ def is_hallucination(text: str, duration: float) -> bool: return True if duration <= 2.5 and norm in _HALLUCINATIONS: return True - # A very short clip that produced only a stock phrase is suspect too. - if duration <= 1.5 and len(norm.split()) <= 2 and norm in _HALLUCINATIONS: - return True return False diff --git a/linux/blitztext/wakeword.py b/linux/blitztext/wakeword.py new file mode 100644 index 0000000..7c82b3e --- /dev/null +++ b/linux/blitztext/wakeword.py @@ -0,0 +1,141 @@ +"""Wakeword detection using Wyoming protocol (openwakeword). + +Runs a background thread that captures audio and streams it to a Wyoming +server (e.g. rhasspy/wyoming-openwakeword). When a detection event occurs, +it triggers the main daemon. + +Respects /tmp/wake_muted to allow easy desktop integration via scripts. +""" + +from __future__ import annotations + +import json +import os +import socket +import subprocess +import threading +import time +from urllib.parse import urlparse + +from . import logbuffer + +_MUTE_FILE = "/tmp/wake_muted" + +class WakewordListener: + def __init__(self, uri: str, model: str, mic: str, on_detect): + self.uri = uri + self.model = model + self.mic = mic + self.on_detect = on_detect + + self._stop_event = threading.Event() + self._thread = None + self._cooldown_until = 0.0 + + def start(self): + if self._thread is not None: + return + self._stop_event.clear() + self._thread = threading.Thread(target=self._run, daemon=True, name="WakewordListener") + self._thread.start() + + def stop(self): + self._stop_event.set() + if self._thread: + self._thread.join(timeout=2.0) + self._thread = None + + def _run(self): + while not self._stop_event.is_set(): + try: + self._stream() + except Exception as e: + logbuffer.log(f"[wakeword] Connection error: {e}") + time.sleep(3) # Retry backoff + + def _stream(self): + parsed = urlparse(self.uri) + host = parsed.hostname or "127.0.0.1" + port = parsed.port or 10400 + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(5.0) + sock.connect((host, port)) + logbuffer.log(f"[wakeword] Connected to {self.uri}") + + # Request detection for the specific model + detect_msg = {"type": "detect", "data": {"names": [self.model]}} + sock.sendall((json.dumps(detect_msg) + "\n").encode("utf-8")) + + audio_start = {"type": "audio-start", "data": {"rate": 16000, "width": 2, "channels": 1}} + sock.sendall((json.dumps(audio_start) + "\n").encode("utf-8")) + + # Start recording subprocess (16kHz, 16-bit, mono) + cmd = ["pw-record", "--rate=16000", "--channels=1", "--format=s16", "-"] + if self.mic: + cmd.extend(["--target", self.mic]) + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + + try: + sock.settimeout(1.0) + while not self._stop_event.is_set() and proc.poll() is None: + # Read chunk + chunk = proc.stdout.read(3200) # 100ms of 16kHz 16-bit mono + if not chunk: + break + + # Send chunk + header = {"type": "audio-chunk", "data": {"rate": 16000, "width": 2, "channels": 1}, "payload_length": len(chunk)} + sock.sendall((json.dumps(header) + "\n").encode("utf-8")) + sock.sendall(chunk) + + # Check for responses (detections) + try: + while True: + # Read line + line = b"" + while not line.endswith(b"\n"): + byte = sock.recv(1) + if not byte: + break + line += byte + + if not line: + break + + msg = json.loads(line.decode("utf-8")) + + if msg.get("type") == "detection": + self._handle_detection() + + payload_len = msg.get("payload_length", 0) + if payload_len > 0: + # Consume payload + remaining = payload_len + while remaining > 0: + received = sock.recv(min(remaining, 4096)) + if not received: + break + remaining -= len(received) + + except socket.timeout: + pass # No messages received, continue streaming + finally: + proc.terminate() + try: + proc.wait(timeout=1.0) + except subprocess.TimeoutExpired: + proc.kill() + + def _handle_detection(self): + if time.time() < self._cooldown_until: + return + + if os.path.exists(_MUTE_FILE): + logbuffer.log("[wakeword] Detected, but muted via /tmp/wake_muted") + return + + logbuffer.log(f"[wakeword] Detected '{self.model}'!") + self._cooldown_until = time.time() + 3.0 # 3s cooldown + self.on_detect() diff --git a/linux/tests/test_config.py b/linux/tests/test_config.py new file mode 100644 index 0000000..16de815 --- /dev/null +++ b/linux/tests/test_config.py @@ -0,0 +1,70 @@ +import pytest +import tempfile +import os +from pathlib import Path +from blitztext.config import load, save, Config, Workflow, STTEngine, LLMEngine + +def test_load_default_config(): + with tempfile.TemporaryDirectory() as tempdir: + config_path = Path(tempdir) / "config.toml" + # Since file doesn't exist, it should create default + cfg = load(config_path) + + assert config_path.exists() + assert cfg.recorder == "auto" + assert cfg.output == "type" + assert cfg.language == "de" + assert len(cfg.workflows) > 0 + assert cfg.workflows[0].name == "Transcribe" + assert cfg.workflows[0].mode == "transcribe" + assert cfg.routing_enabled == True + assert cfg.stt_active == "Local faster-whisper" + +def test_save_and_load_config(): + with tempfile.TemporaryDirectory() as tempdir: + config_path = Path(tempdir) / "config.toml" + + # Create a config + cfg = Config() + cfg.recorder = "pw-record" + cfg.output = "paste" + cfg.language = "en" + cfg.workflows = [ + Workflow(name="Test Workflow", hotkey="+t", mode="rewrite", prompt="Testing prompt") + ] + cfg.stt_engines = [STTEngine("TestSTT", "openai", "http://localhost", "model")] + cfg.stt_active = "TestSTT" + + save(cfg, config_path) + assert config_path.exists() + + # Load it back + loaded_cfg = load(config_path) + assert loaded_cfg.recorder == "pw-record" + assert loaded_cfg.output == "paste" + assert loaded_cfg.language == "en" + assert len(loaded_cfg.workflows) == 1 + assert loaded_cfg.workflows[0].name == "Test Workflow" + assert loaded_cfg.workflows[0].prompt == "Testing prompt" + assert loaded_cfg.stt_engines[0].name == "TestSTT" + assert loaded_cfg.stt_active == "TestSTT" + +def test_properties(): + cfg = Config() + cfg.stt_engines = [ + STTEngine("Engine1", "local"), + STTEngine("Engine2", "openai") + ] + cfg.stt_active = "Engine2" + assert cfg.active_stt.name == "Engine2" + + cfg.stt_active = "NonExistent" + # Fallback to first engine if active is not found + assert cfg.active_stt.name == "Engine1" + + cfg.workflows = [ + Workflow(name="WF1", hotkey="", mode="transcribe"), + Workflow(name="WF2", hotkey="", mode="transcribe") + ] + assert cfg.preset_by_name("WF2").name == "WF2" + assert cfg.preset_by_name("NonExistent") is None diff --git a/linux/tests/test_quality.py b/linux/tests/test_quality.py new file mode 100644 index 0000000..35885e7 --- /dev/null +++ b/linux/tests/test_quality.py @@ -0,0 +1,31 @@ +import pytest +from blitztext.quality import too_quiet, is_hallucination, clean + +def test_too_quiet(): + # Below duration + assert too_quiet(0.5, 1000.0, min_seconds=1.0, silence_rms=500.0) == True + # Below rms + assert too_quiet(2.0, 300.0, min_seconds=1.0, silence_rms=500.0) == True + # Good quality + assert too_quiet(2.0, 1000.0, min_seconds=1.0, silence_rms=500.0) == False + +def test_is_hallucination(): + # Hallucination for short duration + assert is_hallucination("Thank you.", 2.0) == True + assert is_hallucination("Vielen Dank", 1.5) == True + + # Same phrase but longer duration -> Not considered a hallucination + assert is_hallucination("Thank you.", 3.0) == False + + # Non-hallucination + assert is_hallucination("This is a valid sentence.", 1.0) == False + + # Empty string + assert is_hallucination("", 1.0) == True + assert is_hallucination(" ", 1.0) == True + +def test_clean(): + assert clean(" hello world ") == "hello world" + assert clean("hello world. ", strip_trailing_punctuation=True) == "hello world" + assert clean("hello world!", strip_trailing_punctuation=True) == "hello world" + assert clean("hello world", strip_trailing_punctuation=True) == "hello world" diff --git a/linux/tests/test_routing.py b/linux/tests/test_routing.py new file mode 100644 index 0000000..80e7a9f --- /dev/null +++ b/linux/tests/test_routing.py @@ -0,0 +1,53 @@ +import pytest +from dataclasses import dataclass +from blitztext.routing import route, normalize, _strip_span + +@dataclass +class DummyPreset: + name: str + keywords: list[str] + +def test_normalize(): + assert normalize("Nicer E-Mail.") == ["nicer", "e", "mail"] + assert normalize("calm down!") == ["calm", "down"] + +def test_route_no_presets(): + res = route("Hello world", []) + assert res.preset_name is None + assert res.text == "Hello world" + assert res.keyword is None + +def test_route_match_start(): + presets = [ + DummyPreset("Email", ["nicer email", "bessere email"]), + DummyPreset("Calm", ["calm down"]), + ] + res = route("Nicer e-mail can you send me the report", presets) + assert res.preset_name == "Email" + assert res.keyword == "nicer email" + assert res.position == "start" + assert res.text == "can you send me the report" + +def test_route_match_end(): + presets = [ + DummyPreset("Email", ["nicer email"]), + DummyPreset("Calm", ["calm down"]), + ] + res = route("Can you send me the report nicer email", presets) + assert res.preset_name == "Email" + assert res.keyword == "nicer email" + assert res.position == "end" + assert res.text == "Can you send me the report" + +def test_route_no_match_middle(): + presets = [ + DummyPreset("Email", ["nicer email"]), + ] + res = route("Can you send me a nicer email please", presets) + assert res.preset_name is None + assert res.text == "Can you send me a nicer email please" + +def test_strip_span(): + # 'nicer e-mail' normalizes to 3 tokens + res = _strip_span("Nicer e-mail can you send me the report", 3, "start") + assert res == "can you send me the report" diff --git a/workspace.code-workspace b/workspace.code-workspace new file mode 100644 index 0000000..362d7c2 --- /dev/null +++ b/workspace.code-workspace @@ -0,0 +1,7 @@ +{ + "folders": [ + { + "path": "." + } + ] +} \ No newline at end of file