From d55cdafcc12cf6d03c0ef10e18d437ede7c59169 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Sun, 7 Jun 2026 13:13:41 +0200 Subject: [PATCH] on-screen dictation overlay at the cursor; Release 1.5.0 Show a translucent, click-through bubble the moment recording starts (by hotkey or wakeword): a pulsing microphone, a live waveform of the mic level, and the recognised text (word-by-word when streaming, otherwise a brief final-result confirmation). The tail points at the text caret via AT-SPI accessibility, falling back to the mouse pointer, then a screen corner. Also gives hands-free wakeword sessions visible feedback, whose notifications are suppressed by design. X11 only. - overlay.py: GTK override-redirect HUD (mic + waveform + bubble), drawn with Cairo; thread-safe, marshalled onto the GTK loop. - caret.py: best-effort anchor (AT-SPI caret -> pointer -> window/corner). - daemon: optional level_cb/text_cb hooks; reuses the VAD level meter for non-streaming, a dedicated meter for streaming. Stays UI-agnostic. - gtkui: instantiate the overlay, drive show/update/hide from status. - General settings: "Visual overlay" toggle; config overlay_enabled / overlay_anchor (default on, "caret"). - Docs: CHANGELOG 1.5.0, version bump, README + MANUAL. Co-Authored-By: Claude Opus 4.8 (1M context) --- MANUAL.md | 9 +- README.md | 6 +- linux/CHANGELOG.md | 20 +- linux/README.md | 10 +- linux/blitztext/__init__.py | 2 +- linux/blitztext/caret.py | 256 ++++++++++++++++++++++ linux/blitztext/config.py | 9 + linux/blitztext/daemon.py | 46 +++- linux/blitztext/gtksettings.py | 5 + linux/blitztext/gtkui.py | 67 +++++- linux/blitztext/overlay.py | 382 +++++++++++++++++++++++++++++++++ 11 files changed, 804 insertions(+), 8 deletions(-) create mode 100644 linux/blitztext/caret.py create mode 100644 linux/blitztext/overlay.py diff --git a/MANUAL.md b/MANUAL.md index fb768b2..38dbe09 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -167,7 +167,8 @@ hands-free wakeword sounds above are **separate and independent**. ## General tab -Microphone, text delivery, language, notifications, and autostart. +Microphone, text delivery, language, notifications, the on-screen overlay, and +autostart. | Setting | TOML key | Description | |---|---|---| @@ -177,6 +178,7 @@ Microphone, text delivery, language, notifications, and autostart. | **Language hint** | `language` | Spoken-language code (`de`, `en`, …). Blank = auto-detect. | | **Notifications** | `notify` | Show desktop notifications for recording/transcription status and errors (manual sessions). | | **Announce matched preset** | `notify_routing` | After a voice command, pop a notification showing which preset (and spoken keyword) matched — shown **even for hands-free** sessions, with the preset's emoji. Only fires on a real match. | +| **Visual overlay** | `overlay_enabled` | Show a translucent bubble at the cursor while you dictate — a pulsing **microphone**, a **live waveform** of your mic level, and the **recognised text** (word-by-word with a streaming engine, or the final result as a brief confirmation). The tail points at where the text lands, and it gives **hands-free** sessions visible feedback. Click-through; never takes focus. *(X11 only.)* | | **Launch on login** | *(autostart file)* | Start Blitztext automatically when you log in (writes a desktop autostart entry, not `config.toml`). | --- @@ -239,3 +241,8 @@ A few behaviours live in `config.toml` without a dedicated tab control: no keyword matches), and `threshold` (`0`–`1` fuzzy-match strictness). - **`timeout`** — network timeout (seconds) for remote STT/LLM requests. - **`type_delay_ms`** — delay between simulated keystrokes in `type` output mode. +- **`overlay_anchor`** — where the overlay's tail points: `caret` (best-effort — + follows the focused app's text caret via AT-SPI accessibility, falling back to + the pointer), `pointer` (always the mouse pointer), or `corner` (a fixed screen + corner; also the automatic fallback on Wayland or when the cursor can't be + located). Paired with the **Visual overlay** toggle above. diff --git a/README.md b/README.md index 126d6db..ec65ecb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Blitztext is a native Linux dictation tool that captures your voice, transcribes it locally with [faster-whisper](https://github.com/SYSTRAN/faster-whisper), optionally rewrites the text through an LLM, and types the result directly into whatever application has focus. Think macOS Dictation, but open-source, extensible, and designed for power users who want full control over their speech-to-text pipeline. -> **Status:** Experimental open-source Linux/X11 desktop app (v1.4.0). +> **Status:** Experimental open-source Linux/X11 desktop app (v1.5.0). > No hosted backend — bring your own models and endpoints.

@@ -57,6 +57,7 @@ Stream: hotkey → mic PCM chunks → Riva/NIM WebSocket → live words typed - **Voice-keyword routing.** One hotkey, multiple workflows. Say "nicer email" at the start or end of your speech and the email-rewrite preset activates automatically (fuzzy-matched, ASR-tolerant). - **Quality gate.** Silent clips, too-short recordings, and Whisper hallucinations ("Thank you.", "Untertitel…") are caught and rejected before they reach your text field. - **Realtime streaming.** Connect a Riva/NIM realtime STT server and see stable words typed live as you speak. +- **On-screen overlay at the cursor.** The moment you start dictating — by hotkey *or* wakeword — a translucent bubble pops up at the cursor with a pulsing microphone, a live waveform of your mic level, and the recognised text (live with a streaming engine, or as a brief confirmation otherwise). Its tail points at the text caret (via accessibility) and finally gives hands-free wakeword sessions visible feedback. Click-through, never steals focus; toggle in Settings → General. - **Built-in benchmarking.** Compare all your configured STT engines against a reference WAV + transcript to find the fastest and most accurate. --- @@ -88,7 +89,7 @@ and screen-reader (ATK) support. Click any image to open it full size.

General settings tab
- General — core preferences: microphone with a live level meter, output mode (type vs. paste), language hint, type delay, and autostart on login. + General — core preferences: microphone with a live level meter, output mode (type vs. paste), language hint, type delay, the on-screen dictation overlay, and autostart on login.

@@ -413,6 +414,7 @@ systemctl --user enable --now blitztext - **Wayland support** requires `wtype` or `ydotool`. Wayland security prevents global window focus manipulation, so text is delivered to whatever window is active when delivery occurs. - **No automated tests yet.** Contributions welcome (routing, quality gate, config parsing are all highly testable). - **Realtime streaming** requires a compatible Riva/NIM server. +- **The on-screen overlay is X11-only** (it positions a window at the cursor and reads the pointer/caret); on Wayland it falls back to a fixed screen corner. Caret-accurate anchoring further needs the focused app to expose its text caret over AT-SPI accessibility — otherwise it follows the mouse pointer. - **Local STT speed** depends on your hardware, Whisper model size, and CTranslate2 build (CPU `int8` by default). - This is experimental software provided as-is. diff --git a/linux/CHANGELOG.md b/linux/CHANGELOG.md index 41835b5..0682eb1 100644 --- a/linux/CHANGELOG.md +++ b/linux/CHANGELOG.md @@ -9,6 +9,23 @@ The version is defined in [`blitztext/__init__.py`](blitztext/__init__.py). ## [Unreleased] +## [1.5.0] - 2026-06-07 + +### Added +- **On-screen dictation overlay** (Settings → General → "Visual overlay", or + `[general] overlay_enabled`, default on): the moment recording starts — by + hotkey **or** wakeword — a translucent bubble appears at the cursor showing a + pulsing **microphone**, a **live waveform** of your mic level, and the + **recognised text** (word-by-word with a realtime streaming STT engine, or the + final result as a brief confirmation otherwise). Its tail points at where the + text will land: it follows the **text caret** when the focused app exposes it + over accessibility (AT-SPI), otherwise the **mouse pointer**, otherwise a + screen corner — tune via `[general] overlay_anchor = "caret" | "pointer" | + "corner"`. The window is click-through and never takes focus, and it finally + gives **hands-free wakeword sessions** visible feedback (their notifications + are suppressed by design). X11 only; falls back to a corner where the cursor + can't be located. + ### Changed - **Presets are speakable by name**: voice routing now matches a preset's *name* as an implicit keyword, so a preset works by voice even with no keywords @@ -185,7 +202,8 @@ into that field. AppIndicator typelibs and GNOME `ubuntu-appindicators` extension are already present on the target host). -[Unreleased]: https://github.com/mARTin-B78/blitztext-app-linux/compare/v1.1.0...HEAD +[Unreleased]: https://github.com/mARTin-B78/blitztext-app-linux/compare/v1.5.0...HEAD +[1.5.0]: https://github.com/mARTin-B78/blitztext-app-linux/compare/v1.4.0...v1.5.0 [1.1.0]: https://github.com/mARTin-B78/blitztext-app-linux/compare/v1.0.1...v1.1.0 [1.0.1]: https://github.com/mARTin-B78/blitztext-app-linux/compare/v1.0.0...v1.0.1 [1.0.0]: https://github.com/mARTin-B78/blitztext-app-linux/releases/tag/v1.0.0 diff --git a/linux/README.md b/linux/README.md index efcac20..24f3ed5 100644 --- a/linux/README.md +++ b/linux/README.md @@ -39,6 +39,14 @@ Each normal hotkey **toggles**: press to start recording, press again to stop then it transcribes, optionally rewrites, and types the result where your cursor is. Streaming workflows type stable words live while you speak. +While you dictate, an optional **on-screen overlay** (Settings → General → +"Visual overlay", default on) shows a translucent bubble at the cursor with a +pulsing microphone, a live waveform of your mic level, and the recognised text — +word-by-word in streaming mode, or the final result as a brief confirmation. Its +tail points at the text caret (via AT-SPI accessibility) or the mouse pointer; it +is click-through, never steals focus, and also gives hands-free wakeword sessions +visible feedback. Tune the anchor with `[general] overlay_anchor`. X11 only. + ## Screenshots Everything is configured in the **Settings** window — every tab has tooltips and @@ -66,7 +74,7 @@ screen-reader (ATK) support. Click any image to open it full size.

General settings tab
- General — core preferences: microphone with a live level meter, output mode (type vs. paste), language hint, type delay, and autostart on login. + General — core preferences: microphone with a live level meter, output mode (type vs. paste), language hint, type delay, the on-screen dictation overlay, and autostart on login.

diff --git a/linux/blitztext/__init__.py b/linux/blitztext/__init__.py index 75842ef..1f04722 100644 --- a/linux/blitztext/__init__.py +++ b/linux/blitztext/__init__.py @@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host (not in a container) so it can type into any application via xdotool. """ -__version__ = "1.4.0" +__version__ = "1.5.0" diff --git a/linux/blitztext/caret.py b/linux/blitztext/caret.py new file mode 100644 index 0000000..22c8c42 --- /dev/null +++ b/linux/blitztext/caret.py @@ -0,0 +1,256 @@ +"""Best-effort screen anchor for the overlay: *where* to point the bubble's tip. + +The overlay wants to sit at "the cursor where the text will land". On X11 there +is no portable way to read the text caret of an arbitrary app, so we degrade +through a chain of decreasing precision: + + 1. AT-SPI caret — the real text insertion point, when the focused app exposes + it over accessibility (native GTK/Qt apps do; many terminals / Electron / + web views do not). Tracked passively via the GLib main loop, so reads are + instant and never block. + 2. Mouse pointer — `xdotool getmouselocation`. Always available on X11; a good + proxy since the pointer is usually near where you're typing. + 3. Window / screen — top-centre of the target window, else screen bottom-centre. + +Everything here is defensive: any failure falls through to the next tier, and +the whole module is optional — if AT-SPI isn't running you simply get the +pointer anchor. Returns a `(x, y, region)` anchor in **root/screen** pixels, +where `region` is the caret/line box so the overlay can avoid covering it. +""" + +from __future__ import annotations + +import shutil +import subprocess +import time +from dataclasses import dataclass + +from .logbuffer import log + + +@dataclass +class Anchor: + x: int # root-relative x to point the tail tip at + y: int # root-relative y (top of the caret/line box) + height: int = 0 # caret/line height, so the bubble can clear the line + source: str = "" # "caret" | "pointer" | "window" | "corner" — for logging + + +# --------------------------------------------------------------------------- # +# Tier 1: AT-SPI caret tracking (best-effort, passive) +# --------------------------------------------------------------------------- # +class _CaretTracker: + """Passively follow the focused editable text's caret via AT-SPI events. + + We never poll the a11y tree on the hot path (that can be slow and can block + on unresponsive apps). Instead we subscribe to caret-moved / focus events and + cache the last caret rectangle; `rect()` just returns the cached value if it + is fresh enough to still be meaningful. + """ + + STALE_SECONDS = 30.0 # ignore a cached caret older than this + + def __init__(self) -> None: + self._ok = False + self._listener = None + self._rect: tuple[int, int, int, int] | None = None # x, y, w, h (screen) + self._stamp = 0.0 + self._Atspi = None + + def start(self) -> bool: + """Register AT-SPI listeners on the (already running) GLib main loop. + + Safe to call when accessibility is disabled — it just returns False and + the anchor logic skips this tier from then on. + """ + try: + import gi + gi.require_version("Atspi", "2.0") + from gi.repository import Atspi # noqa: N813 + except (ImportError, ValueError) as exc: + log(f"[overlay] AT-SPI unavailable, caret anchor disabled: {exc}") + return False + try: + # init() is idempotent; returns 0/1. Connects to the a11y registry. + Atspi.init() + self._Atspi = Atspi + self._listener = Atspi.EventListener.new(self._on_event) + # Caret moves give us the live position; focus changes let us grab the + # caret of a freshly-focused field even before it moves. + self._listener.register("object:text-caret-moved") + self._listener.register("object:state-changed:focused") + self._ok = True + log("[overlay] AT-SPI caret tracking active") + return True + except Exception as exc: # noqa: BLE001 - a11y bus may be down/locked + log(f"[overlay] AT-SPI init failed, caret anchor disabled: {exc}") + self._ok = False + return False + + def stop(self) -> None: + try: + if self._listener is not None: + self._listener.deregister("object:text-caret-moved") + self._listener.deregister("object:state-changed:focused") + except Exception: # noqa: BLE001 + pass + self._listener = None + self._ok = False + + def _on_event(self, event) -> None: + # Runs on the GLib main thread (same loop GTK uses). Keep it cheap and + # never raise — an exception here would bubble into the a11y dispatcher. + try: + if event.type.startswith("object:state-changed:focused") and not event.detail1: + return # a *de*focus event — nothing to read + source = event.source + if source is None: + return + rect = self._caret_rect(source) + if rect is not None: + self._rect = rect + self._stamp = time.time() + except Exception: # noqa: BLE001 + pass + + def _caret_rect(self, acc) -> tuple[int, int, int, int] | None: + Atspi = self._Atspi + text = None + try: + text = acc.get_text_iface() + except Exception: # noqa: BLE001 + text = None + if text is None: + return None + try: + offset = text.get_caret_offset() + # Extents of the character at the caret, in absolute screen coords. + ext = text.get_character_extents(offset, Atspi.CoordType.SCREEN) + x, y, w, h = ext.x, ext.y, ext.width, ext.height + if w == 0 and h == 0: + # End-of-line / empty field: fall back to the component box so we + # at least anchor on the right widget. + comp = acc.get_component_iface() + if comp is None: + return None + cext = comp.get_extents(Atspi.CoordType.SCREEN) + return int(cext.x), int(cext.y), 2, int(cext.height) or 18 + if x < 0 or y < 0: + return None + return int(x), int(y), int(w) or 2, int(h) or 18 + except Exception: # noqa: BLE001 + return None + + def rect(self) -> tuple[int, int, int, int] | None: + if not self._ok or self._rect is None: + return None + if time.time() - self._stamp > self.STALE_SECONDS: + return None + return self._rect + + +# --------------------------------------------------------------------------- # +# Tier 2 + 3: pointer and window fallbacks +# --------------------------------------------------------------------------- # +def _pointer() -> tuple[int, int] | None: + if not shutil.which("xdotool"): + return None + try: + out = subprocess.run( + ["xdotool", "getmouselocation", "--shell"], + capture_output=True, text=True, check=True, timeout=0.5, + ).stdout + except (OSError, subprocess.SubprocessError): + return None + vals: dict[str, str] = {} + for line in out.splitlines(): + if "=" in line: + k, _, v = line.partition("=") + vals[k.strip()] = v.strip() + try: + return int(vals["X"]), int(vals["Y"]) + except (KeyError, ValueError): + return None + + +def _window_box(window_id: str | None) -> tuple[int, int, int] | None: + """Top-centre of the target window: (x, y, height_hint).""" + if not window_id or not shutil.which("xdotool"): + return None + try: + out = subprocess.run( + ["xdotool", "getwindowgeometry", "--shell", window_id], + capture_output=True, text=True, check=True, timeout=0.5, + ).stdout + except (OSError, subprocess.SubprocessError): + return None + vals: dict[str, str] = {} + for line in out.splitlines(): + if "=" in line: + k, _, v = line.partition("=") + vals[k.strip()] = v.strip() + try: + x, y = int(vals["X"]), int(vals["Y"]) + w = int(vals["WIDTH"]) + return x + w // 2, y + 40, 0 + except (KeyError, ValueError): + return None + + +def _screen_bottom_centre() -> tuple[int, int]: + try: + import gi + gi.require_version("Gdk", "3.0") + from gi.repository import Gdk + disp = Gdk.Display.get_default() + mon = disp.get_primary_monitor() or disp.get_monitor(0) + geo = mon.get_geometry() + return geo.x + geo.width // 2, geo.y + geo.height - 140 + except Exception: # noqa: BLE001 + return 960, 800 # last-ditch constant; better than crashing + + +_tracker: _CaretTracker | None = None + + +def start_tracking() -> None: + """Begin passive caret tracking (call once, from the GTK main thread).""" + global _tracker + if _tracker is None: + _tracker = _CaretTracker() + _tracker.start() + + +def stop_tracking() -> None: + global _tracker + if _tracker is not None: + _tracker.stop() + _tracker = None + + +def resolve(anchor_mode: str, window_id: str | None) -> Anchor: + """Resolve the on-screen anchor for the overlay, honouring `anchor_mode`. + + anchor_mode: + "caret" -> caret → pointer → window → corner + "pointer" -> pointer → window → corner + "corner" -> window → corner + Always returns an Anchor (never None) so callers don't special-case failure. + """ + if anchor_mode == "caret" and _tracker is not None: + r = _tracker.rect() + if r is not None: + x, y, _w, h = r + return Anchor(x=x, y=y, height=h or 18, source="caret") + + if anchor_mode in ("caret", "pointer"): + p = _pointer() + if p is not None: + return Anchor(x=p[0], y=p[1], height=0, source="pointer") + + wb = _window_box(window_id) + if wb is not None: + return Anchor(x=wb[0], y=wb[1], height=0, source="window") + + x, y = _screen_bottom_centre() + return Anchor(x=x, y=y, height=0, source="corner") diff --git a/linux/blitztext/config.py b/linux/blitztext/config.py index 286668c..b9f8db0 100644 --- a/linux/blitztext/config.py +++ b/linux/blitztext/config.py @@ -40,6 +40,9 @@ class Config: notify: bool = True notify_routing: bool = True # announce which preset/keyword a voice command matched language: str = "de" # whisper hint; "" = autodetect + # on-screen overlay (mic + live waveform + recognised-text bubble) + overlay_enabled: bool = True + overlay_anchor: str = "caret" # caret (best-effort AT-SPI) | pointer | corner # input scheme input_mode: str = "modifiers" # "modifiers" (Ctrl+Win/Ctrl/Alt/Esc) | "hotkeys" (combos) push_to_talk: bool = False @@ -160,6 +163,8 @@ def load(path: Path = CONFIG_PATH) -> Config: notify=bool(g.get("notify", True)), notify_routing=bool(g.get("notify_routing", True)), language=g.get("language", "de"), + overlay_enabled=bool(g.get("overlay_enabled", True)), + overlay_anchor=g.get("overlay_anchor", "caret"), model=w.get("model", "small"), device=w.get("device", "auto"), compute_type=w.get("compute_type", "auto"), @@ -258,6 +263,8 @@ def save(cfg: Config, path: Path = CONFIG_PATH) -> None: "notify": cfg.notify, "notify_routing": cfg.notify_routing, "language": cfg.language, + "overlay_enabled": cfg.overlay_enabled, + "overlay_anchor": cfg.overlay_anchor, }, "whisper": { "model": cfg.model, @@ -362,6 +369,8 @@ type_delay_ms = 12 # per-keystroke delay for xdotool type (raise if chars notify = true # desktop notifications for each phase notify_routing = true # announce which preset/keyword a voice command matched (shown even hands-free) language = "de" # Whisper language hint; "" = autodetect +overlay_enabled = true # on-screen mic + live waveform + recognised-text bubble at the cursor +overlay_anchor = "caret" # caret (best-effort, follows the text cursor) | pointer | corner [input] # How you start/stop dictation. diff --git a/linux/blitztext/daemon.py b/linux/blitztext/daemon.py index 7b5502e..51878fd 100644 --- a/linux/blitztext/daemon.py +++ b/linux/blitztext/daemon.py @@ -28,9 +28,18 @@ StatusCallback = Callable[[str, str | None, str], None] class Daemon: - def __init__(self, cfg: Config, status_cb: StatusCallback | None = None): + def __init__(self, cfg: Config, status_cb: StatusCallback | None = None, + level_cb: Callable[[float], None] | None = None, + text_cb: Callable[[str], None] | None = None): self.cfg = cfg self.status_cb = status_cb + # Optional UI feedback hooks for the on-screen overlay. The daemon stays + # UI-agnostic: these are no-ops in headless mode. level_cb gets the live + # mic level (0..1); text_cb gets the running transcript while streaming. + self.level_cb = level_cb + self.text_cb = text_cb + self._ov_meter = None + self._ov_text_final = "" self._lock = threading.Lock() self._recording: Recording | None = None self._streaming: RivaRealtimeStreamer | None = None @@ -140,6 +149,10 @@ class Daemon: silence = max(0.5, self.cfg.wakeword_silence_seconds) def on_level(level): + # Feed the overlay waveform (the VAD meter is already capturing, so + # we reuse its level rather than opening a second stream). + if self.level_cb: + self.level_cb(level) now = time.time() if level > 0.05: self._vad_last_speech = now @@ -156,6 +169,23 @@ class Daemon: self._vad_meter.stop() self._vad_meter = None + def _ov_meter_start(self) -> None: + """A level meter purely to drive the overlay waveform in streaming mode. + + Non-streaming recordings reuse the VAD meter instead; this only runs when + an overlay is attached and we have no other level source. Best-effort: + LevelMeter.start() fails quietly if the device is busy.""" + if not self.level_cb: + return + from . import audio + self._ov_meter = audio.LevelMeter(self.cfg.mic, on_level=self.level_cb) + self._ov_meter.start() + + def _ov_meter_stop(self) -> None: + if getattr(self, "_ov_meter", None) is not None: + self._ov_meter.stop() + self._ov_meter = None + def _play_sound(self, sound_name: str) -> None: if not self.cfg.sounds_enabled: return @@ -204,6 +234,7 @@ class Daemon: self._notify("Streaming unavailable", "Select a riva_realtime STT engine.", "critical") return self._stream_segment_text = "" + self._ov_text_final = "" streamer = RivaRealtimeStreamer( engine, device=self.cfg.mic, @@ -222,7 +253,9 @@ class Daemon: self._dnotify(f"● {wf.name}", "Live transcript…") try: streamer.start() + self._ov_meter_start() except Exception as exc: # noqa: BLE001 + self._ov_meter_stop() with self._lock: if self._streaming is streamer: self._streaming = None @@ -253,6 +286,7 @@ class Daemon: self._busy = True if streamer is not None: streamer.stop() + self._ov_meter_stop() if send_enter: from .paste import press_enter press_enter(win) @@ -284,6 +318,7 @@ class Daemon: self._active_workflow = None if streamer is not None: streamer.stop() + self._ov_meter_stop() self._emit("idle", None, "Cancelled") self._notify("Cancelled", "Streaming stopped.", "low") self._play_sound("device-removed") @@ -323,6 +358,14 @@ class Daemon: return text[:cut + 1] if cut >= 0 else "" def _on_stream_text(self, text: str, final: bool) -> None: + # Mirror the live hypothesis into the overlay bubble (the full current + # guess, which is more responsive than the delivered stable prefix). + if self.text_cb: + disp_seg = quality.clean(text, strip_trailing_punctuation=False) + running = (self._ov_text_final + " " + disp_seg).strip() + self.text_cb(running) + if final: + self._ov_text_final = running stable = self._stable_stream_text(text, final) if not stable: return @@ -343,6 +386,7 @@ class Daemon: self._stream_segment_text = "" if final else stable def _on_stream_error(self, label: str, exc: Exception) -> None: + self._ov_meter_stop() with self._lock: self._streaming = None self._active_workflow = None diff --git a/linux/blitztext/gtksettings.py b/linux/blitztext/gtksettings.py index 45c6aee..3ec6793 100644 --- a/linux/blitztext/gtksettings.py +++ b/linux/blitztext/gtksettings.py @@ -904,6 +904,10 @@ class SettingsDialog: self.gen_notify_routing = Gtk.Switch(); self.gen_notify_routing.set_active(self.cfg.notify_routing) _switch_row(page, "Announce matched preset", self.gen_notify_routing, "After a voice command, show which preset and keyword matched — even hands-free.") + self.gen_overlay = Gtk.Switch(); self.gen_overlay.set_active(self.cfg.overlay_enabled) + _switch_row(page, "Visual overlay", self.gen_overlay, + "Show a microphone, a live waveform, and the recognised text in a bubble " + "at the cursor while you dictate (also gives hands-free sessions feedback).") self.gen_boot = Gtk.Switch(); self.gen_boot.set_active(autostart.is_enabled()) _switch_row(page, "Launch on login", self.gen_boot, "Start Blitztext automatically when you log in.") @@ -1206,6 +1210,7 @@ class SettingsDialog: c.language = self.gen_lang.get_text().strip() c.notify = self.gen_notify.get_active() c.notify_routing = self.gen_notify_routing.get_active() + c.overlay_enabled = self.gen_overlay.get_active() c.device = self.stt_device.get_active_text() or "auto" c.compute_type = self.stt_compute.get_active_text() or "auto" autostart.set_enabled(self.gen_boot.get_active()) diff --git a/linux/blitztext/gtkui.py b/linux/blitztext/gtkui.py index 370d62c..2521ecc 100644 --- a/linux/blitztext/gtkui.py +++ b/linux/blitztext/gtkui.py @@ -85,10 +85,30 @@ class App: self.cfg = cfg self.tray_mode = tray_mode self.tray = None - self.daemon = Daemon(cfg, status_cb=self._status_cb) self._rows: dict[str, dict] = {} self._active: str | None = None + # On-screen dictation HUD (mic + waveform + recognised-text bubble). + self.overlay = None + self._ov_state = "idle" + self._ov_streaming = False + if cfg.overlay_enabled: + try: + from .overlay import Overlay + + self.overlay = Overlay(anchor_mode=cfg.overlay_anchor) + if cfg.overlay_anchor == "caret": + from . import caret + + caret.start_tracking() + except Exception: # noqa: BLE001 - overlay is optional eye-candy + self.overlay = None + + self.daemon = Daemon( + cfg, status_cb=self._status_cb, + level_cb=self._on_level, text_cb=self._on_text, + ) + _install_css() self._build_window() @@ -242,6 +262,44 @@ class App: return threading.Thread(target=lambda: self.daemon.toggle(wf), daemon=True).start() + # -- overlay feedback ----------------------------------------------------- + def _on_level(self, level: float) -> None: + if self.overlay is not None: + self.overlay.set_level(level) + + def _on_text(self, text: str) -> None: + if self.overlay is not None: + self.overlay.set_text(text) + + def _overlay_status(self, state: str, message: str) -> None: + """Translate engine phases into overlay show/update/hide (GTK thread).""" + ov = self.overlay + if ov is None: + return + if state in ("recording", "streaming"): + self._ov_streaming = state == "streaming" + self._ov_state = state + ov.show(state, getattr(self.daemon, "_target_window", None)) + elif state == "busy": + self._ov_state = state + ov.set_state("busy", message) + elif state == "done": + # Non-streaming: the 'done' message carries the final text. Streaming + # already showed it live, so don't overwrite with "Streaming stopped". + if not self._ov_streaming and message: + ov.set_text(message) + self._ov_state = state + ov.set_state("done", message) + elif state == "error": + self._ov_state = state + ov.set_state("error", message) + elif state == "idle": + # Only when a session was actually live — and never clip a 'done' + # linger (idle is emitted right after done in the worker's finally). + if self._ov_state in ("recording", "streaming", "busy"): + ov.set_state("idle", message) + self._ov_state = "idle" + # -- status (marshalled to GTK thread) ------------------------------------ def _status_cb(self, state: str, workflow: str | None, message: str) -> None: GLib.idle_add(self._apply_status, state, workflow, message) @@ -257,6 +315,8 @@ class App: self._set_dot(colors.get(state, "#7b818b")) self.status_lbl.set_text(labels.get(state, message)) + self._overlay_status(state, message) + if self.tray is not None: self.tray.update_status(state, labels.get(state, message)) @@ -312,6 +372,11 @@ class App: def quit_all(self) -> None: try: self.daemon.stop_input() + if self.overlay is not None: + self.overlay.destroy() + from . import caret + + caret.stop_tracking() finally: Gtk.main_quit() diff --git a/linux/blitztext/overlay.py b/linux/blitztext/overlay.py new file mode 100644 index 0000000..e37c92b --- /dev/null +++ b/linux/blitztext/overlay.py @@ -0,0 +1,382 @@ +"""On-screen dictation HUD: a translucent bubble at the cursor. + +Shows, while you dictate: + • a microphone glyph that pulses red as it listens, + • a live waveform driven by the real mic level, + • the recognised text (word-by-word in streaming mode, or the final result as + a brief confirmation in record-then-transcribe mode), +with a little tail whose tip points at the cursor where the text will land +(see :mod:`blitztext.caret` for how that anchor is resolved). + +It is a click-through, focus-free override-redirect window so it never steals +input from the field you're dictating into. All public methods are safe to call +from worker threads — they marshal onto the GTK main loop via ``GLib.idle_add``. +Pure feedback: nothing here touches recording, transcription, or delivery, and +the whole feature is gated by ``cfg.overlay_enabled`` in the caller. +""" + +from __future__ import annotations + +import math +import time +from collections import deque + +import cairo +import gi + +gi.require_version("Gtk", "3.0") +gi.require_version("Gdk", "3.0") +gi.require_version("PangoCairo", "1.0") +from gi.repository import Gdk, GLib, Gtk, Pango, PangoCairo # noqa: E402 + +from . import caret # noqa: E402 + +# Layout constants (logical px). +_WIDTH = 360 +_PAD = 16 +_HEADER_H = 40 # mic + waveform row +_RADIUS = 16 +_TAIL_W = 20 +_TAIL_H = 11 +_GAP = 12 # clearance between the tail tip and the anchor +_BARS = 30 # waveform bar count +_MIN_TEXT_H = 0 +_MAX_TEXT_H = 120 + +_FPS_MS = 33 # ~30 fps animation tick + +# Phase → (mic colour, label). Recording/streaming pulse; others are steady. +_PHASES = { + "recording": ((1.0, 0.27, 0.23), "Listening…"), + "streaming": ((1.0, 0.27, 0.23), "Listening…"), + "busy": ((1.0, 0.74, 0.16), "Transcribing…"), + "done": ((0.30, 0.80, 0.36), ""), + "error": ((1.0, 0.35, 0.35), "Error"), +} + + +class Overlay: + def __init__(self, anchor_mode: str = "caret") -> None: + self.anchor_mode = anchor_mode + self._visible = False + self._state = "recording" + self._text = "" + self._phase_label = "Listening…" + self._anchor: caret.Anchor | None = None + self._tail_up = False # tail on top edge (bubble below anchor)? + self._tail_x = _WIDTH // 2 # tail tip, window-local x + self._height = _HEADER_H + 2 * _PAD + _TAIL_H + self._levels: deque[float] = deque([0.0] * _BARS, maxlen=_BARS) + self._disp = [0.0] * _BARS # eased bar heights for smooth motion + self._pulse = 0.0 + self._tick_id: int | None = None + self._hide_id: int | None = None + self._t0 = time.time() + + self._win = Gtk.Window(type=Gtk.WindowType.POPUP) + self._win.set_app_paintable(True) + self._win.set_resizable(False) + self._win.set_skip_taskbar_hint(True) + self._win.set_skip_pager_hint(True) + self._win.set_accept_focus(False) + self._win.set_focus_on_map(False) + self._win.set_keep_above(True) + screen = self._win.get_screen() + visual = screen.get_rgba_visual() if screen else None + if visual is not None: + self._win.set_visual(visual) + self._area = Gtk.DrawingArea() + if visual is not None: + self._area.set_visual(visual) + self._area.connect("draw", self._on_draw) + self._win.add(self._area) + self._win.connect("realize", self._on_realize) + self._win.set_default_size(_WIDTH, self._height) + + # -- click-through -------------------------------------------------------- + def _on_realize(self, _w) -> None: + gdkwin = self._win.get_window() + if gdkwin is not None: + # Empty input region → the HUD ignores all clicks; they fall through + # to whatever is underneath (the field you're typing into). + gdkwin.input_shape_combine_region(cairo.Region(), 0, 0) + + # -- thread-safe public API ---------------------------------------------- + def show(self, state: str, window_id: str | None) -> None: + GLib.idle_add(self._show, state, window_id) + + def set_level(self, level: float) -> None: + GLib.idle_add(self._set_level, float(level)) + + def set_text(self, text: str) -> None: + GLib.idle_add(self._set_text, text or "") + + def set_state(self, state: str, message: str = "") -> None: + GLib.idle_add(self._set_state, state, message) + + def hide(self) -> None: + GLib.idle_add(self._hide) + + def destroy(self) -> None: + GLib.idle_add(self._destroy) + + # -- main-thread handlers ------------------------------------------------- + def _show(self, state: str, window_id: str | None) -> bool: + self._cancel_hide() + self._state = state + self._phase_label = _PHASES.get(state, ((1, 1, 1), ""))[1] + self._text = "" + self._levels = deque([0.0] * _BARS, maxlen=_BARS) + self._disp = [0.0] * _BARS + self._anchor = caret.resolve(self.anchor_mode, window_id) + self._relayout() + self._visible = True + self._win.show_all() + if self._tick_id is None: + self._tick_id = GLib.timeout_add(_FPS_MS, self._tick) + return False + + def _set_level(self, level: float) -> bool: + self._levels.append(max(0.0, min(1.0, level))) + return False + + def _set_text(self, text: str) -> bool: + if text == self._text: + return False + self._text = text + self._relayout() + self._area.queue_draw() + return False + + def _set_state(self, state: str, message: str) -> bool: + self._state = state + self._phase_label = _PHASES.get(state, ((1, 1, 1), message[:40]))[1] or message[:40] + if state in ("recording", "streaming", "busy"): + self._cancel_hide() + elif state in ("done", "idle", "error"): + # Linger on the final text, then fade out. Errors stay a touch longer. + delay = 2500 if state == "error" else (1600 if self._text else 700) + self._schedule_hide(delay) + self._area.queue_draw() + return False + + def _hide(self) -> bool: + self._visible = False + self._cancel_hide() + if self._tick_id is not None: + GLib.source_remove(self._tick_id) + self._tick_id = None + self._win.hide() + return False + + def _destroy(self) -> bool: + self._hide() + self._win.destroy() + return False + + # -- hide scheduling ------------------------------------------------------ + def _schedule_hide(self, delay_ms: int) -> None: + self._cancel_hide() + self._hide_id = GLib.timeout_add(delay_ms, self._hide) + + def _cancel_hide(self) -> None: + if self._hide_id is not None: + GLib.source_remove(self._hide_id) + self._hide_id = None + + # -- animation ------------------------------------------------------------ + def _tick(self) -> bool: + if not self._visible: + self._tick_id = None + return False + # Ease displayed bars toward the latest levels for fluid motion even + # though real levels arrive at ~10 Hz. + targets = list(self._levels) + for i, t in enumerate(targets): + self._disp[i] += (t - self._disp[i]) * 0.35 + self._pulse = (math.sin((time.time() - self._t0) * 5.0) + 1.0) * 0.5 + self._area.queue_draw() + return True + + # -- geometry ------------------------------------------------------------- + def _monitor_geo(self): + disp = Gdk.Display.get_default() + if self._anchor is not None: + mon = disp.get_monitor_at_point(self._anchor.x, self._anchor.y) + else: + mon = disp.get_primary_monitor() or disp.get_monitor(0) + return mon.get_geometry() + + def _text_height(self) -> int: + if not self._text: + return _MIN_TEXT_H + layout = self._win.create_pango_layout(self._text) + layout.set_width((_WIDTH - 2 * _PAD) * Pango.SCALE) + layout.set_wrap(Pango.WrapMode.WORD_CHAR) + font = Pango.FontDescription("Sans 11") + layout.set_font_description(font) + _w, h = layout.get_pixel_size() + return min(_MAX_TEXT_H, max(_MIN_TEXT_H, h)) + + def _relayout(self) -> None: + text_h = self._text_height() + gap_text = 8 if text_h else 0 + body_h = _HEADER_H + gap_text + text_h + 2 * _PAD + self._height = body_h + _TAIL_H + + geo = self._monitor_geo() + a = self._anchor + ax = a.x if a else geo.x + geo.width // 2 + ay = a.y if a else geo.y + geo.height - 140 + ah = a.height if a else 0 + + # Prefer placing the bubble above the anchor; flip below if it won't fit. + above_y = ay - _GAP - self._height + if above_y >= geo.y + 4: + self._tail_up = False + win_y = above_y + else: + self._tail_up = True + win_y = ay + ah + _GAP + + tip_x = max(geo.x + _RADIUS + _TAIL_W, min(ax, geo.x + geo.width - _RADIUS - _TAIL_W)) + win_x = tip_x - _WIDTH // 2 + win_x = max(geo.x + 6, min(win_x, geo.x + geo.width - _WIDTH - 6)) + self._tail_x = tip_x - win_x + + self._win.resize(_WIDTH, self._height) + self._win.move(int(win_x), int(win_y)) + self._area.set_size_request(_WIDTH, self._height) + + # -- drawing -------------------------------------------------------------- + def _on_draw(self, _area, cr) -> bool: + # Start fully transparent. + cr.set_operator(cairo.OPERATOR_SOURCE) + cr.set_source_rgba(0, 0, 0, 0) + cr.paint() + cr.set_operator(cairo.OPERATOR_OVER) + + w = _WIDTH + body_top = _TAIL_H if self._tail_up else 0 + body_h = self._height - _TAIL_H + body_bottom = body_top + body_h + + # Bubble + tail as one path, so the fill/stroke wrap the tail cleanly. + self._bubble_path(cr, 0, body_top, w, body_h) + cr.set_source_rgba(0.10, 0.11, 0.14, 0.94) + cr.fill_preserve() + cr.set_source_rgba(1, 1, 1, 0.08) + cr.set_line_width(1.0) + cr.stroke() + + cx = _PAD + 14 + cy = body_top + _PAD + 12 + self._draw_mic(cr, cx, cy) + + # Waveform fills the space right of the mic across the header row. + wf_x = cx + 26 + wf_w = w - _PAD - wf_x + self._draw_wave(cr, wf_x, body_top + _PAD, wf_w, _HEADER_H) + + # Phase label (top-right, small) when there's room and no text yet. + if self._phase_label and not self._text: + self._draw_label(cr, w - _PAD, body_top + _PAD + 12, self._phase_label) + + # Recognised text below the header row. + if self._text: + self._draw_text(cr, _PAD, body_top + _PAD + _HEADER_H + 8, w - 2 * _PAD) + return False + + def _bubble_path(self, cr, x, y, w, h) -> None: + r = _RADIUS + cr.new_sub_path() + cr.arc(x + w - r, y + r, r, -math.pi / 2, 0) + cr.arc(x + w - r, y + h - r, r, 0, math.pi / 2) + cr.arc(x + r, y + h - r, r, math.pi / 2, math.pi) + cr.arc(x + r, y + r, r, math.pi, 1.5 * math.pi) + cr.close_path() + # Tail: a small triangle on the top or bottom edge at self._tail_x. + tx = max(r + _TAIL_W, min(self._tail_x, w - r - _TAIL_W)) + if self._tail_up: + cr.move_to(tx - _TAIL_W / 2, y) + cr.line_to(tx, y - _TAIL_H) + cr.line_to(tx + _TAIL_W / 2, y) + else: + cr.move_to(tx - _TAIL_W / 2, y + h) + cr.line_to(tx, y + h + _TAIL_H) + cr.line_to(tx + _TAIL_W / 2, y + h) + cr.close_path() + + def _draw_mic(self, cr, cx, cy) -> None: + colour, _ = _PHASES.get(self._state, ((1, 1, 1), "")) + pulsing = self._state in ("recording", "streaming") + # Soft pulsing halo while listening. + if pulsing: + rad = 12 + self._pulse * 6 + cr.set_source_rgba(*colour, 0.18 * (1 - self._pulse * 0.6)) + cr.arc(cx, cy, rad, 0, 2 * math.pi) + cr.fill() + cr.set_source_rgba(*colour, 1.0) + cr.set_line_width(2.0) + # Capsule head. + head_w, head_top, head_bot = 9.0, cy - 11, cy + 1 + cr.arc(cx, head_top + head_w / 2, head_w / 2, math.pi, 2 * math.pi) + cr.arc(cx, head_bot - head_w / 2, head_w / 2, 0, math.pi) + cr.close_path() + cr.fill() + # Stand arc + post + base. + cr.set_source_rgba(*colour, 0.95) + cr.arc(cx, cy, 8, math.radians(25), math.radians(155)) + cr.stroke() + cr.move_to(cx, cy + 8) + cr.line_to(cx, cy + 12) + cr.stroke() + cr.move_to(cx - 5, cy + 12) + cr.line_to(cx + 5, cy + 12) + cr.stroke() + + def _draw_wave(self, cr, x, y, w, h) -> None: + gap = 2.0 + bw = max(1.5, (w - gap * (_BARS - 1)) / _BARS) + mid = y + h / 2 + listening = self._state in ("recording", "streaming") + for i in range(_BARS): + v = self._disp[i] if i < len(self._disp) else 0.0 + # Idle baseline shimmer so the meter never looks frozen. + if listening and v < 0.04: + v = 0.04 + 0.03 * math.sin((self._t0 - time.time()) * 4 + i * 0.5) + bh = max(2.0, v * (h - 2)) + bx = x + i * (bw + gap) + alpha = 0.85 if listening else 0.4 + cr.set_source_rgba(0.42, 0.62, 1.0, alpha) + self._round_rect(cr, bx, mid - bh / 2, bw, bh, min(bw / 2, 2)) + cr.fill() + + def _draw_label(self, cr, right_x, cy, text) -> None: + layout = self._win.create_pango_layout(text) + layout.set_font_description(Pango.FontDescription("Sans 9")) + tw, th = layout.get_pixel_size() + cr.set_source_rgba(1, 1, 1, 0.55) + cr.move_to(right_x - tw, cy - th / 2) + PangoCairo.show_layout(cr, layout) + + def _draw_text(self, cr, x, y, w) -> None: + layout = self._win.create_pango_layout(self._text) + layout.set_width(w * Pango.SCALE) + layout.set_wrap(Pango.WrapMode.WORD_CHAR) + layout.set_ellipsize(Pango.EllipsizeMode.END) + layout.set_height(_MAX_TEXT_H * Pango.SCALE) + layout.set_font_description(Pango.FontDescription("Sans 11")) + cr.set_source_rgba(0.95, 0.96, 0.99, 0.97) + cr.move_to(x, y) + PangoCairo.show_layout(cr, layout) + + @staticmethod + def _round_rect(cr, x, y, w, h, r) -> None: + r = min(r, w / 2, h / 2) + cr.new_sub_path() + cr.arc(x + w - r, y + r, r, -math.pi / 2, 0) + cr.arc(x + w - r, y + h - r, r, 0, math.pi / 2) + cr.arc(x + r, y + h - r, r, math.pi / 2, math.pi) + cr.arc(x + r, y + r, r, math.pi, 1.5 * math.pi) + cr.close_path()