feat: language metadata in benchmark — Lang column, filter support (v2.02.00)
- stt.py: add ModelMeta dataclass, fmt_languages(), list_models_meta(), detect_remote_device(); refactor list_models() to delegate - benchmark.py: add languages field to BenchRow; fetch via _get_langs() with URL-level caching using list_models_meta() - gtksettings.py: show language labels per engine in checkbox list; add language codes to search filter; add Lang column to results table Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
51ab6d5aff
commit
2df5be3212
@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host
|
|||||||
(not in a container) so it can type into any application via xdotool.
|
(not in a container) so it can type into any application via xdotool.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
__version__ = "2.01.03"
|
__version__ = "2.02.00"
|
||||||
|
|||||||
@ -23,6 +23,7 @@ class BenchRow:
|
|||||||
model: str
|
model: str
|
||||||
device: str # "CPU" | "CUDA" | "remote"
|
device: str # "CPU" | "CUDA" | "remote"
|
||||||
best_for: str # "Short clips" | "Short / medium" | "Long / batch" | "Streaming"
|
best_for: str # "Short clips" | "Short / medium" | "Long / batch" | "Streaming"
|
||||||
|
languages: list[str] # ISO 639-1 codes from /v1/models, empty if unknown
|
||||||
ok: bool
|
ok: bool
|
||||||
seconds: float
|
seconds: float
|
||||||
wer: float
|
wer: float
|
||||||
@ -111,6 +112,19 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
|||||||
run_list.append(e)
|
run_list.append(e)
|
||||||
|
|
||||||
device_cache: dict = {}
|
device_cache: dict = {}
|
||||||
|
meta_cache: dict = {} # url → list[ModelMeta]
|
||||||
|
|
||||||
|
def _get_langs(e) -> list[str]:
|
||||||
|
if e.is_local:
|
||||||
|
return []
|
||||||
|
url = e.url
|
||||||
|
if url not in meta_cache:
|
||||||
|
meta_cache[url] = stt.list_models_meta(url, e.api_key_env)
|
||||||
|
for m in meta_cache[url]:
|
||||||
|
if not e.model or m.id == e.model or m.id.endswith("/" + e.model):
|
||||||
|
return m.languages
|
||||||
|
return meta_cache[url][0].languages if meta_cache[url] else []
|
||||||
|
|
||||||
rows: list[BenchRow] = []
|
rows: list[BenchRow] = []
|
||||||
for e in run_list:
|
for e in run_list:
|
||||||
tr = get_local_transcriber(e) if (e.is_local and get_local_transcriber) else None
|
tr = get_local_transcriber(e) if (e.is_local and get_local_transcriber) else None
|
||||||
@ -122,6 +136,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
|||||||
model=e.model or ("local" if e.is_local else "(default)"),
|
model=e.model or ("local" if e.is_local else "(default)"),
|
||||||
device=_engine_device(e, tr, device_cache),
|
device=_engine_device(e, tr, device_cache),
|
||||||
best_for=_engine_best_for(e),
|
best_for=_engine_best_for(e),
|
||||||
|
languages=_get_langs(e),
|
||||||
ok=res.ok,
|
ok=res.ok,
|
||||||
seconds=res.seconds,
|
seconds=res.seconds,
|
||||||
wer=w,
|
wer=w,
|
||||||
|
|||||||
@ -1919,7 +1919,8 @@ notebook.bt-nb tab:checked label {
|
|||||||
|
|
||||||
self._bench_checks: dict[str, Gtk.CheckButton] = {}
|
self._bench_checks: dict[str, Gtk.CheckButton] = {}
|
||||||
self._bench_dots: dict[str, Gtk.Label] = {}
|
self._bench_dots: dict[str, Gtk.Label] = {}
|
||||||
self._bench_sel_rows: list[tuple[Gtk.Box, str, str]] = [] # (row, name, search_text)
|
self._bench_lang_labels: dict[str, Gtk.Label] = {}
|
||||||
|
self._bench_sel_rows: list[tuple[Gtk.Box, str, list[str]]] = [] # (row, name, mutable_search)
|
||||||
|
|
||||||
for e in self.cfg.stt_engines:
|
for e in self.cfg.stt_engines:
|
||||||
row = Gtk.Box(spacing=6)
|
row = Gtk.Box(spacing=6)
|
||||||
@ -1933,24 +1934,51 @@ notebook.bt-nb tab:checked label {
|
|||||||
cb.set_active(True)
|
cb.set_active(True)
|
||||||
self._bench_checks[e.name] = cb
|
self._bench_checks[e.name] = cb
|
||||||
row.pack_start(cb, True, True, 0)
|
row.pack_start(cb, True, True, 0)
|
||||||
|
lang_lbl = Gtk.Label(xalign=1.0)
|
||||||
|
lang_lbl.set_markup(f"<small><span foreground='{GREY}'>—</span></small>")
|
||||||
|
lang_lbl.set_margin_end(4)
|
||||||
|
self._bench_lang_labels[e.name] = lang_lbl
|
||||||
|
row.pack_end(lang_lbl, False, False, 0)
|
||||||
sel_list.pack_start(row, False, False, 2)
|
sel_list.pack_start(row, False, False, 2)
|
||||||
search_text = f"{e.name} {e.model} {e.url}".lower()
|
search_parts = [e.name, e.model, e.url]
|
||||||
self._bench_sel_rows.append((row, e.name, search_text))
|
self._bench_sel_rows.append((row, e.name, search_parts))
|
||||||
|
|
||||||
def _filter_bench(_e=None):
|
def _filter_bench(_e=None):
|
||||||
q = self._bench_filter.get_text().lower()
|
q = self._bench_filter.get_text().lower()
|
||||||
for row, _name, stext in self._bench_sel_rows:
|
for row, _name, parts in self._bench_sel_rows:
|
||||||
row.set_visible(not q or q in stext)
|
row.set_visible(not q or any(q in p.lower() for p in parts))
|
||||||
self._bench_filter.connect("changed", _filter_bench)
|
self._bench_filter.connect("changed", _filter_bench)
|
||||||
|
|
||||||
# Background reachability check for all engines
|
# Background: reachability + language metadata
|
||||||
def _check_bench_status():
|
def _check_bench_status():
|
||||||
|
meta_cache: dict[str, list] = {}
|
||||||
for e in self.cfg.stt_engines:
|
for e in self.cfg.stt_engines:
|
||||||
ok = stt.status(e, timeout=2.0)
|
ok = stt.status(e, timeout=2.0)
|
||||||
color = GREEN if ok else RED
|
color = GREEN if ok else RED
|
||||||
dot = self._bench_dots.get(e.name)
|
dot = self._bench_dots.get(e.name)
|
||||||
if dot:
|
if dot:
|
||||||
GLib.idle_add(dot.set_markup, _dot(color))
|
GLib.idle_add(dot.set_markup, _dot(color))
|
||||||
|
# Fetch language metadata for remote engines
|
||||||
|
if not e.is_local and e.url:
|
||||||
|
if e.url not in meta_cache:
|
||||||
|
meta_cache[e.url] = stt.list_models_meta(e.url, e.api_key_env, timeout=4.0)
|
||||||
|
langs: list[str] = []
|
||||||
|
for m in meta_cache[e.url]:
|
||||||
|
if not e.model or m.id == e.model or m.id.endswith("/" + e.model):
|
||||||
|
langs = m.languages
|
||||||
|
break
|
||||||
|
if not langs and meta_cache[e.url]:
|
||||||
|
langs = meta_cache[e.url][0].languages
|
||||||
|
if langs:
|
||||||
|
# Update search parts so filter works on language codes
|
||||||
|
for row, name, parts in self._bench_sel_rows:
|
||||||
|
if name == e.name:
|
||||||
|
parts.extend(langs)
|
||||||
|
lang_str = stt.fmt_languages(langs)
|
||||||
|
lbl = self._bench_lang_labels.get(e.name)
|
||||||
|
if lbl:
|
||||||
|
GLib.idle_add(lbl.set_markup,
|
||||||
|
f"<small><span foreground='{GREY}'>{GLib.markup_escape_text(lang_str)}</span></small>")
|
||||||
threading.Thread(target=_check_bench_status, daemon=True).start()
|
threading.Thread(target=_check_bench_status, daemon=True).start()
|
||||||
|
|
||||||
# ── Run controls ──────────────────────────────────────────────────────
|
# ── Run controls ──────────────────────────────────────────────────────
|
||||||
@ -1970,22 +1998,29 @@ notebook.bt-nb tab:checked label {
|
|||||||
page.pack_start(run_row, False, False, 0)
|
page.pack_start(run_row, False, False, 0)
|
||||||
|
|
||||||
# ── Resizable pane: engine list (top) ↕ results table (bottom) ───────
|
# ── Resizable pane: engine list (top) ↕ results table (bottom) ───────
|
||||||
self.bench_store = Gtk.ListStore(str, str, str, str, str, str, str, str, str)
|
# engine, url, model, device, best_for, lang, time, accuracy, output, tooltip
|
||||||
|
self.bench_store = Gtk.ListStore(str, str, str, str, str, str, str, str, str, str)
|
||||||
bench_sort = Gtk.TreeModelSort(model=self.bench_store)
|
bench_sort = Gtk.TreeModelSort(model=self.bench_store)
|
||||||
tree = Gtk.TreeView(model=bench_sort)
|
tree = Gtk.TreeView(model=bench_sort)
|
||||||
tree.set_has_tooltip(True)
|
tree.set_has_tooltip(True)
|
||||||
tree.set_tooltip_column(8)
|
tree.set_tooltip_column(9)
|
||||||
for title, i, expand in [("Engine", 0, False), ("URL", 1, False),
|
for title, i, expand, max_w in [
|
||||||
("Model", 2, False), ("Device", 3, False),
|
("Engine", 0, False, 0),
|
||||||
("Best for", 4, False), ("Time (s)", 5, False),
|
("URL", 1, False, 180),
|
||||||
("Accuracy", 6, False), ("Output", 7, True)]:
|
("Model", 2, False, 0),
|
||||||
|
("Device", 3, False, 0),
|
||||||
|
("Best for", 4, False, 0),
|
||||||
|
("Lang", 5, False, 160),
|
||||||
|
("Time (s)", 6, False, 0),
|
||||||
|
("Accuracy", 7, False, 0),
|
||||||
|
("Output", 8, True, 0)]:
|
||||||
r = Gtk.CellRendererText()
|
r = Gtk.CellRendererText()
|
||||||
r.set_property("ellipsize", Pango.EllipsizeMode.END)
|
r.set_property("ellipsize", Pango.EllipsizeMode.END)
|
||||||
col = Gtk.TreeViewColumn(title, r, text=i); col.set_resizable(True)
|
col = Gtk.TreeViewColumn(title, r, text=i); col.set_resizable(True)
|
||||||
col.set_sort_column_id(i)
|
col.set_sort_column_id(i)
|
||||||
col.set_expand(expand)
|
col.set_expand(expand)
|
||||||
if i == 1:
|
if max_w:
|
||||||
col.set_max_width(180)
|
col.set_max_width(max_w)
|
||||||
tree.append_column(col)
|
tree.append_column(col)
|
||||||
tree_sw = Gtk.ScrolledWindow()
|
tree_sw = Gtk.ScrolledWindow()
|
||||||
tree_sw.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
|
tree_sw.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
|
||||||
@ -2136,8 +2171,9 @@ notebook.bt-nb tab:checked label {
|
|||||||
tooltip = row.error
|
tooltip = row.error
|
||||||
# Strip scheme from URL for display brevity (http://192.168.1.1:8080 → 192.168.1.1:8080)
|
# Strip scheme from URL for display brevity (http://192.168.1.1:8080 → 192.168.1.1:8080)
|
||||||
url_display = row.url.removeprefix("https://").removeprefix("http://").rstrip("/")
|
url_display = row.url.removeprefix("https://").removeprefix("http://").rstrip("/")
|
||||||
|
lang_display = stt.fmt_languages(row.languages)
|
||||||
self.bench_store.append([row.engine, url_display, row.model, row.device, row.best_for,
|
self.bench_store.append([row.engine, url_display, row.model, row.device, row.best_for,
|
||||||
f"{row.seconds:.2f}", acc, out_friendly, tooltip])
|
lang_display, f"{row.seconds:.2f}", acc, out_friendly, tooltip])
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def _bench_done(self, rows) -> bool:
|
def _bench_done(self, rows) -> bool:
|
||||||
|
|||||||
@ -61,8 +61,31 @@ def status(engine: STTEngine, timeout: float = 2.0) -> bool:
|
|||||||
return reachable(engine.url, timeout)
|
return reachable(engine.url, timeout)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ModelMeta:
|
||||||
|
"""Model id plus optional metadata (languages, etc.) from the server."""
|
||||||
|
id: str
|
||||||
|
languages: list[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
def fmt_languages(langs: list[str]) -> str:
|
||||||
|
"""Compact display string for a language list, e.g. 'en, de, fr +45'."""
|
||||||
|
if not langs:
|
||||||
|
return "—"
|
||||||
|
if len(langs) >= 50:
|
||||||
|
return f"multilingual ({len(langs)})"
|
||||||
|
if len(langs) > 5:
|
||||||
|
return f"{', '.join(langs[:5])} +{len(langs) - 5}"
|
||||||
|
return ", ".join(langs)
|
||||||
|
|
||||||
|
|
||||||
def list_models(base_url: str, api_key_env: str = "", timeout: float = 5.0) -> list[str]:
|
def list_models(base_url: str, api_key_env: str = "", timeout: float = 5.0) -> list[str]:
|
||||||
"""Fetch model ids from an OpenAI-compatible, Ollama-style, or Riva/NIM /models endpoint."""
|
"""Fetch model ids from an OpenAI-compatible, Ollama-style, or Riva/NIM /models endpoint."""
|
||||||
|
return [m.id for m in list_models_meta(base_url, api_key_env, timeout)]
|
||||||
|
|
||||||
|
|
||||||
|
def list_models_meta(base_url: str, api_key_env: str = "", timeout: float = 5.0) -> list[ModelMeta]:
|
||||||
|
"""Like list_models() but returns ModelMeta with language info when available."""
|
||||||
import os
|
import os
|
||||||
|
|
||||||
if not base_url:
|
if not base_url:
|
||||||
@ -81,26 +104,27 @@ def list_models(base_url: str, api_key_env: str = "", timeout: float = 5.0) -> l
|
|||||||
except (urllib.error.URLError, json.JSONDecodeError, OSError):
|
except (urllib.error.URLError, json.JSONDecodeError, OSError):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 1. Standard OpenAI /models
|
# 1. Standard OpenAI /models — faster-whisper-server also returns "language"
|
||||||
data = _get(base + "/models")
|
data = _get(base + "/models")
|
||||||
if isinstance(data, dict):
|
if isinstance(data, dict):
|
||||||
items = data.get("data")
|
items = data.get("data")
|
||||||
if isinstance(items, list): # OpenAI shape: {"data":[{"id":...}]}
|
if isinstance(items, list):
|
||||||
return [m["id"] for m in items if isinstance(m, dict) and m.get("id")]
|
result = [ModelMeta(id=m["id"], languages=m.get("language") or [])
|
||||||
|
for m in items if isinstance(m, dict) and m.get("id")]
|
||||||
|
if result:
|
||||||
|
return result
|
||||||
items = data.get("models")
|
items = data.get("models")
|
||||||
if isinstance(items, list): # Ollama shape: {"models":[{"name"/"model":...}]}
|
if isinstance(items, list): # Ollama shape
|
||||||
return [m.get("name") or m.get("model") for m in items
|
return [ModelMeta(id=m.get("name") or m.get("model", ""))
|
||||||
if (m.get("name") or m.get("model"))]
|
for m in items if m.get("name") or m.get("model")]
|
||||||
|
|
||||||
# 2. NVIDIA Riva / NIM — exposes model info at /metadata
|
# 2. NVIDIA Riva / NIM
|
||||||
data = _get(base + "/metadata")
|
data = _get(base + "/metadata")
|
||||||
if isinstance(data, dict):
|
if isinstance(data, dict):
|
||||||
for info in data.get("modelInfo") or []:
|
for info in data.get("modelInfo") or []:
|
||||||
name = info.get("shortName") or info.get("modelUrl") or ""
|
name = info.get("shortName") or info.get("modelUrl") or ""
|
||||||
if name:
|
if name:
|
||||||
# Strip the long tag suffix: keep everything before the first ':'
|
return [ModelMeta(id=name.split(":")[0])]
|
||||||
short = name.split(":")[0]
|
|
||||||
return [short]
|
|
||||||
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user