From 840e70463af108cd202b6f54d372a898fa4c6b51 Mon Sep 17 00:00:00 2001 From: oztturk Date: Fri, 28 Aug 2026 16:16:45 +0300 Subject: [PATCH] Say what the local models are running on "Use the graphics card" was a flag and nothing else: whisper got -ng when it was off, llama got -ngl 99 or 0, and nobody looked at what happened next. A build without a GPU backend runs on the processor while the box stays ticked, which is what the release archives for Linux do, every time. Nothing anywhere reported whether the local server was even up. Both servers already say where the model went, in the log Dikte captures. It is read back once the server reports ready and turned into a backend, a card and, for llama, the layers it offloaded. The verdict comes from what whisper committed to -- "using X backend" and the model buffer -- rather than from the devices it merely listed: a card that is found and then fails to initialise sends it back to the processor, and the listing alone would have called that a graphics card. A log that says nothing stays "could not tell" instead of being guessed at; a hand-built macOS whisper has Metal compiled in and prints no backend line at all. The state is then somewhere to be seen. Server.state() is a snapshot of the process and what it settled on, and it reaches `dikte status`, `dikte doctor` and a line under each local model box in the settings window. doctor reads the log from disk when no instance is running, so it still answers on a machine where Dikte is closed, and it says which of the three it is: what the last run used, that the last run said nothing, or that none ever ran here. Where the card was asked for and not obtained, the line says which of the two it is -- none was found, or this build carries none -- because only the second is worth replacing a download over. The tests grew a second isolation. They read ggml's own data directory, which is the real one on the machine running them, and program_path prefers a whisper-server on the PATH, so the suite answered from whatever the developer happened to have installed. Both are now the test's own. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019zqCqhmeaNT6m1GPp8ZWPp --- dikte/app.py | 15 +++ dikte/cli.py | 107 ++++++++++++++++-- dikte/ggml.py | 250 ++++++++++++++++++++++++++++++++++++++++++- dikte/i18n.py | 15 +++ dikte/settings_ui.py | 63 ++++++++++- tests/support.py | 17 +++ tests/test_cli.py | 108 +++++++++++++++++++ tests/test_ggml.py | 230 +++++++++++++++++++++++++++++++++++++++ tests/test_ui.py | 56 +++++++++- 9 files changed, 851 insertions(+), 10 deletions(-) diff --git a/dikte/app.py b/dikte/app.py index d321256..1fbd24b 100644 --- a/dikte/app.py +++ b/dikte/app.py @@ -604,6 +604,10 @@ class Dikte: "agent": assistant.display_name(self.conf), "provider": assistant.provider(self.conf), "listener": self.evdev.running, + # Whether each model on this machine is loaded, and what it ended up + # running on. Only this process knows: the servers are its children, + # and the command line has no way to ask them anything. + "local": self._local_state(), # Asked here rather than by the command line, because on macOS # there is no registry to read: a combination is held by this # process and by nothing else, so this is the only process that @@ -612,6 +616,17 @@ class Dikte: for name, spec in hotkey.SHORTCUTS.items()}, } + def _local_state(self): + """ggml.state(), with a mark for the servers this setup actually uses. + + A server that is neither wanted nor loaded is not worth a line anywhere; + one that is wanted and not loaded is exactly the line worth reading. + """ + local = ggml.state() + local["whisper"]["used"] = self.conf["transcribe_provider"] == "local" + local["llama"]["used"] = self.conf.uses_local_llm() + return local + def reload_settings(self): """Read the config file back after something outside changed it.""" self.conf.load() diff --git a/dikte/cli.py b/dikte/cli.py index c1ef786..0900c76 100644 --- a/dikte/cli.py +++ b/dikte/cli.py @@ -30,6 +30,7 @@ from . import audio from . import cleanup from . import config as cfg from . import filetranscribe +from . import ggml from . import hotkey from . import hub from . import ipc @@ -836,6 +837,68 @@ def cmd_update(opts): f"{release.url}") +# --- the models on this machine -------------------------------------------- + + +def _local_where(entry): + """Where a local model ran, in a phrase: the card, the processor, or neither. + + The backend and the card keep the names the server printed for them. A + graphics card is a product somebody sells under that name, and translating + it would be inventing hardware. + """ + kind = ggml.accel_kind({**entry, "running": True}) + where = {"gpu": "the graphics card", "cpu": "the processor"}.get( + kind, "something it did not name") + detail = ggml.accel_detail(entry) + return where + (f" ({detail})" if detail else "") + + +def _local_note(entry): + """Why the card is not in use, for a setup that asked for it and got none.""" + if not entry.get("gpu_wanted"): + return "" + if ggml.accel_kind({**entry, "running": True}) != "cpu": + return "" + return (" - the graphics card is switched on and this build carries none" + if ggml.cpu_only_build(entry) + else " - the graphics card is switched on and none was found") + + +def _local_line(entry): + if not entry.get("running"): + return "not loaded" + model = entry.get("model") or "" + return (f"loaded on {_local_where(entry)}" + + (f", {model}" if model else "") + _local_note(entry)) + + +def _last_local(conf): + """What the local servers last ran on, read off the logs they left behind. + + For a command line asking while nothing is running: there is no process to + put the question to, and the log outlives the process that wrote it. Every + entry says `running` is false, because this is an account of the last start + rather than a reading of a live one. + """ + rows = {} + for program, used, gpu in ( + (ggml.WHISPER, conf["transcribe_provider"] == "local", + bool(conf["local_gpu"])), + (ggml.LLAMA, conf.uses_local_llm(), bool(conf["local_llm_gpu"]))): + accel = ggml.last_accel(program) + rows[program.name] = { + # Whether one ever started here at all, which the backend cannot + # say on its own: a server that ran and named no backend and one + # that never ran both leave it empty. + "ran": ggml.server_log(program).exists(), + "running": False, "used": used, "gpu_wanted": gpu, + "backend": accel.backend, "device": accel.device, + "layers": accel.layers, "available": list(accel.available), + } + return rows + + def cmd_status(opts): reply = ipc.send("status") if reply is None: @@ -855,6 +918,11 @@ def cmd_status(opts): + (f" {reply['meeting_message']}" if reply.get("meeting_message") else ""), f"listener: {'on' if reply.get('listener') else 'off'}", ] + # Nothing for a setup that uses no model on this machine, and nothing at all + # from an instance too old to have been asked. + for name, entry in (reply.get("local") or {}).items(): + if entry.get("used") or entry.get("running"): + lines.append(f"{name + ':':11}{_local_line(entry)}") return out(opts, reply, "\n".join(lines)) @@ -867,6 +935,9 @@ def cmd_doctor(opts): # Mac shells out for one half and Windows for neither. A row saying ydotool # is missing on a machine that would never have run it is not a diagnosis, # it is a red mark to explain away. + # Asked once and read twice: whether an instance is running, and what its + # local servers are doing, which is a question only that process can answer. + live = ipc.send("status") or {} here = paste.desktop() wanted = [here.clipboard, here.keyboard] if sys.platform.startswith("linux"): @@ -908,8 +979,14 @@ def cmd_doctor(opts): "ready": cleanup_ready}, "agent": {"provider": assistant.provider(conf), "directory": assistant.working_dir(conf)}, - "running": ipc.send("status") is not None, + "running": bool(live), + # Live when there is an instance to ask, off the logs when there is not. + "local": live.get("local") or _last_local(conf), } + # An instance from before this field existed is not an instance saying + # nothing is loaded; it is one that cannot be asked, and the two must not + # print the same line. + stale = bool(live) and "local" not in live if target.provider == "local": transcribe_line = (f"{'✓' if transcribe_ready else '✗'} {target.service}, " f"transcribing on {target.model or 'no model yet'}") @@ -929,12 +1006,30 @@ def cmd_doctor(opts): f"{cleanup.model(conf)}") lines = [f"{'✓' if path else '✗'} {name:14} {path or 'not on your PATH'}" for name, path in programs.items()] - lines += [ - transcribe_line, - cleanup_line, + lines += [transcribe_line, cleanup_line] + # Only the models this setup actually uses: a machine transcribing in the + # cloud has nothing loaded here and no reason to read about it. + for name, entry in checks["local"].items(): + if not entry.get("used"): + continue + if stale: + lines.append(f"· {name:14} the running instance is too old to say; " + f"reload it with: dikte restart") + elif entry.get("running"): + lines.append(f"✓ {name:14} {_local_line(entry)}") + elif live: + lines.append(f"· {name:14} not loaded") + elif entry.get("backend"): + lines.append(f"· {name:14} last run on " + f"{_local_where(entry)}{_local_note(entry)}") + elif entry.get("ran"): + lines.append(f"· {name:14} last run said nothing about what it " + f"was running on") + else: + lines.append(f"· {name:14} never run here") + lines.append( f"{'✓' if checks['running'] else '·'} application " - + ("running" if checks["running"] else "not running"), - ] + + ("running" if checks["running"] else "not running")) return out(opts, {"ok": True, **checks}, "\n".join(lines)) diff --git a/dikte/ggml.py b/dikte/ggml.py index c2f8da8..51229ef 100644 --- a/dikte/ggml.py +++ b/dikte/ggml.py @@ -33,6 +33,7 @@ import json import os import pathlib import platform +import re import shutil import signal import socket @@ -606,6 +607,142 @@ def _tail(path, lines=3): return " | ".join(found[-lines:]) +# --- what the server is running on ---------------------------------------- + +# Both programs say where the model went, and neither is asked: it is printed +# while they start and captured in the log Dikte already keeps. Reading it back +# is the only way to tell a graphics card that was asked for from one that was +# found, which is a difference the settings checkbox cannot make on its own. +Accel = collections.namedtuple("Accel", "backend device layers available") + +NO_ACCEL = Accel("", "", "", ()) + +# ggml loads each backend from a shared object and says which; whisper then says +# whether it found a card, and llama says how many layers went onto it. +_BACKEND_LOADED = re.compile(r"^load_backend: loaded (\w+) backend", re.M) +_WHISPER_NO_GPU = "whisper_backend_init_gpu: no GPU found" +# What whisper committed to, which is not what it enumerated: it lists every +# device it can see, then tries them, and a card that fails to initialise sends +# it back to the processor. "using" is printed only once one has worked, and the +# model buffer is named after wherever the weights actually ended up. +_WHISPER_USING = re.compile( + r"^whisper_backend_init_gpu: using (\S+) backend", re.M) +_WHISPER_BUFFER = re.compile(r"^whisper_model_load:\s+(\S+) total size", re.M) +# The device listing, read for the card's name rather than for the verdict. +_WHISPER_DEVICE = re.compile( + r"^whisper_backend_init_gpu: device (\d+): (.+?) \(type: (\d+)\)", re.M) +_LLAMA_OFFLOAD = re.compile( + r"^load_tensors: offloaded (\d+)/(\d+) layers to GPU", re.M) + +# whisper names the device by its ggml handle, "Vulkan0" or "CUDA0", which says +# which slot rather than which card. Each backend prints the real name as it +# enumerates, one line further up. +_HANDLE = re.compile(r"^([A-Za-z]+?)(\d*)$") +_BARE = re.compile(r"^(?:Vulkan|CUDA|ROCm|SYCL|Metal|GPU|CPU)\d*$", re.I) +_METAL_DEVICE = re.compile(r"^ggml_metal.*picking default device: (.+)$", re.M) +# The driver in brackets after the card's own name: "(radv)", "(nvidia)". The +# name carries brackets of its own, but in capitals, so the case is what tells +# a driver tag from part of the name. +_DRIVER_TAG = re.compile(r"\s*\([a-z0-9_.\- ]+\)$") + + +def _enumerated(text, backend, index): + """The name the backend printed for one of its own devices, by slot. + + Asked by backend rather than by whichever listing came first: a machine + with both a CUDA build and a Vulkan loader prints two listings, and the + card named in the wrong one is somebody else's card. + """ + listings = { + "vulkan": rf"^ggml_vulkan: {index} = (.+?) \| ", + "cuda": rf"^\s*Device {index}: (.+?), compute capability", + "rocm": rf"^\s*Device {index}: (.+?), compute capability", + } + pattern = listings.get(backend.lower()) + found = re.search(pattern, text, re.M) if pattern else None + if found is None and backend.lower() == "metal": + found = _METAL_DEVICE.search(text) + if found is None: + return "" + return _DRIVER_TAG.sub("", found.group(1).strip()) + + +def _card_name(text, handle): + """The card behind a ggml handle like "Vulkan0", named the way it sells. + + Three places carry a name and only the third is always meaningful: the + handle says which slot, whisper's own device listing says whichever the + backend reported, and the backend's enumeration says what the thing is + called. Whichever of them is not just the handle again wins. + """ + parts = _HANDLE.match(handle or "") + backend, index = (parts.group(1), parts.group(2) or "0") if parts else ("", "0") + for slot, name, _kind in _WHISPER_DEVICE.findall(text): + if slot == index and name.strip() and not _BARE.match(name.strip()): + return name.strip() + # The handle itself is not an answer: it says which slot, and a line + # reading "Vulkan, Vulkan0" tells nobody which card is doing the work. + return _enumerated(text, backend, index) + +# The startup chatter is the first few hundred lines; the rest of the file is a +# line per request and grows for as long as the server lives. +_LOG_HEAD = 64 << 10 + + +def _read_accel(program, log_path): + """What the server that wrote `log_path` is running on. + + An empty backend is a real answer rather than a failure: a whisper built by + hand on a Mac has Metal compiled in and prints no load_backend line at all, + and calling that "the processor" would be a confident lie about the one + thing this is here to be honest about. + """ + try: + with open(log_path, encoding="utf-8", errors="replace") as fh: + text = fh.read(_LOG_HEAD) + except OSError: + return NO_ACCEL + # dict.fromkeys rather than a set: the order they were loaded in is the + # order they are worth showing in, and CPU is always one of them. + available = tuple(dict.fromkeys(_BACKEND_LOADED.findall(text))) + cards = [name for name in available if name.upper() != "CPU"] + if program is WHISPER: + # The handle whisper settled on. "using" is printed once a backend has + # initialised, so a card that was listed and then failed never reaches + # here; the model buffer is the same answer from the other end, named + # after wherever the weights were actually put. + using = _WHISPER_USING.search(text) + buffered = _WHISPER_BUFFER.search(text) + handle = (using or buffered).group(1) if (using or buffered) else "" + if _WHISPER_NO_GPU in text or handle.upper().startswith("CPU"): + return Accel("CPU", handle or "", "", available) + if handle: + parts = _HANDLE.match(handle) + backend = parts.group(1) if parts else "" + # The backend as the build spells it, so "Vulkan" rather than the + # capitalisation the handle happened to use. + backend = next((name for name in cards + if name.lower() == backend.lower()), backend) + return Accel(backend or "GPU", _card_name(text, handle), "", + available) + if available and not cards: + # Nothing but a processor backend in this build: there was nowhere + # else the model could have gone. + return Accel("CPU", "", "", available) + return Accel("", "", "", available) + found = _LLAMA_OFFLOAD.search(text) + if found: + layers = f"{found.group(1)}/{found.group(2)}" + if int(found.group(1)) > 0: + backend = cards[0] if cards else "GPU" + return Accel(backend, _card_name(text, f"{backend}0"), layers, + available) + return Accel("CPU", "", layers, available) + if available and not cards: + return Accel("CPU", "", "", available) + return Accel("", "", "", available) + + def _win_image_name(pid): """The full, lower-cased path of the process's executable, or ''. @@ -659,6 +796,15 @@ class Server: self._port = 0 self._log = "" self._key = None + # What the running child settled on, read out of its log once it was + # ready. Kept beside the process because it belongs to that process and + # to no other: a restart on new settings may land somewhere else. + self._accel = NO_ACCEL + # The settings the running child was started on, which is not what + # _settings holds: a change made while a start is in flight lands there + # first, and reporting the new model beside the old process would name + # a model this server is not running. + self._live = {} # The pid this instance last wrote to its pid file, so _forget never # removes a file some other Dikte wrote after us. self._pid = 0 @@ -694,6 +840,32 @@ class Server: with self._lock: return f"http://{HOST}:{self._port}/v1" if self._port else "" + def state(self): + """A snapshot of what this server is doing, for something to show. + + Taken under the short lock rather than the start one, so the interface + is answered at once even while a model is being read in. Plain types + throughout, because this travels over the socket to the command line. + """ + with self._lock: + up = self._proc is not None and self._proc.poll() is None + accel = self._accel if up else NO_ACCEL + # What it is running, when it is running; what it would run + # otherwise. The two differ for as long as a change waits for the + # restart that will pick it up. + settings = self._live if up else self._settings + return { + "running": up, + "pid": self._proc.pid if up else 0, + "port": self._port if up else 0, + "model": settings.get("model", ""), + "gpu_wanted": bool(settings.get("gpu")), + "backend": accel.backend, + "device": accel.device, + "layers": accel.layers, + "available": list(accel.available), + } + def error(self): """The last thing the server printed, for a failure after it started.""" with self._lock: @@ -715,9 +887,10 @@ class Server: self._stop_now() with self._lock: settings, key = dict(self._settings), self._settings_key() - proc, port, log = self._launch(settings) + proc, port, log, accel = self._launch(settings) with self._lock: self._proc, self._port, self._log, self._key = proc, port, log, key + self._accel, self._live = accel, settings return self.base_url() def _current_url(self): @@ -767,7 +940,9 @@ class Server: self._forget() raise if reason == "ready": - return proc, port, str(log) + # Read now rather than on demand: the startup lines are at the + # head of a file a long-lived server keeps appending to. + return proc, port, str(log), _read_accel(self.program, log) last = _tail(log) self._forget() # Losing the port between the probe and the bind is the one @@ -843,6 +1018,7 @@ class Server: with self._lock: proc, self._proc = self._proc, None self._port, self._log, self._key = 0, "", None + self._accel, self._live = NO_ACCEL, {} self._kill(proc, gently=True) if proc is not None: self._forget() @@ -1026,6 +1202,76 @@ def sweep(): return any([server.sweep() for server in SERVERS]) +def state(): + """What each local server is doing, keyed by program name.""" + return {server.program.name: server.state() for server in SERVERS} + + +def server_log(program): + """Where this program's server writes, which outlives the process.""" + return DATA_DIR / f"{program.name}-server.log" + + +def last_accel(program): + """What the last server for `program` ran on, from the log it left behind. + + For a command line asking with nothing running: the log outlives the process + and is the only account of the last start there is. + """ + return _read_accel(program, server_log(program)) + + +def accel_kind(state): + """"off" | "gpu" | "cpu" | "unknown", for a state() or an Accel. + + A tag rather than a sentence, because the two places that show this write + their own: the command line answers in English and the settings window in + whatever language it was opened in. + """ + if isinstance(state, Accel): + state = {"running": True, "backend": state.backend} + if not state.get("running"): + return "off" + backend = state.get("backend") or "" + if not backend: + return "unknown" + return "cpu" if backend.upper() == "CPU" else "gpu" + + +def accel_detail(state): + """The backend, the card and the layers, joined, or "" when none were said. + + Names as the server printed them: "CUDA", "Vulkan", the card's own model + name. Translating those would be inventing hardware nobody sells. + """ + if isinstance(state, Accel): + state = state._asdict() + parts = [state.get("backend") or "", state.get("device") or ""] + if state.get("layers"): + parts.append(f"{state['layers']} layers") + # A whisper on the processor prints "CPU" as its device too, and saying it + # twice reads like two different things. + seen, out = set(), [] + for part in parts: + if part and part.lower() not in seen: + seen.add(part.lower()) + out.append(part) + return ", ".join(out) + + +def cpu_only_build(state): + """Whether the binary that is running carries no GPU backend at all. + + The difference between "no card was found" and "this build could never have + used one", which is the difference between a machine to shrug at and a + download to replace. + """ + if isinstance(state, Accel): + state = state._asdict() + available = [name.upper() for name in (state.get("available") or [])] + return available == ["CPU"] + + def stop_all(): for server in SERVERS: server.stop() diff --git a/dikte/i18n.py b/dikte/i18n.py index 23ff6ff..3619797 100644 --- a/dikte/i18n.py +++ b/dikte/i18n.py @@ -777,6 +777,21 @@ TR = { "Load the model when Dikte starts": "Modeli Dikte açılırken yükle", "Local whisper": "Yerel whisper", "Local model": "Yerel model", + "Not loaded.": "Yüklü değil.", + "Loaded; it did not say what it is running on.": + "Yüklendi; neyin üzerinde çalıştığını söylemedi.", + "Loaded on the graphics card ({detail}).": + "Ekran kartına yüklendi ({detail}).", + "Loaded on the processor ({detail}).": "İşlemciye yüklendi ({detail}).", + "Loaded on the processor: this build carries no graphics backend, so the " + "box above cannot change that. A build from your distribution, or one you " + "point at above, may reach the card.": + "İşlemciye yüklendi: bu sürümde ekran kartı arka ucu yok, yukarıdaki " + "kutu bunu değiştiremez. Dağıtımınızın kendi sürümü ya da yukarıda yol " + "göstereceğiniz bir kopya karta ulaşabilir.", + "Loaded on the processor: the graphics card is switched on, but none was " + "found.": + "İşlemciye yüklendi: ekran kartı açık, ama bulunamadı.", "Not installed.": "Kurulu değil.", "Installed on the system: {path}": "Sistemde kurulu: {path}", "Downloaded, version {version}.": "İndirildi, sürüm {version}.", diff --git a/dikte/settings_ui.py b/dikte/settings_ui.py index 56771f8..ec17495 100644 --- a/dikte/settings_ui.py +++ b/dikte/settings_ui.py @@ -6,7 +6,7 @@ import shutil import sys import threading -from PyQt6.QtCore import QEvent, QObject, QRect, Qt, QUrl, pyqtSignal +from PyQt6.QtCore import QEvent, QObject, QRect, Qt, QTimer, QUrl, pyqtSignal from PyQt6.QtGui import QDesktopServices, QGuiApplication, QKeySequence, QShortcut from PyQt6.QtWidgets import ( QAbstractItemView, QAbstractSpinBox, QCheckBox, QComboBox, QDialog, @@ -670,6 +670,57 @@ class SettingsWindow(QDialog): # because of that, so open it on the tab that fixes it. if not conf.transcribe_ready(): self.tabs.setCurrentIndex(self.api_tab_index) + # A model takes up to ggml.STARTUP_TIMEOUT to load, so a line written + # once as the window opens would be wrong for most of the wait. Runs + # only while the window is on screen: there is nobody to read it + # otherwise, and it costs a lock and a poll() each time. + self._local_state_timer = QTimer(self) + self._local_state_timer.setInterval(2000) + self._local_state_timer.timeout.connect(self._show_local_state) + self._show_local_state() + + def showEvent(self, event): + super().showEvent(event) + self._show_local_state() + self._local_state_timer.start() + + def hideEvent(self, event): + self._local_state_timer.stop() + super().hideEvent(event) + + def _show_local_state(self): + """What each model on this machine is loaded on, as it is now.""" + local = ggml.state() + self.local_state.setText(self._local_state_text(local.get("whisper", {}))) + self.local_llm_state.setText(self._local_state_text(local.get("llama", {}))) + + @staticmethod + def _local_state_text(entry): + """One line: whether the model is loaded, and what it ended up on. + + Four answers rather than two, because "could not tell" is a real one: a + whisper built by hand on a Mac prints nothing about its backend, and + answering "the processor" there would be a confident lie about the one + thing this line exists to be honest about. + """ + kind = ggml.accel_kind(entry) + if kind == "off": + return t("Not loaded.") + # The backend and the card keep the names the server printed for them. + detail = ggml.accel_detail(entry) + if kind == "unknown": + return t("Loaded; it did not say what it is running on.") + if kind == "gpu": + return t("Loaded on the graphics card ({detail}).", detail=detail) + if not entry.get("gpu_wanted"): + return t("Loaded on the processor ({detail}).", detail=detail) + if ggml.cpu_only_build(entry): + return t("Loaded on the processor: this build carries no graphics " + "backend, so the box above cannot change that. A build " + "from your distribution, or one you point at above, may " + "reach the card.") + return t("Loaded on the processor: the graphics card is switched on, " + "but none was found.") def _scrolled(self, page): """A tab that scrolls instead of growing the window to fit.""" @@ -903,6 +954,12 @@ class SettingsWindow(QDialog): options_form.addRow("", self.local_preload) options_form.addRow(t("Threads"), self.local_threads) stt_form.addRow(self.local_options) + # What the model is actually doing, as against what the boxes above + # ask for. The checkbox can only ask: whether a card was found is + # decided by the build and by the machine, and is read back off the + # server's own log once it has loaded. + self.local_state = WrappedLabel("") + stt_form.addRow(self.local_state) self.transcribe_provider.currentIndexChanged.connect(self._provider_changed) outer.addWidget(stt) @@ -1004,6 +1061,8 @@ class SettingsWindow(QDialog): llm_form.addRow("", self.local_llm_preload) llm_form.addRow(t("Thinking"), self.local_llm_reasoning) orr_form.addRow(self.local_llm_options) + self.local_llm_state = WrappedLabel("") + orr_form.addRow(self.local_llm_state) outer.addWidget(orr) outer.addStretch(1) @@ -2040,6 +2099,7 @@ class SettingsWindow(QDialog): self.stt_form.setRowVisible(self.transcribe_status, not local) self.stt_form.setRowVisible(self.local_whisper, local) self.stt_form.setRowVisible(self.local_options, local) + self.stt_form.setRowVisible(self.local_state, local) if local: return self.transcribe_model.clear() @@ -2538,6 +2598,7 @@ class SettingsWindow(QDialog): provider != "local") self.cleanup_form.setRowVisible(self.local_llm, provider == "local") self.cleanup_form.setRowVisible(self.local_llm_options, provider == "local") + self.cleanup_form.setRowVisible(self.local_llm_state, provider == "local") binary = cleanup.executable(provider) found = shutil.which(binary) if binary else "" if provider == "local": diff --git a/tests/support.py b/tests/support.py index 8a005f1..5ee5f62 100644 --- a/tests/support.py +++ b/tests/support.py @@ -24,6 +24,7 @@ from unittest import mock from dikte import assistant from dikte import config as cfg +from dikte import ggml from dikte import i18n from dikte import update @@ -91,6 +92,22 @@ class DikteTest(unittest.TestCase): # down when it last ran. self.patch_attr(assistant, "SESSION_FILE", data_dir / "assistant.json") self.patch_attr(update, "STATE_FILE", data_dir / "update.json") + # ggml resolves its own three from paths.DATA_DIR at import, the same + # way cfg does. Left alone, a test asking what is installed or what the + # last server ran on would be reading whatever this machine happens to + # have downloaded, and passing or failing on somebody's home directory. + # program_path prefers a whisper-server or llama-server on the PATH + # over the copy Dikte downloaded, so on a machine with whisper.cpp + # installed these tests would be answering from that copy instead of + # from the install they set up. Every other tool still resolves; the + # tests that are about the system build patch this again themselves. + _which = shutil.which + self.patch_attr(shutil, "which", lambda tool, *args, **rest: ( + None if tool in ("whisper-server", "llama-server") + else _which(tool, *args, **rest))) + self.patch_attr(ggml, "DATA_DIR", data_dir) + self.patch_attr(ggml, "BIN_DIR", data_dir / "bin") + self.patch_attr(ggml, "MODELS_DIR", data_dir / "models") i18n.set_language("en") self.addCleanup(i18n.set_language, "en") diff --git a/tests/test_cli.py b/tests/test_cli.py index 67c6f21..93ca3f2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -775,6 +775,114 @@ class Replies(DikteTest): self.assertFalse(launched.called) +class LocalModels(DikteTest): + """Whether the model on this machine is loaded, and what it is loaded on.""" + + def status(self, local, **rest): + reply = {"ok": True, "running": True, "dictation": "idle", "ask": "idle", + "meeting": "idle", "listener": True, "local": local, **rest} + with mock.patch.object(ipc, "send", return_value=reply), \ + captured() as (out, _err): + cli.cmd_status(Options(json=False)) + return out.getvalue() + + def entry(self, **values): + base = {"running": True, "used": True, "pid": 7, "port": 4321, + "model": "ggml-small.bin", "gpu_wanted": True, + "backend": "CUDA", "device": "RTX 4070", "layers": "", + "available": ["CUDA", "CPU"]} + base.update(values) + return base + + def test_a_loaded_model_says_what_it_is_loaded_on(self): + line = self.status({"whisper": self.entry()}) + self.assertIn("whisper:", line) + self.assertIn("loaded on the graphics card (CUDA, RTX 4070)", line) + self.assertIn("ggml-small.bin", line) + + def test_a_card_asked_for_and_not_found_is_said_out_loud(self): + line = self.status({"whisper": self.entry( + backend="CPU", device="CPU", available=["CPU"])}) + self.assertIn("loaded on the processor", line) + self.assertIn("this build carries none", line) + + def test_a_card_the_build_could_have_used_says_something_else(self): + line = self.status({"whisper": self.entry( + backend="CPU", device="CPU", available=["CUDA", "CPU"])}) + self.assertIn("none was found", line) + self.assertNotIn("carries none", line) + + def test_a_card_nobody_asked_for_is_not_a_complaint(self): + line = self.status({"whisper": self.entry( + backend="CPU", device="CPU", gpu_wanted=False, available=["CPU"])}) + self.assertIn("loaded on the processor", line) + self.assertNotIn("switched on", line) + + def test_a_model_that_is_wanted_and_not_loaded_says_so(self): + line = self.status({"whisper": self.entry(running=False)}) + self.assertIn("whisper:", line) + self.assertIn("not loaded", line) + + def test_a_model_neither_used_nor_loaded_is_not_worth_a_line(self): + line = self.status({"llama": self.entry(running=False, used=False)}) + self.assertNotIn("llama", line) + + def test_an_instance_too_old_to_have_been_asked_says_nothing(self): + reply = {"ok": True, "running": True, "dictation": "idle", "ask": "idle", + "meeting": "idle", "listener": True} + with mock.patch.object(ipc, "send", return_value=reply), \ + captured() as (out, _err): + cli.cmd_status(Options(json=False)) + self.assertNotIn("whisper", out.getvalue()) + + # ---- doctor, which can be asked with nothing running ----------------- + + def doctor(self, as_json=False, **settings): + self.write_config(settings) + with mock.patch.object(ipc, "send", return_value=None), \ + captured() as (out, _err): + cli.cmd_doctor(Options(json=as_json)) + return json.loads(out.getvalue()) if as_json else out.getvalue() + + def log(self, text): + path = ggml.DATA_DIR / "whisper-server.log" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + def test_with_nothing_running_the_last_start_is_read_off_its_log(self): + self.log("load_backend: loaded CPU backend from /x.so\n" + "whisper_backend_init_gpu: device 0: CPU (type: 0)\n" + "whisper_backend_init_gpu: no GPU found\n") + line = self.doctor(transcribe_provider="local", local_gpu=True) + self.assertIn("last run on the processor", line) + self.assertIn("this build carries none", line) + + def test_a_run_that_named_no_backend_is_not_read_as_no_run_at_all(self): + # A log with nothing recognisable in it still says a server started + # here once, which is a different thing from never having started. + self.log("whisper_model_load: model size = 147.37 MB\n") + line = self.doctor(transcribe_provider="local") + self.assertIn("said nothing about what it was running on", line) + self.assertNotIn("never run here", line) + + def test_a_machine_that_never_ran_one_is_not_made_up_a_history_for(self): + line = self.doctor(transcribe_provider="local") + self.assertIn("never run here", line) + + def test_a_setup_that_transcribes_in_the_cloud_reads_about_none_of_it(self): + line = self.doctor(transcribe_provider="openai", cleanup_enabled=False) + self.assertNotIn("whisper ", line) + self.assertNotIn("never run here", line) + + def test_an_instance_that_cannot_be_asked_is_not_read_as_a_no(self): + """It used to print "not loaded", which is a different claim.""" + self.write_config({"transcribe_provider": "local"}) + with mock.patch.object(ipc, "send", return_value={"ok": True}), \ + captured() as (out, _err): + cli.cmd_doctor(Options(json=False)) + self.assertIn("too old to say", out.getvalue()) + + class TranscribeRunsHere(DikteTest): """`dikte transcribe` runs in this process, not in the instance.""" diff --git a/tests/test_ggml.py b/tests/test_ggml.py index 44b9305..994b6cd 100644 --- a/tests/test_ggml.py +++ b/tests/test_ggml.py @@ -525,6 +525,180 @@ class Catalogue(Local): "model.gguf") +# --- what it ended up running on ------------------------------------------ + + +# Trimmed from real logs. The first is this project's own bug report: the +# graphics card is switched on, whisper asked for one, and the build had none +# to give. +WHISPER_CPU = """\ +load_backend: loaded CPU backend from /opt/whisper/libggml-cpu-haswell.so +whisper_init_from_file_with_params_no_state: loading model from 'ggml-small.bin' +whisper_init_with_params_no_state: use gpu = 1 +whisper_model_load: CPU total size = 189.49 MB +whisper_backend_init_gpu: device 0: CPU (type: 0) +whisper_backend_init_gpu: no GPU found +""" + +WHISPER_CUDA = """\ +load_backend: loaded CUDA backend from /opt/whisper/libggml-cuda.so +load_backend: loaded CPU backend from /opt/whisper/libggml-cpu-haswell.so +whisper_init_with_params_no_state: use gpu = 1 +whisper_model_load: CUDA0 total size = 189.49 MB +whisper_backend_init_gpu: device 0: NVIDIA GeForce RTX 4070 (type: 1) +whisper_backend_init_gpu: using CUDA0 backend +""" + +# A card listed, tried, and refused: whisper says so and carries on without it, +# and the weights stay where they were put. Reading the listing alone would +# report a graphics card that is doing nothing. +WHISPER_GPU_FAILED = """\ +load_backend: loaded Vulkan backend from /usr/lib/ggml/libggml-vulkan.so +load_backend: loaded CPU backend from /usr/lib/ggml/libggml-cpu-haswell.so +whisper_model_load: CPU total size = 189.49 MB +whisper_backend_init_gpu: device 0: Vulkan0 (type: 1) +whisper_backend_init_gpu: found GPU device 0: Vulkan0 (type: 1, cnt: 0) +whisper_backend_init_gpu: failed to initialize Vulkan0 backend +""" + +# Both backends in one build. The Vulkan listing is there and is not the one +# that ran, so naming the card out of it would name the wrong device. +WHISPER_MIXED = """\ +ggml_vulkan: Found 1 Vulkan devices: +ggml_vulkan: 0 = Intel UHD Graphics 770 (ANV TGL) (anv) | uma: 1 +load_backend: loaded CUDA backend from /opt/whisper/libggml-cuda.so +load_backend: loaded Vulkan backend from /opt/whisper/libggml-vulkan.so +load_backend: loaded CPU backend from /opt/whisper/libggml-cpu-haswell.so + Device 0: NVIDIA GeForce RTX 4070, compute capability 8.9, VMM: yes +whisper_model_load: CUDA0 total size = 189.49 MB +whisper_backend_init_gpu: device 0: CUDA0 (type: 1) +whisper_backend_init_gpu: using CUDA0 backend +""" + +# The same start on a card whisper names only by its slot. The card's own name +# is one line further up, printed by the backend as it enumerates. +WHISPER_VULKAN = """\ +ggml_vulkan: Found 1 Vulkan devices: +ggml_vulkan: 0 = AMD Radeon RX 6600 (RADV NAVI23) (radv) | uma: 0 | fp16: dot2 +load_backend: loaded Vulkan backend from /usr/lib/ggml/libggml-vulkan.so +load_backend: loaded CPU backend from /usr/lib/ggml/libggml-cpu-haswell.so +whisper_model_load: Vulkan0 total size = 189.49 MB +whisper_backend_init_gpu: device 0: Vulkan0 (type: 1) +whisper_backend_init_gpu: using Vulkan0 backend +""" + +# A whisper built by hand on a Mac: Metal is compiled in rather than loaded, so +# there is no line to read and no honest answer but "it did not say". +WHISPER_QUIET = """\ +whisper_init_from_file_with_params_no_state: loading model from 'ggml-base.bin' +whisper_model_load: model size = 147.37 MB +""" + +LLAMA_GPU = """\ +load_backend: loaded Vulkan backend from /opt/llama/libggml-vulkan.so +load_backend: loaded CPU backend from /opt/llama/libggml-cpu.so +load_tensors: offloading 28 repeating layers to GPU +load_tensors: offloaded 29/29 layers to GPU +""" + +LLAMA_CPU = """\ +load_backend: loaded Vulkan backend from /opt/llama/libggml-vulkan.so +load_backend: loaded CPU backend from /opt/llama/libggml-cpu.so +load_tensors: offloaded 0/29 layers to GPU +""" + + +class WhatItRunsOn(Local): + """Reading the backend back out of the log the server wrote.""" + + def log(self, text): + path = self.path("server.log") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return path + + def read(self, program, text): + return ggml._read_accel(program, self.log(text)) + + def test_a_card_that_was_asked_for_and_not_found_is_the_processor(self): + accel = self.read(ggml.WHISPER, WHISPER_CPU) + self.assertEqual(accel.backend, "CPU") + self.assertEqual(ggml.accel_kind(accel), "cpu") + + def test_a_build_with_nothing_but_a_processor_backend_says_so(self): + self.assertTrue(ggml.cpu_only_build(self.read(ggml.WHISPER, WHISPER_CPU))) + self.assertFalse(ggml.cpu_only_build(self.read(ggml.WHISPER, WHISPER_CUDA))) + + def test_a_card_that_was_found_is_named(self): + accel = self.read(ggml.WHISPER, WHISPER_CUDA) + self.assertEqual(accel.backend, "CUDA") + self.assertEqual(accel.device, "NVIDIA GeForce RTX 4070") + self.assertEqual(ggml.accel_kind(accel), "gpu") + self.assertEqual(ggml.accel_detail(accel), + "CUDA, NVIDIA GeForce RTX 4070") + + def test_a_card_named_only_by_its_slot_is_looked_up(self): + accel = self.read(ggml.WHISPER, WHISPER_VULKAN) + self.assertEqual(accel.backend, "Vulkan") + # "Vulkan0" says which slot; the point of the line is which card. + self.assertEqual(accel.device, "AMD Radeon RX 6600 (RADV NAVI23)") + + def test_the_driver_behind_the_card_is_not_part_of_its_name(self): + # "(radv)" is how it is reached; "(RADV NAVI23)" is what it is called. + self.assertNotIn("(radv)", + self.read(ggml.WHISPER, WHISPER_VULKAN).device) + + def test_a_card_that_failed_to_start_is_not_a_card_in_use(self): + # It was listed, it was tried, it did not work, and whisper went on + # without it. The listing alone would have called this a graphics card. + accel = self.read(ggml.WHISPER, WHISPER_GPU_FAILED) + self.assertEqual(accel.backend, "CPU") + self.assertEqual(ggml.accel_kind(accel), "cpu") + + def test_the_card_named_is_the_one_that_ran(self): + accel = self.read(ggml.WHISPER, WHISPER_MIXED) + self.assertEqual(accel.backend, "CUDA") + self.assertEqual(accel.device, "NVIDIA GeForce RTX 4070") + self.assertNotIn("Intel", ggml.accel_detail(accel)) + + def test_a_slot_number_is_not_a_name(self): + # "Vulkan0" says which slot; with no listing to look it up in, saying + # nothing beats saying that. + self.assertEqual(self.read(ggml.LLAMA, LLAMA_GPU).device, "") + + def test_a_log_that_says_nothing_is_not_guessed_at(self): + accel = self.read(ggml.WHISPER, WHISPER_QUIET) + self.assertEqual(accel.backend, "") + self.assertEqual(ggml.accel_kind(accel), "unknown") + + def test_a_log_that_is_not_there_is_not_guessed_at_either(self): + self.assertEqual(ggml._read_accel(ggml.WHISPER, self.path("gone.log")), + ggml.NO_ACCEL) + + def test_the_layers_llama_offloaded_are_read_back(self): + accel = self.read(ggml.LLAMA, LLAMA_GPU) + self.assertEqual(accel.backend, "Vulkan") + self.assertEqual(accel.layers, "29/29") + self.assertEqual(ggml.accel_detail(accel), "Vulkan, 29/29 layers") + + def test_a_llama_that_offloaded_nothing_is_on_the_processor(self): + accel = self.read(ggml.LLAMA, LLAMA_CPU) + self.assertEqual(accel.backend, "CPU") + self.assertEqual(ggml.accel_kind(accel), "cpu") + # The build could have used the card; this run did not. + self.assertFalse(ggml.cpu_only_build(accel)) + + def test_the_processor_is_not_named_twice(self): + # whisper prints CPU as the backend and as the device, and saying it + # twice reads like two different things. + self.assertEqual(ggml.accel_detail(self.read(ggml.WHISPER, WHISPER_CPU)), + "CPU") + + def test_nothing_is_running_is_not_a_backend(self): + self.assertEqual(ggml.accel_kind({"running": False, "backend": "CUDA"}), + "off") + + # --- keeping a server alive ----------------------------------------------- @@ -544,6 +718,18 @@ STAND_IN = textwrap.dedent(""" print("could not load model: no such file") sys.exit(2) + # The startup chatter a real server prints before it binds, so that the + # log has something for _read_accel to find. Flushed, because stdout here + # is a file and nothing would reach it before the port opened. + if "--backend" in args: + print("load_backend: loaded " + opt("--backend") + " backend from /x.so", + flush=True) + print("whisper_backend_init_gpu: device 0: Test Card (type: 1)", + flush=True) + # The line that says one of them worked, which is the one read back. + print("whisper_backend_init_gpu: using " + opt("--backend") + "0 backend", + flush=True) + started = time.monotonic() healthy_after = float(opt("--healthy-after", "0")) @@ -602,6 +788,50 @@ class Servers(Local): self.assertRegex(url, r"^http://127\.0\.0\.1:\d+/v1$") self.assertTrue(server.running) + def test_nothing_started_is_a_state_saying_so(self): + state = self.server().state() + self.assertFalse(state["running"]) + self.assertEqual(ggml.accel_kind(state), "off") + + def test_a_running_server_says_what_it_settled_on(self): + server = self.server(extra=["--backend", "CUDA"], gpu=True) + server.serve() + state = server.state() + self.assertTrue(state["running"]) + self.assertIn(f":{state['port']}/v1", server.base_url()) + self.assertEqual(state["backend"], "CUDA") + self.assertEqual(state["device"], "Test Card") + self.assertTrue(state["gpu_wanted"]) + self.assertEqual(ggml.accel_kind(state), "gpu") + + def test_a_setting_changed_mid_start_does_not_rename_what_is_running(self): + # A save that lands while the model is being read in finds no process + # to stop, so it changes the settings under a start already in flight. + # The line must name the model that is loaded, not the one that will be. + server = self.server(model="first") + launch = server._launch + + def during(settings): + result = launch(settings) + server.configure(model="second") + return result + + self.patch_attr(server, "_launch", during) + server.serve() + self.assertEqual(server.state()["model"], "first") + self.assertEqual(server.settings()["model"], "second") + + def test_stopping_takes_the_backend_with_it(self): + server = self.server(extra=["--backend", "CUDA"]) + server.serve() + server.stop() + self.assertEqual(server.state()["backend"], "") + + def test_a_server_that_announced_nothing_is_not_guessed_at(self): + server = self.server() + server.serve() + self.assertEqual(ggml.accel_kind(server.state()), "unknown") + def test_the_second_call_does_not_start_a_second_one(self): server = self.server() first = server.serve() diff --git a/tests/test_ui.py b/tests/test_ui.py index 6273447..d87a54c 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -13,7 +13,7 @@ from typing import ClassVar from unittest import mock from PyQt6.QtCore import QPoint, QPointF, Qt -from PyQt6.QtGui import QWheelEvent +from PyQt6.QtGui import QHideEvent, QShowEvent, QWheelEvent from PyQt6.QtWidgets import QApplication, QMessageBox from dikte import audio @@ -1177,6 +1177,60 @@ class LocalModels(DikteTest): self.window(conf)._save() self.assertEqual(conf["local_model"], "ggml-large-v3-turbo-q5_0.bin") + def state(self, **values): + base = {"running": True, "pid": 3, "port": 4321, "model": "ggml-small.bin", + "gpu_wanted": True, "backend": "CUDA", "device": "RTX 4070", + "layers": "", "available": ["CUDA", "CPU"]} + base.update(values) + return base + + def shown(self, **values): + """The line the window writes under the local model boxes.""" + window = self.window(self.config(transcribe_provider="local")) + with mock.patch.object(ggml, "state", + return_value={"whisper": self.state(**values), + "llama": self.state(running=False)}): + window._show_local_state() + return window.local_state.text(), window.local_llm_state.text() + + def test_a_loaded_model_says_which_card_it_is_on(self): + whisper, llm = self.shown() + self.assertIn("graphics card", whisper) + self.assertIn("RTX 4070", whisper) + # The other box is about the other model, and that one is not loaded. + self.assertIn("Not loaded", llm) + + def test_a_card_asked_for_and_missing_is_not_left_to_be_guessed_at(self): + whisper, _ = self.shown(backend="CPU", device="CPU", available=["CPU"]) + self.assertIn("processor", whisper) + self.assertIn("no graphics backend", whisper) + + def test_a_build_that_could_have_used_one_says_the_other_thing(self): + whisper, _ = self.shown(backend="CPU", device="CPU", + available=["CUDA", "CPU"]) + self.assertIn("none was found", whisper) + self.assertNotIn("no graphics backend", whisper) + + def test_a_processor_nobody_argued_about_is_stated_plainly(self): + whisper, _ = self.shown(backend="CPU", device="CPU", gpu_wanted=False, + available=["CPU"]) + self.assertEqual(whisper, "Loaded on the processor (CPU).") + + def test_a_server_that_said_nothing_is_not_answered_for(self): + """A whisper built by hand on a Mac prints no backend line at all.""" + whisper, _ = self.shown(backend="", device="", available=[]) + self.assertIn("did not say", whisper) + + def test_the_line_stops_being_written_while_the_window_is_away(self): + # The events rather than show() and hide(): showing the window for real + # would send the same event down to the download boxes, which answer it + # by asking Hugging Face what models there are. + window = self.window(self.config(transcribe_provider="local")) + window.showEvent(QShowEvent()) + self.assertTrue(window._local_state_timer.isActive()) + window.hideEvent(QHideEvent()) + self.assertFalse(window._local_state_timer.isActive()) + def test_nothing_is_fetched_for_a_window_nobody_opened(self): # DikteTest closes the network, so a request would fail the test. The # lists are asked for when the box is shown, not when it is built.