diff --git a/dikte/app.py b/dikte/app.py index d321256..1fbd24b 100644 --- a/dikte/app.py +++ b/dikte/app.py @@ -604,6 +604,10 @@ class Dikte: "agent": assistant.display_name(self.conf), "provider": assistant.provider(self.conf), "listener": self.evdev.running, + # Whether each model on this machine is loaded, and what it ended up + # running on. Only this process knows: the servers are its children, + # and the command line has no way to ask them anything. + "local": self._local_state(), # Asked here rather than by the command line, because on macOS # there is no registry to read: a combination is held by this # process and by nothing else, so this is the only process that @@ -612,6 +616,17 @@ class Dikte: for name, spec in hotkey.SHORTCUTS.items()}, } + def _local_state(self): + """ggml.state(), with a mark for the servers this setup actually uses. + + A server that is neither wanted nor loaded is not worth a line anywhere; + one that is wanted and not loaded is exactly the line worth reading. + """ + local = ggml.state() + local["whisper"]["used"] = self.conf["transcribe_provider"] == "local" + local["llama"]["used"] = self.conf.uses_local_llm() + return local + def reload_settings(self): """Read the config file back after something outside changed it.""" self.conf.load() diff --git a/dikte/cli.py b/dikte/cli.py index c1ef786..0900c76 100644 --- a/dikte/cli.py +++ b/dikte/cli.py @@ -30,6 +30,7 @@ from . import audio from . import cleanup from . import config as cfg from . import filetranscribe +from . import ggml from . import hotkey from . import hub from . import ipc @@ -836,6 +837,68 @@ def cmd_update(opts): f"{release.url}") +# --- the models on this machine -------------------------------------------- + + +def _local_where(entry): + """Where a local model ran, in a phrase: the card, the processor, or neither. + + The backend and the card keep the names the server printed for them. A + graphics card is a product somebody sells under that name, and translating + it would be inventing hardware. + """ + kind = ggml.accel_kind({**entry, "running": True}) + where = {"gpu": "the graphics card", "cpu": "the processor"}.get( + kind, "something it did not name") + detail = ggml.accel_detail(entry) + return where + (f" ({detail})" if detail else "") + + +def _local_note(entry): + """Why the card is not in use, for a setup that asked for it and got none.""" + if not entry.get("gpu_wanted"): + return "" + if ggml.accel_kind({**entry, "running": True}) != "cpu": + return "" + return (" - the graphics card is switched on and this build carries none" + if ggml.cpu_only_build(entry) + else " - the graphics card is switched on and none was found") + + +def _local_line(entry): + if not entry.get("running"): + return "not loaded" + model = entry.get("model") or "" + return (f"loaded on {_local_where(entry)}" + + (f", {model}" if model else "") + _local_note(entry)) + + +def _last_local(conf): + """What the local servers last ran on, read off the logs they left behind. + + For a command line asking while nothing is running: there is no process to + put the question to, and the log outlives the process that wrote it. Every + entry says `running` is false, because this is an account of the last start + rather than a reading of a live one. + """ + rows = {} + for program, used, gpu in ( + (ggml.WHISPER, conf["transcribe_provider"] == "local", + bool(conf["local_gpu"])), + (ggml.LLAMA, conf.uses_local_llm(), bool(conf["local_llm_gpu"]))): + accel = ggml.last_accel(program) + rows[program.name] = { + # Whether one ever started here at all, which the backend cannot + # say on its own: a server that ran and named no backend and one + # that never ran both leave it empty. + "ran": ggml.server_log(program).exists(), + "running": False, "used": used, "gpu_wanted": gpu, + "backend": accel.backend, "device": accel.device, + "layers": accel.layers, "available": list(accel.available), + } + return rows + + def cmd_status(opts): reply = ipc.send("status") if reply is None: @@ -855,6 +918,11 @@ def cmd_status(opts): + (f" {reply['meeting_message']}" if reply.get("meeting_message") else ""), f"listener: {'on' if reply.get('listener') else 'off'}", ] + # Nothing for a setup that uses no model on this machine, and nothing at all + # from an instance too old to have been asked. + for name, entry in (reply.get("local") or {}).items(): + if entry.get("used") or entry.get("running"): + lines.append(f"{name + ':':11}{_local_line(entry)}") return out(opts, reply, "\n".join(lines)) @@ -867,6 +935,9 @@ def cmd_doctor(opts): # Mac shells out for one half and Windows for neither. A row saying ydotool # is missing on a machine that would never have run it is not a diagnosis, # it is a red mark to explain away. + # Asked once and read twice: whether an instance is running, and what its + # local servers are doing, which is a question only that process can answer. + live = ipc.send("status") or {} here = paste.desktop() wanted = [here.clipboard, here.keyboard] if sys.platform.startswith("linux"): @@ -908,8 +979,14 @@ def cmd_doctor(opts): "ready": cleanup_ready}, "agent": {"provider": assistant.provider(conf), "directory": assistant.working_dir(conf)}, - "running": ipc.send("status") is not None, + "running": bool(live), + # Live when there is an instance to ask, off the logs when there is not. + "local": live.get("local") or _last_local(conf), } + # An instance from before this field existed is not an instance saying + # nothing is loaded; it is one that cannot be asked, and the two must not + # print the same line. + stale = bool(live) and "local" not in live if target.provider == "local": transcribe_line = (f"{'✓' if transcribe_ready else '✗'} {target.service}, " f"transcribing on {target.model or 'no model yet'}") @@ -929,12 +1006,30 @@ def cmd_doctor(opts): f"{cleanup.model(conf)}") lines = [f"{'✓' if path else '✗'} {name:14} {path or 'not on your PATH'}" for name, path in programs.items()] - lines += [ - transcribe_line, - cleanup_line, + lines += [transcribe_line, cleanup_line] + # Only the models this setup actually uses: a machine transcribing in the + # cloud has nothing loaded here and no reason to read about it. + for name, entry in checks["local"].items(): + if not entry.get("used"): + continue + if stale: + lines.append(f"· {name:14} the running instance is too old to say; " + f"reload it with: dikte restart") + elif entry.get("running"): + lines.append(f"✓ {name:14} {_local_line(entry)}") + elif live: + lines.append(f"· {name:14} not loaded") + elif entry.get("backend"): + lines.append(f"· {name:14} last run on " + f"{_local_where(entry)}{_local_note(entry)}") + elif entry.get("ran"): + lines.append(f"· {name:14} last run said nothing about what it " + f"was running on") + else: + lines.append(f"· {name:14} never run here") + lines.append( f"{'✓' if checks['running'] else '·'} application " - + ("running" if checks["running"] else "not running"), - ] + + ("running" if checks["running"] else "not running")) return out(opts, {"ok": True, **checks}, "\n".join(lines)) diff --git a/dikte/ggml.py b/dikte/ggml.py index c2f8da8..51229ef 100644 --- a/dikte/ggml.py +++ b/dikte/ggml.py @@ -33,6 +33,7 @@ import json import os import pathlib import platform +import re import shutil import signal import socket @@ -606,6 +607,142 @@ def _tail(path, lines=3): return " | ".join(found[-lines:]) +# --- what the server is running on ---------------------------------------- + +# Both programs say where the model went, and neither is asked: it is printed +# while they start and captured in the log Dikte already keeps. Reading it back +# is the only way to tell a graphics card that was asked for from one that was +# found, which is a difference the settings checkbox cannot make on its own. +Accel = collections.namedtuple("Accel", "backend device layers available") + +NO_ACCEL = Accel("", "", "", ()) + +# ggml loads each backend from a shared object and says which; whisper then says +# whether it found a card, and llama says how many layers went onto it. +_BACKEND_LOADED = re.compile(r"^load_backend: loaded (\w+) backend", re.M) +_WHISPER_NO_GPU = "whisper_backend_init_gpu: no GPU found" +# What whisper committed to, which is not what it enumerated: it lists every +# device it can see, then tries them, and a card that fails to initialise sends +# it back to the processor. "using" is printed only once one has worked, and the +# model buffer is named after wherever the weights actually ended up. +_WHISPER_USING = re.compile( + r"^whisper_backend_init_gpu: using (\S+) backend", re.M) +_WHISPER_BUFFER = re.compile(r"^whisper_model_load:\s+(\S+) total size", re.M) +# The device listing, read for the card's name rather than for the verdict. +_WHISPER_DEVICE = re.compile( + r"^whisper_backend_init_gpu: device (\d+): (.+?) \(type: (\d+)\)", re.M) +_LLAMA_OFFLOAD = re.compile( + r"^load_tensors: offloaded (\d+)/(\d+) layers to GPU", re.M) + +# whisper names the device by its ggml handle, "Vulkan0" or "CUDA0", which says +# which slot rather than which card. Each backend prints the real name as it +# enumerates, one line further up. +_HANDLE = re.compile(r"^([A-Za-z]+?)(\d*)$") +_BARE = re.compile(r"^(?:Vulkan|CUDA|ROCm|SYCL|Metal|GPU|CPU)\d*$", re.I) +_METAL_DEVICE = re.compile(r"^ggml_metal.*picking default device: (.+)$", re.M) +# The driver in brackets after the card's own name: "(radv)", "(nvidia)". The +# name carries brackets of its own, but in capitals, so the case is what tells +# a driver tag from part of the name. +_DRIVER_TAG = re.compile(r"\s*\([a-z0-9_.\- ]+\)$") + + +def _enumerated(text, backend, index): + """The name the backend printed for one of its own devices, by slot. + + Asked by backend rather than by whichever listing came first: a machine + with both a CUDA build and a Vulkan loader prints two listings, and the + card named in the wrong one is somebody else's card. + """ + listings = { + "vulkan": rf"^ggml_vulkan: {index} = (.+?) \| ", + "cuda": rf"^\s*Device {index}: (.+?), compute capability", + "rocm": rf"^\s*Device {index}: (.+?), compute capability", + } + pattern = listings.get(backend.lower()) + found = re.search(pattern, text, re.M) if pattern else None + if found is None and backend.lower() == "metal": + found = _METAL_DEVICE.search(text) + if found is None: + return "" + return _DRIVER_TAG.sub("", found.group(1).strip()) + + +def _card_name(text, handle): + """The card behind a ggml handle like "Vulkan0", named the way it sells. + + Three places carry a name and only the third is always meaningful: the + handle says which slot, whisper's own device listing says whichever the + backend reported, and the backend's enumeration says what the thing is + called. Whichever of them is not just the handle again wins. + """ + parts = _HANDLE.match(handle or "") + backend, index = (parts.group(1), parts.group(2) or "0") if parts else ("", "0") + for slot, name, _kind in _WHISPER_DEVICE.findall(text): + if slot == index and name.strip() and not _BARE.match(name.strip()): + return name.strip() + # The handle itself is not an answer: it says which slot, and a line + # reading "Vulkan, Vulkan0" tells nobody which card is doing the work. + return _enumerated(text, backend, index) + +# The startup chatter is the first few hundred lines; the rest of the file is a +# line per request and grows for as long as the server lives. +_LOG_HEAD = 64 << 10 + + +def _read_accel(program, log_path): + """What the server that wrote `log_path` is running on. + + An empty backend is a real answer rather than a failure: a whisper built by + hand on a Mac has Metal compiled in and prints no load_backend line at all, + and calling that "the processor" would be a confident lie about the one + thing this is here to be honest about. + """ + try: + with open(log_path, encoding="utf-8", errors="replace") as fh: + text = fh.read(_LOG_HEAD) + except OSError: + return NO_ACCEL + # dict.fromkeys rather than a set: the order they were loaded in is the + # order they are worth showing in, and CPU is always one of them. + available = tuple(dict.fromkeys(_BACKEND_LOADED.findall(text))) + cards = [name for name in available if name.upper() != "CPU"] + if program is WHISPER: + # The handle whisper settled on. "using" is printed once a backend has + # initialised, so a card that was listed and then failed never reaches + # here; the model buffer is the same answer from the other end, named + # after wherever the weights were actually put. + using = _WHISPER_USING.search(text) + buffered = _WHISPER_BUFFER.search(text) + handle = (using or buffered).group(1) if (using or buffered) else "" + if _WHISPER_NO_GPU in text or handle.upper().startswith("CPU"): + return Accel("CPU", handle or "", "", available) + if handle: + parts = _HANDLE.match(handle) + backend = parts.group(1) if parts else "" + # The backend as the build spells it, so "Vulkan" rather than the + # capitalisation the handle happened to use. + backend = next((name for name in cards + if name.lower() == backend.lower()), backend) + return Accel(backend or "GPU", _card_name(text, handle), "", + available) + if available and not cards: + # Nothing but a processor backend in this build: there was nowhere + # else the model could have gone. + return Accel("CPU", "", "", available) + return Accel("", "", "", available) + found = _LLAMA_OFFLOAD.search(text) + if found: + layers = f"{found.group(1)}/{found.group(2)}" + if int(found.group(1)) > 0: + backend = cards[0] if cards else "GPU" + return Accel(backend, _card_name(text, f"{backend}0"), layers, + available) + return Accel("CPU", "", layers, available) + if available and not cards: + return Accel("CPU", "", "", available) + return Accel("", "", "", available) + + def _win_image_name(pid): """The full, lower-cased path of the process's executable, or ''. @@ -659,6 +796,15 @@ class Server: self._port = 0 self._log = "" self._key = None + # What the running child settled on, read out of its log once it was + # ready. Kept beside the process because it belongs to that process and + # to no other: a restart on new settings may land somewhere else. + self._accel = NO_ACCEL + # The settings the running child was started on, which is not what + # _settings holds: a change made while a start is in flight lands there + # first, and reporting the new model beside the old process would name + # a model this server is not running. + self._live = {} # The pid this instance last wrote to its pid file, so _forget never # removes a file some other Dikte wrote after us. self._pid = 0 @@ -694,6 +840,32 @@ class Server: with self._lock: return f"http://{HOST}:{self._port}/v1" if self._port else "" + def state(self): + """A snapshot of what this server is doing, for something to show. + + Taken under the short lock rather than the start one, so the interface + is answered at once even while a model is being read in. Plain types + throughout, because this travels over the socket to the command line. + """ + with self._lock: + up = self._proc is not None and self._proc.poll() is None + accel = self._accel if up else NO_ACCEL + # What it is running, when it is running; what it would run + # otherwise. The two differ for as long as a change waits for the + # restart that will pick it up. + settings = self._live if up else self._settings + return { + "running": up, + "pid": self._proc.pid if up else 0, + "port": self._port if up else 0, + "model": settings.get("model", ""), + "gpu_wanted": bool(settings.get("gpu")), + "backend": accel.backend, + "device": accel.device, + "layers": accel.layers, + "available": list(accel.available), + } + def error(self): """The last thing the server printed, for a failure after it started.""" with self._lock: @@ -715,9 +887,10 @@ class Server: self._stop_now() with self._lock: settings, key = dict(self._settings), self._settings_key() - proc, port, log = self._launch(settings) + proc, port, log, accel = self._launch(settings) with self._lock: self._proc, self._port, self._log, self._key = proc, port, log, key + self._accel, self._live = accel, settings return self.base_url() def _current_url(self): @@ -767,7 +940,9 @@ class Server: self._forget() raise if reason == "ready": - return proc, port, str(log) + # Read now rather than on demand: the startup lines are at the + # head of a file a long-lived server keeps appending to. + return proc, port, str(log), _read_accel(self.program, log) last = _tail(log) self._forget() # Losing the port between the probe and the bind is the one @@ -843,6 +1018,7 @@ class Server: with self._lock: proc, self._proc = self._proc, None self._port, self._log, self._key = 0, "", None + self._accel, self._live = NO_ACCEL, {} self._kill(proc, gently=True) if proc is not None: self._forget() @@ -1026,6 +1202,76 @@ def sweep(): return any([server.sweep() for server in SERVERS]) +def state(): + """What each local server is doing, keyed by program name.""" + return {server.program.name: server.state() for server in SERVERS} + + +def server_log(program): + """Where this program's server writes, which outlives the process.""" + return DATA_DIR / f"{program.name}-server.log" + + +def last_accel(program): + """What the last server for `program` ran on, from the log it left behind. + + For a command line asking with nothing running: the log outlives the process + and is the only account of the last start there is. + """ + return _read_accel(program, server_log(program)) + + +def accel_kind(state): + """"off" | "gpu" | "cpu" | "unknown", for a state() or an Accel. + + A tag rather than a sentence, because the two places that show this write + their own: the command line answers in English and the settings window in + whatever language it was opened in. + """ + if isinstance(state, Accel): + state = {"running": True, "backend": state.backend} + if not state.get("running"): + return "off" + backend = state.get("backend") or "" + if not backend: + return "unknown" + return "cpu" if backend.upper() == "CPU" else "gpu" + + +def accel_detail(state): + """The backend, the card and the layers, joined, or "" when none were said. + + Names as the server printed them: "CUDA", "Vulkan", the card's own model + name. Translating those would be inventing hardware nobody sells. + """ + if isinstance(state, Accel): + state = state._asdict() + parts = [state.get("backend") or "", state.get("device") or ""] + if state.get("layers"): + parts.append(f"{state['layers']} layers") + # A whisper on the processor prints "CPU" as its device too, and saying it + # twice reads like two different things. + seen, out = set(), [] + for part in parts: + if part and part.lower() not in seen: + seen.add(part.lower()) + out.append(part) + return ", ".join(out) + + +def cpu_only_build(state): + """Whether the binary that is running carries no GPU backend at all. + + The difference between "no card was found" and "this build could never have + used one", which is the difference between a machine to shrug at and a + download to replace. + """ + if isinstance(state, Accel): + state = state._asdict() + available = [name.upper() for name in (state.get("available") or [])] + return available == ["CPU"] + + def stop_all(): for server in SERVERS: server.stop() diff --git a/dikte/i18n.py b/dikte/i18n.py index 23ff6ff..3619797 100644 --- a/dikte/i18n.py +++ b/dikte/i18n.py @@ -777,6 +777,21 @@ TR = { "Load the model when Dikte starts": "Modeli Dikte açılırken yükle", "Local whisper": "Yerel whisper", "Local model": "Yerel model", + "Not loaded.": "Yüklü değil.", + "Loaded; it did not say what it is running on.": + "Yüklendi; neyin üzerinde çalıştığını söylemedi.", + "Loaded on the graphics card ({detail}).": + "Ekran kartına yüklendi ({detail}).", + "Loaded on the processor ({detail}).": "İşlemciye yüklendi ({detail}).", + "Loaded on the processor: this build carries no graphics backend, so the " + "box above cannot change that. A build from your distribution, or one you " + "point at above, may reach the card.": + "İşlemciye yüklendi: bu sürümde ekran kartı arka ucu yok, yukarıdaki " + "kutu bunu değiştiremez. Dağıtımınızın kendi sürümü ya da yukarıda yol " + "göstereceğiniz bir kopya karta ulaşabilir.", + "Loaded on the processor: the graphics card is switched on, but none was " + "found.": + "İşlemciye yüklendi: ekran kartı açık, ama bulunamadı.", "Not installed.": "Kurulu değil.", "Installed on the system: {path}": "Sistemde kurulu: {path}", "Downloaded, version {version}.": "İndirildi, sürüm {version}.", diff --git a/dikte/settings_ui.py b/dikte/settings_ui.py index 56771f8..ec17495 100644 --- a/dikte/settings_ui.py +++ b/dikte/settings_ui.py @@ -6,7 +6,7 @@ import shutil import sys import threading -from PyQt6.QtCore import QEvent, QObject, QRect, Qt, QUrl, pyqtSignal +from PyQt6.QtCore import QEvent, QObject, QRect, Qt, QTimer, QUrl, pyqtSignal from PyQt6.QtGui import QDesktopServices, QGuiApplication, QKeySequence, QShortcut from PyQt6.QtWidgets import ( QAbstractItemView, QAbstractSpinBox, QCheckBox, QComboBox, QDialog, @@ -670,6 +670,57 @@ class SettingsWindow(QDialog): # because of that, so open it on the tab that fixes it. if not conf.transcribe_ready(): self.tabs.setCurrentIndex(self.api_tab_index) + # A model takes up to ggml.STARTUP_TIMEOUT to load, so a line written + # once as the window opens would be wrong for most of the wait. Runs + # only while the window is on screen: there is nobody to read it + # otherwise, and it costs a lock and a poll() each time. + self._local_state_timer = QTimer(self) + self._local_state_timer.setInterval(2000) + self._local_state_timer.timeout.connect(self._show_local_state) + self._show_local_state() + + def showEvent(self, event): + super().showEvent(event) + self._show_local_state() + self._local_state_timer.start() + + def hideEvent(self, event): + self._local_state_timer.stop() + super().hideEvent(event) + + def _show_local_state(self): + """What each model on this machine is loaded on, as it is now.""" + local = ggml.state() + self.local_state.setText(self._local_state_text(local.get("whisper", {}))) + self.local_llm_state.setText(self._local_state_text(local.get("llama", {}))) + + @staticmethod + def _local_state_text(entry): + """One line: whether the model is loaded, and what it ended up on. + + Four answers rather than two, because "could not tell" is a real one: a + whisper built by hand on a Mac prints nothing about its backend, and + answering "the processor" there would be a confident lie about the one + thing this line exists to be honest about. + """ + kind = ggml.accel_kind(entry) + if kind == "off": + return t("Not loaded.") + # The backend and the card keep the names the server printed for them. + detail = ggml.accel_detail(entry) + if kind == "unknown": + return t("Loaded; it did not say what it is running on.") + if kind == "gpu": + return t("Loaded on the graphics card ({detail}).", detail=detail) + if not entry.get("gpu_wanted"): + return t("Loaded on the processor ({detail}).", detail=detail) + if ggml.cpu_only_build(entry): + return t("Loaded on the processor: this build carries no graphics " + "backend, so the box above cannot change that. A build " + "from your distribution, or one you point at above, may " + "reach the card.") + return t("Loaded on the processor: the graphics card is switched on, " + "but none was found.") def _scrolled(self, page): """A tab that scrolls instead of growing the window to fit.""" @@ -903,6 +954,12 @@ class SettingsWindow(QDialog): options_form.addRow("", self.local_preload) options_form.addRow(t("Threads"), self.local_threads) stt_form.addRow(self.local_options) + # What the model is actually doing, as against what the boxes above + # ask for. The checkbox can only ask: whether a card was found is + # decided by the build and by the machine, and is read back off the + # server's own log once it has loaded. + self.local_state = WrappedLabel("") + stt_form.addRow(self.local_state) self.transcribe_provider.currentIndexChanged.connect(self._provider_changed) outer.addWidget(stt) @@ -1004,6 +1061,8 @@ class SettingsWindow(QDialog): llm_form.addRow("", self.local_llm_preload) llm_form.addRow(t("Thinking"), self.local_llm_reasoning) orr_form.addRow(self.local_llm_options) + self.local_llm_state = WrappedLabel("") + orr_form.addRow(self.local_llm_state) outer.addWidget(orr) outer.addStretch(1) @@ -2040,6 +2099,7 @@ class SettingsWindow(QDialog): self.stt_form.setRowVisible(self.transcribe_status, not local) self.stt_form.setRowVisible(self.local_whisper, local) self.stt_form.setRowVisible(self.local_options, local) + self.stt_form.setRowVisible(self.local_state, local) if local: return self.transcribe_model.clear() @@ -2538,6 +2598,7 @@ class SettingsWindow(QDialog): provider != "local") self.cleanup_form.setRowVisible(self.local_llm, provider == "local") self.cleanup_form.setRowVisible(self.local_llm_options, provider == "local") + self.cleanup_form.setRowVisible(self.local_llm_state, provider == "local") binary = cleanup.executable(provider) found = shutil.which(binary) if binary else "" if provider == "local": diff --git a/tests/support.py b/tests/support.py index 8a005f1..5ee5f62 100644 --- a/tests/support.py +++ b/tests/support.py @@ -24,6 +24,7 @@ from unittest import mock from dikte import assistant from dikte import config as cfg +from dikte import ggml from dikte import i18n from dikte import update @@ -91,6 +92,22 @@ class DikteTest(unittest.TestCase): # down when it last ran. self.patch_attr(assistant, "SESSION_FILE", data_dir / "assistant.json") self.patch_attr(update, "STATE_FILE", data_dir / "update.json") + # ggml resolves its own three from paths.DATA_DIR at import, the same + # way cfg does. Left alone, a test asking what is installed or what the + # last server ran on would be reading whatever this machine happens to + # have downloaded, and passing or failing on somebody's home directory. + # program_path prefers a whisper-server or llama-server on the PATH + # over the copy Dikte downloaded, so on a machine with whisper.cpp + # installed these tests would be answering from that copy instead of + # from the install they set up. Every other tool still resolves; the + # tests that are about the system build patch this again themselves. + _which = shutil.which + self.patch_attr(shutil, "which", lambda tool, *args, **rest: ( + None if tool in ("whisper-server", "llama-server") + else _which(tool, *args, **rest))) + self.patch_attr(ggml, "DATA_DIR", data_dir) + self.patch_attr(ggml, "BIN_DIR", data_dir / "bin") + self.patch_attr(ggml, "MODELS_DIR", data_dir / "models") i18n.set_language("en") self.addCleanup(i18n.set_language, "en") diff --git a/tests/test_cli.py b/tests/test_cli.py index 67c6f21..93ca3f2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -775,6 +775,114 @@ class Replies(DikteTest): self.assertFalse(launched.called) +class LocalModels(DikteTest): + """Whether the model on this machine is loaded, and what it is loaded on.""" + + def status(self, local, **rest): + reply = {"ok": True, "running": True, "dictation": "idle", "ask": "idle", + "meeting": "idle", "listener": True, "local": local, **rest} + with mock.patch.object(ipc, "send", return_value=reply), \ + captured() as (out, _err): + cli.cmd_status(Options(json=False)) + return out.getvalue() + + def entry(self, **values): + base = {"running": True, "used": True, "pid": 7, "port": 4321, + "model": "ggml-small.bin", "gpu_wanted": True, + "backend": "CUDA", "device": "RTX 4070", "layers": "", + "available": ["CUDA", "CPU"]} + base.update(values) + return base + + def test_a_loaded_model_says_what_it_is_loaded_on(self): + line = self.status({"whisper": self.entry()}) + self.assertIn("whisper:", line) + self.assertIn("loaded on the graphics card (CUDA, RTX 4070)", line) + self.assertIn("ggml-small.bin", line) + + def test_a_card_asked_for_and_not_found_is_said_out_loud(self): + line = self.status({"whisper": self.entry( + backend="CPU", device="CPU", available=["CPU"])}) + self.assertIn("loaded on the processor", line) + self.assertIn("this build carries none", line) + + def test_a_card_the_build_could_have_used_says_something_else(self): + line = self.status({"whisper": self.entry( + backend="CPU", device="CPU", available=["CUDA", "CPU"])}) + self.assertIn("none was found", line) + self.assertNotIn("carries none", line) + + def test_a_card_nobody_asked_for_is_not_a_complaint(self): + line = self.status({"whisper": self.entry( + backend="CPU", device="CPU", gpu_wanted=False, available=["CPU"])}) + self.assertIn("loaded on the processor", line) + self.assertNotIn("switched on", line) + + def test_a_model_that_is_wanted_and_not_loaded_says_so(self): + line = self.status({"whisper": self.entry(running=False)}) + self.assertIn("whisper:", line) + self.assertIn("not loaded", line) + + def test_a_model_neither_used_nor_loaded_is_not_worth_a_line(self): + line = self.status({"llama": self.entry(running=False, used=False)}) + self.assertNotIn("llama", line) + + def test_an_instance_too_old_to_have_been_asked_says_nothing(self): + reply = {"ok": True, "running": True, "dictation": "idle", "ask": "idle", + "meeting": "idle", "listener": True} + with mock.patch.object(ipc, "send", return_value=reply), \ + captured() as (out, _err): + cli.cmd_status(Options(json=False)) + self.assertNotIn("whisper", out.getvalue()) + + # ---- doctor, which can be asked with nothing running ----------------- + + def doctor(self, as_json=False, **settings): + self.write_config(settings) + with mock.patch.object(ipc, "send", return_value=None), \ + captured() as (out, _err): + cli.cmd_doctor(Options(json=as_json)) + return json.loads(out.getvalue()) if as_json else out.getvalue() + + def log(self, text): + path = ggml.DATA_DIR / "whisper-server.log" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + def test_with_nothing_running_the_last_start_is_read_off_its_log(self): + self.log("load_backend: loaded CPU backend from /x.so\n" + "whisper_backend_init_gpu: device 0: CPU (type: 0)\n" + "whisper_backend_init_gpu: no GPU found\n") + line = self.doctor(transcribe_provider="local", local_gpu=True) + self.assertIn("last run on the processor", line) + self.assertIn("this build carries none", line) + + def test_a_run_that_named_no_backend_is_not_read_as_no_run_at_all(self): + # A log with nothing recognisable in it still says a server started + # here once, which is a different thing from never having started. + self.log("whisper_model_load: model size = 147.37 MB\n") + line = self.doctor(transcribe_provider="local") + self.assertIn("said nothing about what it was running on", line) + self.assertNotIn("never run here", line) + + def test_a_machine_that_never_ran_one_is_not_made_up_a_history_for(self): + line = self.doctor(transcribe_provider="local") + self.assertIn("never run here", line) + + def test_a_setup_that_transcribes_in_the_cloud_reads_about_none_of_it(self): + line = self.doctor(transcribe_provider="openai", cleanup_enabled=False) + self.assertNotIn("whisper ", line) + self.assertNotIn("never run here", line) + + def test_an_instance_that_cannot_be_asked_is_not_read_as_a_no(self): + """It used to print "not loaded", which is a different claim.""" + self.write_config({"transcribe_provider": "local"}) + with mock.patch.object(ipc, "send", return_value={"ok": True}), \ + captured() as (out, _err): + cli.cmd_doctor(Options(json=False)) + self.assertIn("too old to say", out.getvalue()) + + class TranscribeRunsHere(DikteTest): """`dikte transcribe` runs in this process, not in the instance.""" diff --git a/tests/test_ggml.py b/tests/test_ggml.py index 44b9305..994b6cd 100644 --- a/tests/test_ggml.py +++ b/tests/test_ggml.py @@ -525,6 +525,180 @@ class Catalogue(Local): "model.gguf") +# --- what it ended up running on ------------------------------------------ + + +# Trimmed from real logs. The first is this project's own bug report: the +# graphics card is switched on, whisper asked for one, and the build had none +# to give. +WHISPER_CPU = """\ +load_backend: loaded CPU backend from /opt/whisper/libggml-cpu-haswell.so +whisper_init_from_file_with_params_no_state: loading model from 'ggml-small.bin' +whisper_init_with_params_no_state: use gpu = 1 +whisper_model_load: CPU total size = 189.49 MB +whisper_backend_init_gpu: device 0: CPU (type: 0) +whisper_backend_init_gpu: no GPU found +""" + +WHISPER_CUDA = """\ +load_backend: loaded CUDA backend from /opt/whisper/libggml-cuda.so +load_backend: loaded CPU backend from /opt/whisper/libggml-cpu-haswell.so +whisper_init_with_params_no_state: use gpu = 1 +whisper_model_load: CUDA0 total size = 189.49 MB +whisper_backend_init_gpu: device 0: NVIDIA GeForce RTX 4070 (type: 1) +whisper_backend_init_gpu: using CUDA0 backend +""" + +# A card listed, tried, and refused: whisper says so and carries on without it, +# and the weights stay where they were put. Reading the listing alone would +# report a graphics card that is doing nothing. +WHISPER_GPU_FAILED = """\ +load_backend: loaded Vulkan backend from /usr/lib/ggml/libggml-vulkan.so +load_backend: loaded CPU backend from /usr/lib/ggml/libggml-cpu-haswell.so +whisper_model_load: CPU total size = 189.49 MB +whisper_backend_init_gpu: device 0: Vulkan0 (type: 1) +whisper_backend_init_gpu: found GPU device 0: Vulkan0 (type: 1, cnt: 0) +whisper_backend_init_gpu: failed to initialize Vulkan0 backend +""" + +# Both backends in one build. The Vulkan listing is there and is not the one +# that ran, so naming the card out of it would name the wrong device. +WHISPER_MIXED = """\ +ggml_vulkan: Found 1 Vulkan devices: +ggml_vulkan: 0 = Intel UHD Graphics 770 (ANV TGL) (anv) | uma: 1 +load_backend: loaded CUDA backend from /opt/whisper/libggml-cuda.so +load_backend: loaded Vulkan backend from /opt/whisper/libggml-vulkan.so +load_backend: loaded CPU backend from /opt/whisper/libggml-cpu-haswell.so + Device 0: NVIDIA GeForce RTX 4070, compute capability 8.9, VMM: yes +whisper_model_load: CUDA0 total size = 189.49 MB +whisper_backend_init_gpu: device 0: CUDA0 (type: 1) +whisper_backend_init_gpu: using CUDA0 backend +""" + +# The same start on a card whisper names only by its slot. The card's own name +# is one line further up, printed by the backend as it enumerates. +WHISPER_VULKAN = """\ +ggml_vulkan: Found 1 Vulkan devices: +ggml_vulkan: 0 = AMD Radeon RX 6600 (RADV NAVI23) (radv) | uma: 0 | fp16: dot2 +load_backend: loaded Vulkan backend from /usr/lib/ggml/libggml-vulkan.so +load_backend: loaded CPU backend from /usr/lib/ggml/libggml-cpu-haswell.so +whisper_model_load: Vulkan0 total size = 189.49 MB +whisper_backend_init_gpu: device 0: Vulkan0 (type: 1) +whisper_backend_init_gpu: using Vulkan0 backend +""" + +# A whisper built by hand on a Mac: Metal is compiled in rather than loaded, so +# there is no line to read and no honest answer but "it did not say". +WHISPER_QUIET = """\ +whisper_init_from_file_with_params_no_state: loading model from 'ggml-base.bin' +whisper_model_load: model size = 147.37 MB +""" + +LLAMA_GPU = """\ +load_backend: loaded Vulkan backend from /opt/llama/libggml-vulkan.so +load_backend: loaded CPU backend from /opt/llama/libggml-cpu.so +load_tensors: offloading 28 repeating layers to GPU +load_tensors: offloaded 29/29 layers to GPU +""" + +LLAMA_CPU = """\ +load_backend: loaded Vulkan backend from /opt/llama/libggml-vulkan.so +load_backend: loaded CPU backend from /opt/llama/libggml-cpu.so +load_tensors: offloaded 0/29 layers to GPU +""" + + +class WhatItRunsOn(Local): + """Reading the backend back out of the log the server wrote.""" + + def log(self, text): + path = self.path("server.log") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return path + + def read(self, program, text): + return ggml._read_accel(program, self.log(text)) + + def test_a_card_that_was_asked_for_and_not_found_is_the_processor(self): + accel = self.read(ggml.WHISPER, WHISPER_CPU) + self.assertEqual(accel.backend, "CPU") + self.assertEqual(ggml.accel_kind(accel), "cpu") + + def test_a_build_with_nothing_but_a_processor_backend_says_so(self): + self.assertTrue(ggml.cpu_only_build(self.read(ggml.WHISPER, WHISPER_CPU))) + self.assertFalse(ggml.cpu_only_build(self.read(ggml.WHISPER, WHISPER_CUDA))) + + def test_a_card_that_was_found_is_named(self): + accel = self.read(ggml.WHISPER, WHISPER_CUDA) + self.assertEqual(accel.backend, "CUDA") + self.assertEqual(accel.device, "NVIDIA GeForce RTX 4070") + self.assertEqual(ggml.accel_kind(accel), "gpu") + self.assertEqual(ggml.accel_detail(accel), + "CUDA, NVIDIA GeForce RTX 4070") + + def test_a_card_named_only_by_its_slot_is_looked_up(self): + accel = self.read(ggml.WHISPER, WHISPER_VULKAN) + self.assertEqual(accel.backend, "Vulkan") + # "Vulkan0" says which slot; the point of the line is which card. + self.assertEqual(accel.device, "AMD Radeon RX 6600 (RADV NAVI23)") + + def test_the_driver_behind_the_card_is_not_part_of_its_name(self): + # "(radv)" is how it is reached; "(RADV NAVI23)" is what it is called. + self.assertNotIn("(radv)", + self.read(ggml.WHISPER, WHISPER_VULKAN).device) + + def test_a_card_that_failed_to_start_is_not_a_card_in_use(self): + # It was listed, it was tried, it did not work, and whisper went on + # without it. The listing alone would have called this a graphics card. + accel = self.read(ggml.WHISPER, WHISPER_GPU_FAILED) + self.assertEqual(accel.backend, "CPU") + self.assertEqual(ggml.accel_kind(accel), "cpu") + + def test_the_card_named_is_the_one_that_ran(self): + accel = self.read(ggml.WHISPER, WHISPER_MIXED) + self.assertEqual(accel.backend, "CUDA") + self.assertEqual(accel.device, "NVIDIA GeForce RTX 4070") + self.assertNotIn("Intel", ggml.accel_detail(accel)) + + def test_a_slot_number_is_not_a_name(self): + # "Vulkan0" says which slot; with no listing to look it up in, saying + # nothing beats saying that. + self.assertEqual(self.read(ggml.LLAMA, LLAMA_GPU).device, "") + + def test_a_log_that_says_nothing_is_not_guessed_at(self): + accel = self.read(ggml.WHISPER, WHISPER_QUIET) + self.assertEqual(accel.backend, "") + self.assertEqual(ggml.accel_kind(accel), "unknown") + + def test_a_log_that_is_not_there_is_not_guessed_at_either(self): + self.assertEqual(ggml._read_accel(ggml.WHISPER, self.path("gone.log")), + ggml.NO_ACCEL) + + def test_the_layers_llama_offloaded_are_read_back(self): + accel = self.read(ggml.LLAMA, LLAMA_GPU) + self.assertEqual(accel.backend, "Vulkan") + self.assertEqual(accel.layers, "29/29") + self.assertEqual(ggml.accel_detail(accel), "Vulkan, 29/29 layers") + + def test_a_llama_that_offloaded_nothing_is_on_the_processor(self): + accel = self.read(ggml.LLAMA, LLAMA_CPU) + self.assertEqual(accel.backend, "CPU") + self.assertEqual(ggml.accel_kind(accel), "cpu") + # The build could have used the card; this run did not. + self.assertFalse(ggml.cpu_only_build(accel)) + + def test_the_processor_is_not_named_twice(self): + # whisper prints CPU as the backend and as the device, and saying it + # twice reads like two different things. + self.assertEqual(ggml.accel_detail(self.read(ggml.WHISPER, WHISPER_CPU)), + "CPU") + + def test_nothing_is_running_is_not_a_backend(self): + self.assertEqual(ggml.accel_kind({"running": False, "backend": "CUDA"}), + "off") + + # --- keeping a server alive ----------------------------------------------- @@ -544,6 +718,18 @@ STAND_IN = textwrap.dedent(""" print("could not load model: no such file") sys.exit(2) + # The startup chatter a real server prints before it binds, so that the + # log has something for _read_accel to find. Flushed, because stdout here + # is a file and nothing would reach it before the port opened. + if "--backend" in args: + print("load_backend: loaded " + opt("--backend") + " backend from /x.so", + flush=True) + print("whisper_backend_init_gpu: device 0: Test Card (type: 1)", + flush=True) + # The line that says one of them worked, which is the one read back. + print("whisper_backend_init_gpu: using " + opt("--backend") + "0 backend", + flush=True) + started = time.monotonic() healthy_after = float(opt("--healthy-after", "0")) @@ -602,6 +788,50 @@ class Servers(Local): self.assertRegex(url, r"^http://127\.0\.0\.1:\d+/v1$") self.assertTrue(server.running) + def test_nothing_started_is_a_state_saying_so(self): + state = self.server().state() + self.assertFalse(state["running"]) + self.assertEqual(ggml.accel_kind(state), "off") + + def test_a_running_server_says_what_it_settled_on(self): + server = self.server(extra=["--backend", "CUDA"], gpu=True) + server.serve() + state = server.state() + self.assertTrue(state["running"]) + self.assertIn(f":{state['port']}/v1", server.base_url()) + self.assertEqual(state["backend"], "CUDA") + self.assertEqual(state["device"], "Test Card") + self.assertTrue(state["gpu_wanted"]) + self.assertEqual(ggml.accel_kind(state), "gpu") + + def test_a_setting_changed_mid_start_does_not_rename_what_is_running(self): + # A save that lands while the model is being read in finds no process + # to stop, so it changes the settings under a start already in flight. + # The line must name the model that is loaded, not the one that will be. + server = self.server(model="first") + launch = server._launch + + def during(settings): + result = launch(settings) + server.configure(model="second") + return result + + self.patch_attr(server, "_launch", during) + server.serve() + self.assertEqual(server.state()["model"], "first") + self.assertEqual(server.settings()["model"], "second") + + def test_stopping_takes_the_backend_with_it(self): + server = self.server(extra=["--backend", "CUDA"]) + server.serve() + server.stop() + self.assertEqual(server.state()["backend"], "") + + def test_a_server_that_announced_nothing_is_not_guessed_at(self): + server = self.server() + server.serve() + self.assertEqual(ggml.accel_kind(server.state()), "unknown") + def test_the_second_call_does_not_start_a_second_one(self): server = self.server() first = server.serve() diff --git a/tests/test_ui.py b/tests/test_ui.py index 6273447..d87a54c 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -13,7 +13,7 @@ from typing import ClassVar from unittest import mock from PyQt6.QtCore import QPoint, QPointF, Qt -from PyQt6.QtGui import QWheelEvent +from PyQt6.QtGui import QHideEvent, QShowEvent, QWheelEvent from PyQt6.QtWidgets import QApplication, QMessageBox from dikte import audio @@ -1177,6 +1177,60 @@ class LocalModels(DikteTest): self.window(conf)._save() self.assertEqual(conf["local_model"], "ggml-large-v3-turbo-q5_0.bin") + def state(self, **values): + base = {"running": True, "pid": 3, "port": 4321, "model": "ggml-small.bin", + "gpu_wanted": True, "backend": "CUDA", "device": "RTX 4070", + "layers": "", "available": ["CUDA", "CPU"]} + base.update(values) + return base + + def shown(self, **values): + """The line the window writes under the local model boxes.""" + window = self.window(self.config(transcribe_provider="local")) + with mock.patch.object(ggml, "state", + return_value={"whisper": self.state(**values), + "llama": self.state(running=False)}): + window._show_local_state() + return window.local_state.text(), window.local_llm_state.text() + + def test_a_loaded_model_says_which_card_it_is_on(self): + whisper, llm = self.shown() + self.assertIn("graphics card", whisper) + self.assertIn("RTX 4070", whisper) + # The other box is about the other model, and that one is not loaded. + self.assertIn("Not loaded", llm) + + def test_a_card_asked_for_and_missing_is_not_left_to_be_guessed_at(self): + whisper, _ = self.shown(backend="CPU", device="CPU", available=["CPU"]) + self.assertIn("processor", whisper) + self.assertIn("no graphics backend", whisper) + + def test_a_build_that_could_have_used_one_says_the_other_thing(self): + whisper, _ = self.shown(backend="CPU", device="CPU", + available=["CUDA", "CPU"]) + self.assertIn("none was found", whisper) + self.assertNotIn("no graphics backend", whisper) + + def test_a_processor_nobody_argued_about_is_stated_plainly(self): + whisper, _ = self.shown(backend="CPU", device="CPU", gpu_wanted=False, + available=["CPU"]) + self.assertEqual(whisper, "Loaded on the processor (CPU).") + + def test_a_server_that_said_nothing_is_not_answered_for(self): + """A whisper built by hand on a Mac prints no backend line at all.""" + whisper, _ = self.shown(backend="", device="", available=[]) + self.assertIn("did not say", whisper) + + def test_the_line_stops_being_written_while_the_window_is_away(self): + # The events rather than show() and hide(): showing the window for real + # would send the same event down to the download boxes, which answer it + # by asking Hugging Face what models there are. + window = self.window(self.config(transcribe_provider="local")) + window.showEvent(QShowEvent()) + self.assertTrue(window._local_state_timer.isActive()) + window.hideEvent(QHideEvent()) + self.assertFalse(window._local_state_timer.isActive()) + def test_nothing_is_fetched_for_a_window_nobody_opened(self): # DikteTest closes the network, so a request would fail the test. The # lists are asked for when the box is shown, not when it is built.