diff --git a/README.md b/README.md index 048b652..628b2d6 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,9 @@ running. - **It all runs on this machine by default.** Speech to text on whisper.cpp and cleanup on llama.cpp, neither installed beforehand: the settings window fetches the program and the model, verifies the sha256 and refuses a download published - without one, then keeps a server alive while you dictate. The graphics card is + without one, then keeps a server alive while you dictate. The model list is + grouped by model rather than by file size, and the row this machine's memory + and graphics can take is marked. The graphics card is reached through CUDA, ROCm or Vulkan where the build allows. No key, no account, nothing leaving the machine. On x86_64 Linux the same button fetches a Vulkan build of whisper-server that Dikte publishes itself, because diff --git a/README.tr.md b/README.tr.md index 0fb43e4..a243c12 100644 --- a/README.tr.md +++ b/README.tr.md @@ -158,7 +158,9 @@ olmasını ister. whisper.cpp, temizleme llama.cpp üzerinde; ikisini de önceden kurman gerekmez: ayarlar penceresi programı ve modeli indirir, sha256'sını doğrular, checksum'suz yayınlanmış bir indirmeyi reddeder, sen dikte ettikçe sunucuyu - ayakta tutar. Derleme destekliyorsa ekran kartına CUDA, ROCm ya da Vulkan + ayakta tutar. Model listesi dosya boyutuna değil modele göre gruplanır ve bu + makinenin belleğine ve ekran kartına uyan satır işaretlenir. Derleme + destekliyorsa ekran kartına CUDA, ROCm ya da Vulkan üzerinden ulaşılır. Anahtar yok, hesap yok, makineden çıkan bir şey yok. x86_64 Linux'ta aynı düğme, whisper-server'ın Dikte'nin kendi yayınladığı Vulkan derlemesini indirir; upstream'in Linux arşivi yalnızca işlemci için. diff --git a/dikte/ggml.py b/dikte/ggml.py index 51cf4d1..dbf768f 100644 --- a/dikte/ggml.py +++ b/dikte/ggml.py @@ -26,6 +26,7 @@ interface already knows how to show. import atexit import collections +import ctypes import ctypes.util import hashlib import http.client @@ -98,28 +99,131 @@ NIGHTLY_TAG = "nightly-tag.txt" # hardware and the odd loose file. WHISPER_PREFIX = "ggml-" WHISPER_SUFFIX = ".bin" +# The mark on the whisper models trained on English alone. They are half of the +# list, and they belong under the model they are a variant of rather than +# scattered through it by size. +ENGLISH_ONLY = ".en" + +# Full-precision weights, however they are spelled. Several times the memory of +# a quantisation of the same model, for a difference dictation and cleanup +# cannot see, so nothing here ever points at one. +SIXTEEN_BIT = ("bf16", "f16", "fp16") + +# How many bits a weight is stored in, read off the file name. Every one of +# these lists spells it differently, `q5_1` and `Q4_K_M` and `MXFP4` and +# `BF16`, and the only part of that anybody choosing between two rows needs is +# the number. Longest mark first, so `bf16` is not read as `f16`. +BIT_DEPTHS = (("mxfp4", 4), ("bf16", 16), ("fp16", 16), ("f16", 16), + ("q2", 2), ("q3", 3), ("q4", 4), ("q5", 5), ("q6", 6), ("q8", 8)) # What a GGUF repository holds besides the model: mmproj is the vision half of a -# multimodal model, mtp a draft head for speculative decoding. Neither is a model -# a server can be started on, and offering them is offering a failure. -GGUF_SKIP = ("mmproj", "mtp-") +# multimodal model, and mtp, dflash, dspark and eagle3 are draft heads for +# speculative decoding. None of them is a model a server can be started on, and +# they are the small files in the repository, so a list sorted by size puts them +# at the top where they are likeliest to be clicked. +GGUF_SKIP = ("mmproj", "mtp-", "dflash-", "dspark-", "eagle3-", "draft-") # Big enough for a 12B at Q4 and far past anything cleanup wants; the point is # to keep a 400 GB frontier model out of a list somebody might click. GGUF_MAX_BYTES = 16 << 30 +# Repositories that carry GGUF files but nothing a cleanup server can be started +# on: a vision or audio tower with no text half worth running, a speech model, +# and the base models, which continue text rather than following an instruction +# and answer a cleanup prompt by carrying on writing the transcript. +# Matched as plain substrings, so every one of these carries its own +# delimiters: an unanchored "test-" is also inside "Latest-" and would drop a +# publisher that is perfectly usable. +LLM_REPO_SKIP = ("-Base-GGUF", "-VL-", "-Vision-", "-Omni-", "-Video-", + "-TTS-", "parakeet", "/test-") + +GB = 1 << 30 + # Suggestions, not a catalogue: the list itself is fetched, and these are only -# the rows that float to the top of it. Small instruction-following models, -# because cleanup is punctuation and filler words rather than anything that -# wants thinking about. +# the rows that float to the top of it. Cleanup is punctuation, capitals and +# filler words rather than anything that wants thinking about, so what it is +# picked on is instruction following at a size a desktop can spare. Gemma 4 +# scores 94.6 on IFEval at E2B and 96.7 at E4B, and E2B leads here rather than +# E4B because two points of instruction following is not worth twice the +# weights on a job that runs while somebody waits for their sentence to appear. +# SmolLM3 and Gemma 3 are the older pair below them. Qwen3.5 0.8B is for the +# machines nothing else fits on; it thinks before it answers, which is what the +# Thinking box in the settings window turns off. SUGGESTED_LLM = ( - "ggml-org/gemma-3-4b-it-GGUF", "ggml-org/gemma-4-E2B-it-GGUF", "ggml-org/gemma-4-E4B-it-GGUF", + "ggml-org/gemma-3-4b-it-GGUF", "ggml-org/SmolLM3-3B-GGUF", + "ggml-org/Qwen3.5-0.8B-GGUF", ) +# Roughly what each of those weighs at the quantisation cleanup would run, to +# the nearest half gigabyte. Not a catalogue of files: the sizes on the rows +# come from the publisher, and this only decides which suggestion is offered +# first on a machine that has room for some of them and not others. +SUGGESTED_LLM_SIZE = { + "ggml-org/gemma-4-E2B-it-GGUF": 3 * GB, + "ggml-org/gemma-4-E4B-it-GGUF": 5 * GB, + "ggml-org/gemma-3-4b-it-GGUF": 5 * GB // 2, + "ggml-org/SmolLM3-3B-GGUF": 2 * GB, + "ggml-org/Qwen3.5-0.8B-GGUF": GB // 2, +} + +# What each of them is, in the words somebody choosing between them would +# use. A repository id says the publisher, the parameter count, the shape of +# the weights and nothing at all about whether it is the one to click, and +# `ggml-org/gemma-4-E2B-it-GGUF` reads as four pieces of jargon to everybody +# who has not been reading model cards all year. +SUGGESTED_LLM_NOTE = { + "ggml-org/gemma-4-E2B-it-GGUF": + "Google Gemma 4, the small one. The default: nothing else this size " + "follows an instruction as closely, and cleanup is all instruction.", + "ggml-org/gemma-4-E4B-it-GGUF": + "The same model one size up. A little more accurate, about twice the " + "weights and twice the wait.", + "ggml-org/gemma-3-4b-it-GGUF": + "The previous Gemma. Still good, and the smallest of the Gemmas here.", + "ggml-org/SmolLM3-3B-GGUF": + "Hugging Face's own small model, for a machine the Gemmas crowd.", + "ggml-org/Qwen3.5-0.8B-GGUF": + "The smallest of them, for a machine nothing else fits on. It thinks " + "before it answers unless Thinking below is off.", +} + # Turbo at q5_0 is smaller than `small` and better than it, which makes the -# usual "start small" advice point at the same file as "start good". +# usual "start small" advice point at the same file as "start good". It is +# large-v3 with the decoder cut from 32 layers to 4: several times faster, at +# one to two points of word error in English and about two and a half in the +# other languages. SUGGESTED_WHISPER = "ggml-large-v3-turbo-q5_0.bin" +# Those two and a half points back, for twice the file and several times the +# work per second. Only suggested where there is a card to do the work and +# memory to hold it, because that is where the trade stops costing anything a +# person waiting for a dictation would notice. +ACCURATE_WHISPER = "ggml-large-v3-q5_0.bin" +# What to point at instead on a machine the turbo model would crowd. Same +# quantisation ladder, one rung down in size and in accuracy. +SMALL_MACHINE_WHISPER = "ggml-small-q5_1.bin" +# Under this much system memory, a 600 MB model plus the rest of a desktop is +# already tight, so the suggestion drops to the smaller one. Over the other, +# the accurate model is the one to point at. +SMALL_MACHINE = 4 * GB +# Fifteen and not sixteen: what the machine reports is what is left after the +# firmware and the graphics have taken their reservations out of it, and a +# 16 GB machine answers about 15.4. A threshold written at the number on the +# box is one no machine sold as that size ever reaches. +ROOMY_MACHINE = 15 * GB +# What a model may take of this machine's memory before it is called too big: +# half of it, less a gigabyte for the context and the runtime around the +# weights. A rule of thumb rather than a measurement, and deliberately a +# cautious one, because the failure it is guarding against is a machine that +# swaps itself to a standstill rather than a model that refuses to load. +MEMORY_SHARE = 0.5 +MEMORY_OVERHEAD = GB +# What is left to offer on a machine too small for the sum above to leave +# anything. Enough for the smallest whisper models and for a sub-billion +# cleanup model, which is what such a machine can run. +MEMORY_FLOOR = GB // 2 +# What total_memory() read the one time it asked. None until it has. +_MEMORY = None class LocalError(Exception): @@ -605,9 +709,212 @@ def _drop_old_versions(program, keep): pass +# --- what this machine can run -------------------------------------------- + + +def total_memory(): + """Bytes of memory on this machine, or 0 when it cannot be read. + + Zero is a real answer and not a failure: every caller treats an unknown + machine as one big enough for whatever it is looking at, because a wrong + "too big" is worse advice than none. + + Read once and kept. The memory in a machine does not change while Dikte + runs, and a list of thirty rows asks this question seventy times: on the + Mac path below, where the answer comes from a program rather than a + library call, that was seventy processes started on the interface thread + every time a list was drawn. + """ + global _MEMORY + if _MEMORY is None: + _MEMORY = max(_read_memory(), 0) + return _MEMORY + + +def _read_memory(): + """What the system says, which on a bad day is a negative number. + + sysconf answers -1 for a limit it holds to be indeterminate, and CPython + hands that straight back rather than raising, so the product below can + come out negative. The caller floors it at zero, which is the answer for + a machine nothing could be read from: a 64 GB workstation whose sysconf + shrugged was otherwise being told every model past 512 MB was too big + for it. + """ + try: + return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") + except (AttributeError, ValueError, OSError): + pass + if sys.platform == "darwin": + # Not every build of Python on a Mac has SC_PHYS_PAGES in its sysconf + # table, and this is the number the system itself is asked for. + try: + out = subprocess.run(["sysctl", "-n", "hw.memsize"], check=True, + capture_output=True, text=True, timeout=5) + return int(out.stdout.strip()) + except (OSError, ValueError, subprocess.SubprocessError): + return 0 + if sys.platform != "win32": + return 0 + + class Status(ctypes.Structure): + _fields_ = [("dwLength", ctypes.c_ulong), + ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("ullAvailExtendedVirtual", ctypes.c_ulonglong)] + + try: + status = Status() + status.dwLength = ctypes.sizeof(Status) + if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): + return int(status.ullTotalPhys) + except (AttributeError, OSError, ValueError): + pass + return 0 + + +def accelerator(): + """The graphics interface this machine offers, or "". + + The machine's half of the answer only. Whether a card is actually reached + also depends on which build landed, and the program line above says that: + a processor build ignores the card whatever is installed here. What this + is for is the other half, which nothing else on the window says at all. + """ + if sys.platform == "darwin": + return "Metal" + return "Vulkan" if _has_vulkan() else "" + + +def memory_budget(memory=None): + """What a model may weigh on this machine, or 0 when that is unknown. + + Floored rather than allowed to reach zero: on a 2 GB machine the share + less the overhead is nothing at all, and a budget of nothing is the same + number this returns for a machine it could not read, which would turn the + tightest machine there is into the one where everything is offered. + """ + memory = total_memory() if memory is None else memory + if not memory: + return 0 + return max(int(memory * MEMORY_SHARE) - MEMORY_OVERHEAD, MEMORY_FLOOR) + + +def fits(size, memory=None): + """Whether a model of this size is worth offering on this machine.""" + budget = memory_budget(memory) + return not budget or size <= budget + + +def suggested_whisper(memory=None, graphics=None): + """The whisper model to point at here, by name. + + Three machines. One with no room, which gets the model that leaves some. + One with a card and memory to spare, which gets the accurate model, because + the several times the work it is per second is several times a fraction of + a second there. Everything in between gets turbo, which is the answer + almost every time somebody asks. + + A Vulkan or Metal loader is not proof of a fast card, so the accurate model + waits on the memory as well: a machine with 16 GB in it and a driver + installed is one that will not notice either way. + """ + memory = total_memory() if memory is None else memory + graphics = accelerator() if graphics is None else graphics + if memory and memory < SMALL_MACHINE: + return SMALL_MACHINE_WHISPER + if graphics and memory >= ROOMY_MACHINE: + return ACCURATE_WHISPER + return SUGGESTED_WHISPER + + +def suggested_llm(memory=None): + """The suggested cleanup repositories, the ones that fit here first. + + The order they are written in is the order they are worth having. What + this changes is only which of them a machine that cannot hold the best one + is shown first, and nothing is dropped: a model that does not fit today + fits once something else is closed. + """ + return sorted(SUGGESTED_LLM, + key=lambda repo: not fits(SUGGESTED_LLM_SIZE.get(repo, 0), + memory)) + + +def recommended(items, want="", memory=None): + """The one row out of `items` worth pointing at here, or "". + + `want` is taken when it is on offer and fits. Without it, which is the + cleanup list, the smallest file that does is taken: q4 is where these + lists start, and every rung above it is roughly twice the memory and twice + the wait for a difference neither dictation nor cleanup can see. The + 16-bit weights are left out for the same reason, twice over. + """ + fitting = [i for i in items if fits(i.size, memory)] + if want and any(i.name == want for i in fitting): + return want + usable = [i for i in fitting + if not any(mark in i.name.lower() for mark in SIXTEEN_BIT)] + return min(usable, key=lambda i: i.size).name if usable else "" + + # --- the models ----------------------------------------------------------- +def bit_depth(name): + """The bits per weight the file name says, or 0 when it says nothing.""" + lowered = name.lower() + for mark, bits in BIT_DEPTHS: + if mark in lowered: + return bits + return 0 + + +def whisper_family(name): + """The model a whisper file belongs to: ggml-small.en-q5_1.bin is `small`. + + The list arrives sorted by size and nothing else, which interleaves the + families: `large-v3-turbo-q5_0` lands between the two `medium` + quantisations, half a screen from the turbo model it is a copy of. Grouping + is what puts the choice between models above the choice of quantisation, + which is the order somebody actually makes them in. + """ + stem = name + if stem.startswith(WHISPER_PREFIX): + stem = stem[len(WHISPER_PREFIX):] + if stem.endswith(WHISPER_SUFFIX): + stem = stem[:-len(WHISPER_SUFFIX)] + head, _, last = stem.rpartition("-") + # q5_0, q5_1, q8_0. `turbo` is the other thing a last chunk can be, and it + # is part of the model's name rather than a quantisation of it. + if head and last.startswith("q") and last[1:].replace("_", "").isdigit(): + stem = head + return stem[:-len(ENGLISH_ONLY)] if stem.endswith(ENGLISH_ONLY) else stem + + +def whisper_groups(items): + """[(family, [Item])] for a whisper list: one group per model. + + Groups by how big the model gets rather than by a ladder written down + here, so a family published next year sorts itself. Inside one, the + multilingual files come before the English-only ones and the small + quantisations before the large. + """ + groups = {} + for item in items: + groups.setdefault(whisper_family(item.name), []).append(item) + ordered = sorted(groups.items(), + key=lambda pair: (max(i.size for i in pair[1]), pair[0])) + return [(family, sorted(files, + key=lambda i: (ENGLISH_ONLY in i.name, i.size))) + for family, files in ordered] + + def whisper_models(refresh=False): """[hub.Item] for every whisper model on offer, smallest first.""" try: @@ -620,10 +927,23 @@ def whisper_models(refresh=False): return sorted(models, key=lambda f: f.size) +def can_clean(repo): + """Whether a repository could hold a model cleanup can be started on. + + By name, because the alternative is a file listing per repository and the + list is forty of them. It catches the kinds that are never a cleanup model + rather than the ones that are too big, which the file sizes answer exactly + once a publisher is chosen. + """ + lowered = repo.lower() + return not any(mark.lower() in lowered for mark in LLM_REPO_SKIP) + + def llm_repos(refresh=False): """Repository ids for the GGUF models on offer, suggestions first.""" try: - found = [r.id for r in hub.repos(author=LLM_AUTHOR, refresh=refresh)] + found = [r.id for r in hub.repos(author=LLM_AUTHOR, refresh=refresh) + if can_clean(r.id)] except hub.HubError: # A menu rather than a catalogue: with nothing to show, the suggestions # are still worth showing, and whatever is wrong with the network will @@ -631,6 +951,14 @@ def llm_repos(refresh=False): found = [] if not found: return list(SUGGESTED_LLM) + # Gemma publishes its base models under the instruction-tuned one's name + # with the `-it` taken out, so the two sit next to each other in the list + # and the wrong one answers a cleanup prompt by carrying on writing the + # transcript. Dropped only where the tuned sibling is here to drop it for. + tuned = set(found) + found = [r for r in found + if not r.endswith("-GGUF") + or r[:-len("-GGUF")] + "-it-GGUF" not in tuned] first = [r for r in SUGGESTED_LLM if r in found] return first + [r for r in found if r not in first] diff --git a/dikte/i18n.py b/dikte/i18n.py index 4f9d08e..9bb7fad 100644 --- a/dikte/i18n.py +++ b/dikte/i18n.py @@ -806,6 +806,75 @@ TR = { "ya da başka bir yayıncı seçin.", "downloaded": "indirildi", "not downloaded": "indirilmedi", + "recommended": "önerilen", + "{bits}-bit": "{bits} bit", + "English only": "yalnızca İngilizce", + "All": "Tümü", + "Everything ggml-org publishes, including the models that are too big to " + "run here and the ones that are not for cleaning up text.": + "ggml-org'un yayımladığı her şey; burada çalıştırılamayacak kadar " + "büyük olanlar ve metin temizlemek için olmayanlar dahil.", + "Google Gemma 4, the small one. The default: nothing else this size " + "follows an instruction as closely, and cleanup is all instruction.": + "Google Gemma 4'ün küçüğü. Varsayılan: bu boyutta verilen yönergeyi " + "bu kadar iyi izleyen başka bir model yok, temizleme de baştan sona " + "yönerge demek.", + "The same model one size up. A little more accurate, about twice the " + "weights and twice the wait.": + "Aynı modelin bir boy büyüğü. Biraz daha isabetli, yaklaşık iki katı " + "ağırlık ve iki katı bekleyiş.", + "The previous Gemma. Still good, and the smallest of the Gemmas here.": + "Bir önceki Gemma. Hâlâ iyi ve buradaki Gemma'ların en küçüğü.", + "Hugging Face's own small model, for a machine the Gemmas crowd.": + "Hugging Face'in kendi küçük modeli; Gemma'ların sıkıştırdığı bir " + "makine için.", + "The smallest of them, for a machine nothing else fits on. It thinks " + "before it answers unless Thinking below is off.": + "En küçükleri; başka hiçbir şeyin sığmadığı bir makine için. " + "Aşağıdaki Düşünme kapalı değilse cevaplamadan önce düşünür.", + "too big for this machine": "bu makine için fazla büyük", + "This machine": "Bu makine", + "Graphics: {name}.": "Ekran kartı: {name}.", + "No graphics interface found, so this runs on the processor.": + "Ekran kartı arayüzü bulunamadı, bu yüzden işlemcide çalışıyor.", + "Memory: {size}.": "Bellek: {size}.", + "A model may take half of this memory, less a gigabyte for the context " + "around the weights. Anything past that is marked too big; it may still " + "load, on a machine with nothing else open.": + "Bir model bu belleğin yarısını, ağırlıkların çevresindeki bağlam için " + "bir gigabayt düşülerek kullanabilir. Bunu aşan modeller fazla büyük " + "diye işaretlenir; başka hiçbir şeyin açık olmadığı bir makinede yine " + "de yüklenebilirler.", + "Recommended for this machine": "Bu makine için önerilen", + "Everything this publisher offers": "Bu yayıncının sunduğu her şey", + "Already on this machine": "Bu makinede zaten var", + "Chosen, but not downloaded": "Seçili, ama indirilmedi", + "{repo} publishes nothing that can be run here. Its models are split " + "across files, larger than {cap}, or pieces of a model rather than one. " + "Choose another publisher.": + "{repo} burada çalıştırılabilecek bir şey yayımlamıyor. Modelleri " + "birden çok dosyaya bölünmüş, {cap} boyutundan büyük ya da modelin " + "kendisi değil parçaları. Başka bir yayıncı seçin.", + "large-v3 makes the fewest mistakes and is the slowest of them. " + "large-v3-turbo is that model with a four layer decoder in place of a " + "thirty-two layer one: several times faster, at one to two points of word " + "error in English and about two and a half in the other languages. Below " + "those, every step down the list trades accuracy for size, and the .en " + "models are trained on English alone.": + "En az hatayı large-v3 yapar, en yavaşı da odur. large-v3-turbo, aynı " + "modelin otuz iki katmanlı çözücüsü yerine dört katmanlı bir çözücü " + "konmuş hâli: birkaç kat hızlı, karşılığında İngilizcede bir iki " + "puan, diğer dillerde yaklaşık iki buçuk puan kelime hatası. Bunların " + "altında listede her basamak, doğruluğu boyuta değişir; .en modelleri " + "ise yalnızca İngilizce ile eğitilmiştir.", + "Cleanup is punctuation, capitals and filler words, so what these are " + "picked on is following an instruction rather than knowing anything. " + "Start at a q4 file; the 16-bit ones are several times the memory for a " + "difference this job cannot see.": + "Temizleme; noktalama, büyük harf ve dolgu sözcükleri demek, yani bu " + "modeller bir şey bilmelerine değil verilen yönergeyi izlemelerine " + "göre seçilir. Bir q4 dosyasından başlayın; 16 bitlik olanlar, bu işin " + "göremeyeceği bir fark için kat kat bellek ister.", "Delete model": "Modeli sil", "Delete {name} from this machine?": "{name} bu makineden silinsin mi?", "Runs on this machine, on llama.cpp.": "Bu makinede, llama.cpp üzerinde çalışır.", diff --git a/dikte/settings_ui.py b/dikte/settings_ui.py index b72211a..3f32a72 100644 --- a/dikte/settings_ui.py +++ b/dikte/settings_ui.py @@ -248,6 +248,13 @@ class LocalModelBox(QGroupBox): self._stop = False self._wanted = "" # the model to select once a list arrives self._chosen_in = "" # the publisher the selected model is from + # Whether a list for the publisher on screen has come back. An empty + # box before one has is a box nobody has asked anything yet, and the + # two read the same without this. + self._answered = False + # What the last publisher listing held, so that the switch beside the + # box can be flipped without asking for it again. + self._found_repos = [] # Typing or arrowing through the publisher box changes its text a # character at a time, and each of those would otherwise be a request. self._later = QTimer(self) @@ -263,15 +270,55 @@ class LocalModelBox(QGroupBox): form.addRow(t("Program"), self._side_by_side(self.program_label, self.install_button)) + # What the model rows are judged against, said out loud. Without it, + # "too big for this machine" and the recommendation above the list are + # a verdict with no visible reason behind them. + self.machine_label = WrappedLabel() + self.machine_label.setToolTip( + t("A model may take half of this memory, less a gigabyte for the " + "context around the weights. Anything past that is marked too " + "big; it may still load, on a machine with nothing else open.")) + form.addRow(t("This machine"), self.machine_label) + self._show_machine() + if self._repos is not None: self.repo = QComboBox() self.repo.setEditable(True) self.repo.setToolTip(t("A Hugging Face repository of GGUF files. The " "list is fetched; any other one can be typed in.")) self.repo.currentTextChanged.connect(self._repo_changed) - form.addRow(t("Publisher"), self.repo) + # Forty repository ids is not a choice anybody can make. The few + # that were picked for this job are what the box holds until + # somebody asks for the rest. + self.every_repo = QCheckBox(t("All")) + self.every_repo.setToolTip( + t("Everything ggml-org publishes, including the models that " + "are too big to run here and the ones that are not for " + "cleaning up text.")) + self.every_repo.toggled.connect(self._every_repo_changed) + form.addRow(t("Publisher"), + self._side_by_side(self.repo, self.every_repo)) + # A repository id names the publisher, the parameter count and the + # shape of the weights, and says nothing about whether it is the + # one to click. + self.repo_note = WrappedLabel() + form.addRow("", self.repo_note) self.model = QComboBox() + self.model.setToolTip( + t("large-v3 makes the fewest mistakes and is the slowest of them. " + "large-v3-turbo is that model with a four layer decoder in place " + "of a thirty-two layer one: several times faster, at one to two " + "points of word error in English and about two and a half in " + "the other languages. Below those, every step down the list " + "trades accuracy for size, and the .en models are trained on " + "English alone.") + if program is ggml.WHISPER else + t("Cleanup is punctuation, capitals and filler words, so what " + "these are picked on is following an instruction rather than " + "knowing anything. Start at a q4 file; the 16-bit ones are " + "several times the memory for a difference this job cannot " + "see.")) self.download_button = QPushButton(t("Download")) self.download_button.clicked.connect(self._download) self.delete_button = QPushButton(t("Delete")) @@ -334,16 +381,16 @@ class LocalModelBox(QGroupBox): """ self._wanted = model self._pending = True + self._answered = False self._show_program() - self._chosen_in = repo or (ggml.SUGGESTED_LLM[0] if self._repos is not None - else "") + self._chosen_in = "" if self._repos is not None: + suggested = ggml.suggested_llm() + self._chosen_in = repo or suggested[0] self.repo.blockSignals(True) - self.repo.clear() - self.repo.addItems(list(ggml.SUGGESTED_LLM)) - self.repo.setCurrentText(repo or ggml.SUGGESTED_LLM[0]) + self.repo.setCurrentText(self._chosen_in) self.repo.blockSignals(False) - self._fit_popup(self.repo) + self._fill_repos_box(suggested) self._fill_models([]) def showEvent(self, event): @@ -387,6 +434,15 @@ class LocalModelBox(QGroupBox): t("Downloaded, version {version}.", version=ggml.installed_version(self.program) or "?")) + def _show_machine(self): + where = ggml.accelerator() + memory = ggml.total_memory() + parts = [t("Graphics: {name}.", name=where) if where else + t("No graphics interface found, so this runs on the processor.")] + if memory: + parts.append(t("Memory: {size}.", size=ggml.human_size(memory))) + self.machine_label.setText(" ".join(parts)) + # ---- the lists ------------------------------------------------------- def _fill_repos(self, current): @@ -395,10 +451,50 @@ class LocalModelBox(QGroupBox): threading.Thread(target=work, daemon=True).start() + def _fill_repos_box(self, found): + """The publishers, with the suggested ones kept apart from the rest. + + Forty repositories in one run is a list nobody reads to the end of, and + the few worth starting from are lost in it. A separator rather than a + heading, because this box is typed into as well as chosen from and a + heading would land in the field as though it were a repository. + """ + self._found_repos = found + current = self.repo.currentText() + # Every suggestion, whether or not it came back in the listing: that + # listing is the forty repositories touched most recently, and a + # publisher that has not been updated in a season falls off it while + # still being the one to point at. + first = list(ggml.suggested_llm()) + rest = [r for r in found if r not in first] + if not self.every_repo.isChecked(): + # The one being used stays on offer whatever the switch says, so + # that a repository somebody typed in is not dropped out from + # under them by the next fetch. + rest = [r for r in rest if r == current] + self.repo.blockSignals(True) + self.repo.clear() + self.repo.addItems(first) + if first and rest: + self.repo.insertSeparator(self.repo.count()) + self.repo.addItems(rest) + self.repo.setCurrentText(current) + self.repo.blockSignals(False) + self._fit_popup(self.repo) + self._show_repo_note() + def _repo_changed(self): + self._show_repo_note() if not self._downloading: self._later.start() + def _show_repo_note(self): + note = ggml.SUGGESTED_LLM_NOTE.get(self.repository(), "") + self.repo_note.setText(t(note) if note else "") + + def _every_repo_changed(self): + self._fill_repos_box(self._found_repos) + def _later_fetch(self): # A download that started inside the wait was not there to be seen when # the timer went off, and rebuilding the rows underneath one is exactly @@ -407,6 +503,7 @@ class LocalModelBox(QGroupBox): self._fetch_models(self.repository()) def _fetch_models(self, repo=""): + self._answered = False self.status.setText(t("Fetching the model list…")) def work(): @@ -436,45 +533,131 @@ class LocalModelBox(QGroupBox): self.status.setText(error) return if kind == "repos": - current = self.repo.currentText() - self.repo.blockSignals(True) - self.repo.clear() - self.repo.addItems(found) - self.repo.setCurrentText(current) - self.repo.blockSignals(False) - self._fit_popup(self.repo) + self._fill_repos_box(found) return + self._answered = True self._fill_models(found) + def _sections(self, items, best): + """[(heading, [Item])] for the rows to show, in the order to show them. + + The list arrives sorted by size and nothing else, which for whisper + interleaves the models: `large-v3-turbo-q5_0` lands between the two + `medium` quantisations, half a screen away from the turbo model it is a + copy of. Grouping puts the choice of model above the choice of + quantisation, and the row this machine should take goes on top, where + somebody who does not want to make either choice can stop reading. + """ + if not items: + return [] + groups = (ggml.whisper_groups(items) if self.program is ggml.WHISPER + else [("", items)]) + # A publisher with one file on offer is not a choice, and a row of its + # own above the only row there is would be the same model twice. + top = [i for i in items if i.name == best] if len(items) > 1 else [] + if not top: + return groups + if len(groups) == 1 and not groups[0][0]: + groups = [(t("Everything this publisher offers"), groups[0][1])] + return [(t("Recommended for this machine"), top)] + groups + + def _suggested(self): + """The name to prefer when it is on offer, or "" for whatever fits.""" + if self.program is not ggml.WHISPER: + return "" + # A Vulkan loader on the machine is not a card in play when what was + # installed is the processor build: recommending the accurate model + # off the loader alone would put a 1 GB model on a processor and the + # wait for it in front of somebody who asked for a sentence. + return ggml.suggested_whisper( + graphics="" if ggml.vulkan_missing(self.program) else None) + + def _add_heading(self, text): + """A row that names the group under it and cannot be chosen.""" + self.model.addItem(text) + row = self.model.count() - 1 + font = self.model.font() + font.setBold(True) + self.model.setItemData(row, font, Qt.ItemDataRole.FontRole) + listing = self.model.model() + entry = listing.item(row) if hasattr(listing, "item") else None + if entry is not None: + entry.setEnabled(False) + + def _add_model(self, name, item, best): + """One row: the file, what it weighs, and whether it is worth taking.""" + here = ggml.have_model(self._model_path(name)) + if here: + marks = [t("downloaded")] + elif item is None: + # Chosen but neither here nor on offer: the file was deleted from + # underneath, or the settings came from another machine. + marks = [t("not downloaded")] + else: + marks = [ggml.human_size(item.size)] + # `q5_1`, `Q4_K_M`, `MXFP4`, `BF16`: four spellings of the same thing + # in one list, and the number is the whole of what any of them says. A + # whisper file with no mark at all is the full 16-bit model, which is + # the one convention here that a name does not carry. + bits = ggml.bit_depth(name) or (16 if self.program is ggml.WHISPER + else 0) + if bits: + marks.append(t("{bits}-bit", bits=bits)) + if ggml.ENGLISH_ONLY in name: + marks.append(t("English only")) + # The verdicts last, after everything the row is: what to do about the + # row rather than what it holds. + if item is not None and not here and not ggml.fits(item.size): + marks.append(t("too big for this machine")) + if name == best: + marks.append(t("recommended")) + self.model.addItem(f"{name} ({', '.join(marks)})", name) + self.model.setItemData(self.model.count() - 1, item, + Qt.ItemDataRole.UserRole + 1) + + def _first_model(self): + """The first row that is a model rather than a heading.""" + for row in range(self.model.count()): + if self.model.itemData(row): + return row + return -1 + def _fill_models(self, items): - """One row per model, saying what it weighs and whether it is here.""" + """One row per model, grouped, saying what it weighs and where it is.""" # The selection is only worth carrying over within the publisher it was # made in. Carried across one, a model this repository does not publish # would be added back as "not downloaded" and selected again, and # changing the publisher would leave the model box looking untouched. same = self._repos is None or self.repository() == self._chosen_in wanted = self._wanted or (self.selected() if same else "") - here = [name for name in (self._model_path(i.name).name for i in items)] + best = ggml.recommended(items, self._suggested()) if items else "" self.model.blockSignals(True) self.model.clear() - for item, name in zip(items, here): - mark = (t("downloaded") if ggml.have_model(self._model_path(item.name)) - else ggml.human_size(item.size)) - self.model.addItem(f"{name} ({mark})", name) - self.model.setItemData(self.model.count() - 1, item, Qt.ItemDataRole.UserRole + 1) + listed = set() + for heading, group in self._sections(items, best): + if heading: + self._add_heading(heading) + for item in group: + name = self._model_path(item.name).name + self._add_model(name, item, best) + listed.add(name) # A model that was downloaded and then dropped from the list upstream is - # still on this disk and still works, so it stays on offer. - for name in self._on_disk(): - if self.model.findData(name) < 0: - self.model.addItem(f"{name} ({t('downloaded')})", name) - # And one that is chosen but not here, because the file was deleted from - # underneath or the settings came from another machine, stays chosen: - # Save reads this box, and a row missing here would quietly empty the - # setting rather than showing that the model needs downloading again. - if wanted and self.model.findData(wanted) < 0: - self.model.addItem(f"{wanted} ({t('not downloaded')})", wanted) + # still on this disk and still works, so it stays on offer. So does one + # that is chosen but not here: Save reads this box, and a row missing + # here would quietly empty the setting rather than showing that the + # model needs downloading again. + extras = [(t("Already on this machine"), + [name for name in self._on_disk() if name not in listed])] + if wanted and wanted not in listed \ + and not ggml.have_model(self._model_path(wanted)): + extras.append((t("Chosen, but not downloaded"), [wanted])) + for heading, names in extras: + if names and listed: + self._add_heading(heading) + for name in names: + self._add_model(name, None, best) index = self.model.findData(wanted) - self.model.setCurrentIndex(max(index, 0)) + self.model.setCurrentIndex(index if index >= 0 else self._first_model()) self.model.blockSignals(False) self._fit_popup(self.model) self._wanted = "" @@ -569,10 +752,17 @@ class LocalModelBox(QGroupBox): def _fill_models_from_current(self): """Redraw the rows without asking anybody anything again.""" - items = [self.model.itemData(i, Qt.ItemDataRole.UserRole + 1) - for i in range(self.model.count())] + # By name, because the recommended model has a row of its own at the + # top as well as one in its group, and reading the rows back twice + # would double it in the list every time a download finished. + items, seen = [], set() + for row in range(self.model.count()): + item = self.model.itemData(row, Qt.ItemDataRole.UserRole + 1) + if item is not None and item.name not in seen: + seen.add(item.name) + items.append(item) self._wanted = self.selected() - self._fill_models([i for i in items if i is not None]) + self._fill_models(items) def _delete(self): name = self.selected() @@ -607,7 +797,18 @@ class LocalModelBox(QGroupBox): and not here)) if self._downloading: return - if not name: + if not name and self._repos is not None and self._answered \ + and self._first_model() < 0: + # An empty box under a publisher that answered perfectly well: what + # it publishes is split across files, past the size cap, or a + # projector or draft head rather than a model of its own. Said + # nowhere, it read as though the click had not registered. + self.status.setText( + t("{repo} publishes nothing that can be run here. Its models " + "are split across files, larger than {cap}, or pieces of a " + "model rather than one. Choose another publisher.", + repo=self.repository(), cap=ggml.human_size(ggml.GGUF_MAX_BYTES))) + elif not name: self.status.setText(t("Nothing downloaded yet.")) elif here and not ggml.program_path(self.program): # The model alone runs nothing, and "Ready" over a missing program diff --git a/tests/support.py b/tests/support.py index 8a005f1..f96fab9 100644 --- a/tests/support.py +++ b/tests/support.py @@ -24,6 +24,7 @@ from unittest import mock from dikte import assistant from dikte import config as cfg +from dikte import ggml from dikte import i18n from dikte import update @@ -95,6 +96,10 @@ class DikteTest(unittest.TestCase): i18n.set_language("en") self.addCleanup(i18n.set_language, "en") + # Read once and kept for the life of the process, which across a test + # run means one test's machine answering for the next one's. + self.patch_attr(ggml, "_MEMORY", None) + # cli.launch_gui replaces this process with the application when no # instance is running. A test that reaches it would take the whole run # with it and hang, so it fails loudly here instead. diff --git a/tests/test_ggml.py b/tests/test_ggml.py index 3d86324..649e192 100644 --- a/tests/test_ggml.py +++ b/tests/test_ggml.py @@ -49,6 +49,11 @@ def item(name, data, url="https://example.invalid/f", sha=True): hashlib.sha256(data).hexdigest() if sha else "") +def listed(name, size): + """A row as a listing hands it over: a name and a size, no bytes.""" + return hub.Item(name, f"https://example.invalid/{name}", size, "a" * 64) + + @contextlib.contextmanager def serving(release, archive): """Answer by what is being asked for rather than by what came before. @@ -664,6 +669,53 @@ class Catalogue(Local): with self.assertRaises(ggml.LocalError): ggml.whisper_models() + def test_the_speculative_decoding_heads_are_not_models(self): + # They are the small files in a repository, so a list sorted by size + # puts them first, where the eye lands and the click goes. + tree = GGUF_TREE + [ + {"type": "file", "path": "dflash-Qwen3-8B-Q8_0.gguf", + "size": 1_120_000_000, "lfs": {"oid": "f" * 64}}, + {"type": "file", "path": "eagle3-gpt-oss-20b-Q8_0.gguf", + "size": 920_000_000, "lfs": {"oid": "0" * 64}}, + ] + with fake_urlopen(tree): + names = [q.name for q in ggml.llm_quants("ggml-org/x-GGUF")] + self.assertEqual(names, + ["gemma-3-4b-it-Q4_K_M.gguf", "gemma-3-4b-it-Q8_0.gguf"]) + + def test_a_speech_or_vision_repository_is_not_a_cleanup_publisher(self): + listing = [{"id": "ggml-org/parakeet-GGUF"}, + {"id": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF"}, + {"id": "ggml-org/SmolVLM2-256M-Video-Instruct-GGUF"}, + {"id": "ggml-org/Qwen3-8B-Base-GGUF"}, + {"id": "ggml-org/SmolLM3-3B-GGUF"}] + with fake_urlopen(listing): + found = ggml.llm_repos() + self.assertEqual([r for r in found if r.startswith("ggml-org/Smol")], + ["ggml-org/SmolLM3-3B-GGUF"]) + self.assertNotIn("ggml-org/parakeet-GGUF", found) + self.assertNotIn("ggml-org/Qwen3-8B-Base-GGUF", found) + + def test_a_publisher_is_not_dropped_for_a_word_it_happens_to_contain(self): + # The skip marks are matched as plain substrings, and an unanchored + # "test-" is also inside "Latest-". + self.assertTrue(ggml.can_clean("ggml-org/Qwen3-Latest-GGUF")) + self.assertFalse(ggml.can_clean("ggml-org/test-model-router-download")) + + def test_a_base_model_beside_its_tuned_twin_is_dropped(self): + # Gemma names the base model after the tuned one with the `-it` taken + # out, so the two sit next to each other and the wrong one answers a + # cleanup prompt by carrying on writing the transcript. + listing = [{"id": "ggml-org/gemma-4-E2B-GGUF"}, + {"id": "ggml-org/gemma-4-E2B-it-GGUF"}, + {"id": "ggml-org/Qwen3-0.6B-GGUF"}] + with fake_urlopen(listing): + found = ggml.llm_repos() + self.assertNotIn("ggml-org/gemma-4-E2B-GGUF", found) + self.assertIn("ggml-org/gemma-4-E2B-it-GGUF", found) + # Nothing named it, so nothing says it is the wrong half of a pair. + self.assertIn("ggml-org/Qwen3-0.6B-GGUF", found) + def test_what_is_on_disk_is_read_from_disk(self): self.assertEqual(ggml.installed_whisper_models(), []) path = ggml.whisper_model_path("ggml-base.bin") @@ -1183,3 +1235,205 @@ class WindowsOwnership(Local): # from here", and only one of those makes the pid file safe to drop. self.image("") self.assertIsNone(self.made._is_ours(1234)) + + +class Machine(Local): + """What this machine can hold, and what that makes worth pointing at.""" + + def _sysconf(self, phys_pages, page_size=4096): + """Stand where sysconf answers whatever this test wants it to. + + `create` because Windows has no os.sysconf at all, and a patch that + insists on the real attribute fails there before the test runs. What + the code under test does about that absence is two lines down from + what these are checking, and it is checked on its own below. + """ + return mock.patch.object( + ggml.os, "sysconf", create=True, + side_effect=lambda name: (page_size if name == "SC_PAGE_SIZE" + else phys_pages)) + + def test_the_memory_is_read_the_way_each_system_reports_it(self): + # Linux and most Macs answer through sysconf. + with self._sysconf(4_194_304): + self.assertEqual(ggml.total_memory(), 16 * ggml.GB) + + def test_a_mac_without_the_page_count_is_asked_for_the_number(self): + # Not every build of Python on a Mac carries SC_PHYS_PAGES, and a Mac + # that answered nothing would be a Mac with none of this on it. + def answer(args, **kwargs): + self.assertEqual(args, ["sysctl", "-n", "hw.memsize"]) + return mock.Mock(stdout=f"{32 * ggml.GB}\n") + + with mock.patch.object(ggml.os, "sysconf", create=True, + side_effect=ValueError), \ + mock.patch.object(sys, "platform", "darwin"), \ + mock.patch.object(ggml.subprocess, "run", answer): + self.assertEqual(ggml.total_memory(), 32 * ggml.GB) + + def test_a_sysconf_that_shrugs_is_an_unknown_machine_and_not_a_tiny_one(self): + # sysconf answers -1 for a limit it holds to be indeterminate and + # CPython hands that back rather than raising, so the product came out + # negative: a 64 GB workstation was told every model past 512 MB was + # too big for it, and the machine line read "Memory: -4096 B". + with self._sysconf(-1): + self.assertEqual(ggml.total_memory(), 0) + self.assertTrue(ggml.fits(574 << 20, memory=0)) + + def test_the_memory_is_read_once_and_kept(self): + # A list of thirty rows asks seventy times, and on the Mac path the + # answer comes from a program rather than a library call. + calls = [] + with mock.patch.object(ggml, "_read_memory", + lambda: calls.append(1) or 16 * ggml.GB): + self.assertEqual(ggml.total_memory(), 16 * ggml.GB) + self.assertEqual(ggml.total_memory(), 16 * ggml.GB) + self.assertEqual(len(calls), 1) + + def test_a_system_that_answers_nothing_is_an_unknown_machine(self): + with mock.patch.object(ggml.os, "sysconf", create=True, + side_effect=ValueError), \ + mock.patch.object(sys, "platform", "linux"): + self.assertEqual(ggml.total_memory(), 0) + + def test_a_mac_is_taken_to_have_a_graphics_interface(self): + with mock.patch.object(sys, "platform", "darwin"): + self.assertEqual(ggml.accelerator(), "Metal") + + def test_elsewhere_the_vulkan_loader_is_what_says_so(self): + with mock.patch.object(sys, "platform", "linux"), \ + mock.patch.object(ggml.ctypes.util, "find_library", + lambda name: "/usr/lib/libvulkan.so.1"): + self.assertEqual(ggml.accelerator(), "Vulkan") + with mock.patch.object(sys, "platform", "linux"), \ + mock.patch.object(ggml.ctypes.util, "find_library", + lambda name: None): + self.assertEqual(ggml.accelerator(), "") + + def test_a_model_is_measured_against_half_the_memory(self): + self.assertTrue(ggml.fits(2 * ggml.GB, memory=8 * ggml.GB)) + self.assertFalse(ggml.fits(4 * ggml.GB, memory=8 * ggml.GB)) + + def test_a_machine_whose_memory_could_not_be_read_holds_anything(self): + # A wrong "too big" is worse advice than none. + self.assertTrue(ggml.fits(40 * ggml.GB, memory=0)) + + def test_the_smallest_machine_is_not_the_one_where_everything_fits(self): + # Half of 2 GB less the gigabyte of overhead is nothing, and a budget + # of nothing used to read as the unknown machine above. + self.assertFalse(ggml.fits(3 * ggml.GB, memory=2 * ggml.GB)) + + def test_a_crowded_machine_is_pointed_at_the_smaller_model(self): + self.assertEqual(ggml.suggested_whisper(memory=3 * ggml.GB, graphics=""), + ggml.SMALL_MACHINE_WHISPER) + + def test_a_card_and_the_memory_for_it_are_pointed_at_the_accurate_one(self): + self.assertEqual( + ggml.suggested_whisper(memory=32 * ggml.GB, graphics="Vulkan"), + ggml.ACCURATE_WHISPER) + + def test_memory_without_a_card_is_pointed_at_the_fast_one(self): + # Several times the work per second is several times a long wait on a + # processor, whatever there is room for. + self.assertEqual( + ggml.suggested_whisper(memory=32 * ggml.GB, graphics=""), + ggml.SUGGESTED_WHISPER) + + def test_a_sixteen_gigabyte_machine_counts_as_a_roomy_one(self): + # What a machine reports is what the firmware and the graphics left + # of it: 16 GB answers about 15.4, and a threshold written at the + # number on the box is one no machine ever reaches. + self.assertEqual( + ggml.suggested_whisper(memory=int(15.4 * ggml.GB), graphics="Metal"), + ggml.ACCURATE_WHISPER) + + def test_the_suggestion_that_fits_is_offered_first(self): + first = ggml.suggested_llm(memory=6 * ggml.GB)[0] + self.assertTrue(ggml.fits(ggml.SUGGESTED_LLM_SIZE[first], + memory=6 * ggml.GB)) + # Nothing is dropped: what does not fit today fits once something else + # is closed. + self.assertEqual(sorted(ggml.suggested_llm(memory=6 * ggml.GB)), + sorted(ggml.SUGGESTED_LLM)) + + def test_the_wanted_model_wins_when_there_is_room_for_it(self): + items = [listed("ggml-tiny.bin", 70 << 20), + listed("ggml-large-v3-turbo-q5_0.bin", 574 << 20)] + self.assertEqual( + ggml.recommended(items, "ggml-large-v3-turbo-q5_0.bin", + memory=16 * ggml.GB), + "ggml-large-v3-turbo-q5_0.bin") + + def test_a_model_too_big_for_the_machine_is_not_recommended(self): + items = [listed("small.gguf", 1 << 30), listed("huge.gguf", 12 * ggml.GB)] + self.assertEqual(ggml.recommended(items, "huge.gguf", + memory=8 * ggml.GB), "small.gguf") + + def test_the_full_precision_weights_are_never_the_recommendation(self): + # Twice the memory and twice the wait for a difference this job + # cannot see. + items = [listed("model-Q4_0.gguf", 2 * ggml.GB), + listed("model-BF16.gguf", 3 * ggml.GB)] + self.assertEqual(ggml.recommended(items, memory=32 * ggml.GB), + "model-Q4_0.gguf") + + def test_nothing_is_recommended_when_nothing_fits(self): + self.assertEqual( + ggml.recommended([listed("huge.gguf", 40 * ggml.GB)], + memory=8 * ggml.GB), "") + + +class Grouping(Local): + """One group per model, rather than one long list sorted by size.""" + + def test_every_spelling_of_a_quantisation_reads_as_its_number(self): + # One list holds q5_1, Q4_K_M, MXFP4 and BF16, and the number is the + # whole of what any of them says to somebody choosing a row. + self.assertEqual(ggml.bit_depth("ggml-small-q5_1.bin"), 5) + self.assertEqual(ggml.bit_depth("SmolLM3-Q4_K_M.gguf"), 4) + self.assertEqual(ggml.bit_depth("gpt-oss-20b-MXFP4.gguf"), 4) + self.assertEqual(ggml.bit_depth("gemma-4-E2B-it-Q8_0.gguf"), 8) + # bf16 is not f16 read badly. + self.assertEqual(ggml.bit_depth("gemma-4-E2B-it-BF16.gguf"), 16) + self.assertEqual(ggml.bit_depth("mmproj-model-f16.gguf"), 16) + # A whisper file with no mark is the full model, and its name is the + # one convention here that does not carry the answer. + self.assertEqual(ggml.bit_depth("ggml-large-v3-turbo.bin"), 0) + + def test_a_quantisation_belongs_to_the_model_it_is_a_copy_of(self): + self.assertEqual(ggml.whisper_family("ggml-small.en-q5_1.bin"), "small") + self.assertEqual(ggml.whisper_family("ggml-large-v3-q5_0.bin"), + "large-v3") + self.assertEqual(ggml.whisper_family("ggml-large-v3-turbo.bin"), + "large-v3-turbo") + self.assertEqual(ggml.whisper_family("ggml-medium.en.bin"), "medium") + + def test_turbo_is_a_model_and_not_a_quantisation(self): + # The last chunk of the name is a quantisation for most of the list + # and part of the model's name here. + self.assertEqual(ggml.whisper_family("ggml-large-v3-turbo-q8_0.bin"), + "large-v3-turbo") + + def test_the_turbo_files_are_not_scattered_through_the_medium_ones(self): + # Sorted by size alone, large-v3-turbo-q5_0 lands between the two + # medium quantisations, half a screen from the model it is a copy of. + models = [listed("ggml-medium-q5_0.bin", 539 << 20), + listed("ggml-large-v3-turbo-q5_0.bin", 574 << 20), + listed("ggml-medium-q8_0.bin", 823 << 20), + listed("ggml-large-v3-turbo.bin", 1624 << 20)] + groups = dict(ggml.whisper_groups(models)) + self.assertEqual([i.name for i in groups["large-v3-turbo"]], + ["ggml-large-v3-turbo-q5_0.bin", + "ggml-large-v3-turbo.bin"]) + self.assertEqual([i.name for i in groups["medium"]], + ["ggml-medium-q5_0.bin", "ggml-medium-q8_0.bin"]) + + def test_the_smallest_model_comes_first_and_the_english_ones_last(self): + models = [listed("ggml-small.en-q5_1.bin", 190 << 20), + listed("ggml-small-q5_1.bin", 190 << 20), + listed("ggml-tiny.bin", 77 << 20)] + groups = ggml.whisper_groups(models) + self.assertEqual([family for family, _ in groups], ["tiny", "small"]) + self.assertEqual([i.name for _, group in groups for i in group], + ["ggml-tiny.bin", "ggml-small-q5_1.bin", + "ggml-small.en-q5_1.bin"]) diff --git a/tests/test_ui.py b/tests/test_ui.py index 18820c1..1324f33 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -1352,6 +1352,35 @@ class LocalModels(DikteTest): def _item(name, size=1 << 20): return hub.Item(name, f"https://example.invalid/{name}", size, "") + @staticmethod + def _rows(box): + """Every row's text, headings included.""" + return [box.model.itemText(row) for row in range(box.model.count())] + + @staticmethod + def _repos(box): + return [box.repo.itemText(row) for row in range(box.repo.count())] + + @staticmethod + def _roomy(): + """Stand on a machine with room for every suggestion. + + The order the publishers come in follows the memory, so a test that + reads it has to say which machine it is standing on. A build runner + with 7 GB in it puts the two Gemma 4 rows last and is right to. + """ + return mock.patch.object(ggml, "total_memory", return_value=64 << 30) + + @staticmethod + def _offered(box): + """The model names in the box, headings and duplicates left out.""" + names = [] + for row in range(box.model.count()): + name = box.model.itemData(row) + if name and name not in names: + names.append(name) + return names + def test_a_row_with_nothing_to_fetch_does_not_offer_a_download(self): # The model the settings name is not in the list any more, so its row # was rebuilt from the name alone and carries no file to fetch. The @@ -1388,7 +1417,7 @@ class LocalModels(DikteTest): box._on_listed([("models", [self._item("SmolLM3-Q4_K_M.gguf")], "ggml-org/SmolLM3-3B-GGUF")], "") self.assertEqual(box.selected(), "SmolLM3-Q4_K_M.gguf") - self.assertEqual(box.model.count(), 1) + self.assertEqual(self._offered(box), ["SmolLM3-Q4_K_M.gguf"]) def test_a_list_for_a_publisher_that_is_no_longer_chosen_is_dropped(self): # Every change starts its own request, and they do not come back in the @@ -1416,6 +1445,173 @@ class LocalModels(DikteTest): time.sleep(0.05) _app.processEvents() self.assertEqual(fetch.call_count, 1) + def test_the_models_are_grouped_by_the_model_rather_than_by_size(self): + # Sorted by size alone, the turbo files land between the two medium + # ones, half a screen from the model they are a copy of. + box = self.window(cfg.Config()).local_whisper + with mock.patch.object(ggml, "total_memory", return_value=8 << 30), \ + mock.patch.object(ggml, "accelerator", return_value=""): + box._on_listed([("models", [ + self._item("ggml-medium-q5_0.bin", 539 << 20), + self._item("ggml-large-v3-turbo-q5_0.bin", 574 << 20), + self._item("ggml-medium-q8_0.bin", 823 << 20), + self._item("ggml-large-v3-turbo.bin", 1624 << 20), + ], "")], "") + rows = self._rows(box) + # The two medium files under one heading, the two turbo ones under + # theirs, and the model rather than the file deciding the order. + self.assertEqual(rows[rows.index("medium"):], + ["medium", + "ggml-medium-q5_0.bin (539.0 MB, 5-bit)", + "ggml-medium-q8_0.bin (823.0 MB, 8-bit)", + "large-v3-turbo", + "ggml-large-v3-turbo-q5_0.bin " + "(574.0 MB, 5-bit, recommended)", + "ggml-large-v3-turbo.bin (1.6 GB, 16-bit)"]) + # A heading is not a model, and nothing can be saved from one. + self.assertIsNone(box.model.itemData(rows.index("medium"))) + + def test_the_row_for_this_machine_is_on_top_and_says_so(self): + box = self.window(cfg.Config()).local_whisper + with mock.patch.object(ggml, "total_memory", return_value=8 << 30), \ + mock.patch.object(ggml, "accelerator", return_value=""): + box._on_listed([("models", [ + self._item("ggml-tiny.bin", 77 << 20), + self._item("ggml-large-v3-turbo-q5_0.bin", 574 << 20), + ], "")], "") + self.assertEqual(box.selected(), "ggml-large-v3-turbo-q5_0.bin") + self.assertEqual(box.model.itemData(1), "ggml-large-v3-turbo-q5_0.bin") + self.assertIn(t("recommended"), box.model.itemText(1)) + + def test_a_model_the_memory_cannot_hold_says_so_on_its_row(self): + box = self.window(cfg.Config()).local_llm + box.repo.blockSignals(True) + box.repo.setCurrentText("ggml-org/x-GGUF") + box.repo.blockSignals(False) + with mock.patch.object(ggml, "total_memory", return_value=8 << 30): + box._on_listed([("models", [ + self._item("small-Q4_0.gguf", 1 << 30), + self._item("huge-Q8_0.gguf", 12 << 30), + ], "ggml-org/x-GGUF")], "") + rows = {box.model.itemData(row): box.model.itemText(row) + for row in range(box.model.count())} + self.assertNotIn(t("too big for this machine"), rows["small-Q4_0.gguf"]) + self.assertIn(t("too big for this machine"), rows["huge-Q8_0.gguf"]) + + def test_a_recommended_row_is_not_listed_twice_after_a_download(self): + # It has a row of its own on top as well as one in its group, and + # reading the rows back the way a finished download does was doubling + # it in the list every time. + box = self.window(cfg.Config()).local_whisper + with self._roomy(): + box._on_listed([("models", [ + self._item("ggml-tiny.bin", 77 << 20), + self._item("ggml-large-v3-turbo-q5_0.bin", 574 << 20), + ], "")], "") + before = self._offered(box) + box._fill_models_from_current() + self.assertEqual(self._offered(box), before) + names = [box.model.itemData(row) for row in range(box.model.count())] + self.assertEqual(len([n for n in names if n]), len(before) + 1) + + def test_a_processor_build_is_not_recommended_the_accurate_model(self): + # The Vulkan loader is on the machine but what was installed is the + # processor build, so there is no card in play whatever the loader + # says, and a 1 GB model on a processor is a wait somebody is sitting + # through with a sentence half typed. + binary = self.path("bin/whisper/v1.9.3/whisper-server") + binary.parent.mkdir(parents=True) + binary.write_text("") + binary.chmod(0o755) + self.path("bin/whisper/installed.json").write_text(json.dumps( + {"tag": "v1.9.3", "binary": str(binary), "backend": "processor"})) + self.patch_attr(ggml.shutil, "which", lambda name: None) + box = self.window(cfg.Config()).local_whisper + with mock.patch.object(ggml, "total_memory", return_value=32 << 30), \ + mock.patch.object(ggml, "accelerator", return_value="Vulkan"): + self.assertEqual(box._suggested(), ggml.SUGGESTED_WHISPER) + + def test_a_publisher_with_nothing_to_offer_says_why(self): + # Half of what ggml-org publishes is split across files or past the + # size cap, and an empty box read as though the click had not landed. + box = self.window(cfg.Config()).local_llm + box.repo.blockSignals(True) + box.repo.setCurrentText("ggml-org/gpt-oss-120b-GGUF") + box.repo.blockSignals(False) + box._on_listed([("models", [], "ggml-org/gpt-oss-120b-GGUF")], "") + self.assertIn("ggml-org/gpt-oss-120b-GGUF", box.status.text()) + self.assertIn("publisher", box.status.text()) + + def test_an_empty_box_nobody_has_asked_yet_is_not_a_publisher_fault(self): + box = self.window(cfg.Config()).local_llm + box.load("", "ggml-org/SmolLM3-3B-GGUF") + self.assertNotIn("publisher", box.status.text()) + + def test_only_the_suggested_publishers_are_offered_to_start_with(self): + # Forty repository ids is not a choice anybody can make. + box = self.window(cfg.Config()).local_llm + with self._roomy(): + box._on_listed([("repos", [ggml.SUGGESTED_LLM[0], + "ggml-org/something-else-GGUF"], "")], "") + self.assertEqual(self._repos(box), list(ggml.SUGGESTED_LLM)) + + def test_a_suggestion_missing_from_the_listing_is_still_offered(self): + # The listing is the forty repositories touched most recently, and a + # publisher that has not been updated in a season falls off it while + # still being the one to point at. + box = self.window(cfg.Config()).local_llm + box._on_listed([("repos", ["ggml-org/something-else-GGUF"], "")], "") + self.assertIn(ggml.SUGGESTED_LLM[0], self._repos(box)) + + def test_the_switch_brings_the_rest_and_keeps_them_apart(self): + box = self.window(cfg.Config()).local_llm + with self._roomy(): + box._on_listed([("repos", [ggml.SUGGESTED_LLM[0], + "ggml-org/something-else-GGUF"], "")], "") + box.every_repo.setChecked(True) + rows = self._repos(box) + self.assertEqual(rows[:len(ggml.SUGGESTED_LLM)], + list(ggml.SUGGESTED_LLM)) + # A separator rather than a heading: the box is typed into as well as + # chosen from, and a heading would land in the field as a repository. + self.assertEqual(rows[len(ggml.SUGGESTED_LLM)], "") + self.assertEqual(rows[-1], "ggml-org/something-else-GGUF") + + def test_a_publisher_typed_in_is_not_dropped_by_the_next_fetch(self): + box = self.window(cfg.Config()).local_llm + box.repo.blockSignals(True) + box.repo.setCurrentText("ggml-org/something-else-GGUF") + box.repo.blockSignals(False) + box._on_listed([("repos", [ggml.SUGGESTED_LLM[0], + "ggml-org/something-else-GGUF"], "")], "") + self.assertFalse(box.every_repo.isChecked()) + self.assertIn("ggml-org/something-else-GGUF", self._repos(box)) + self.assertEqual(box.repository(), "ggml-org/something-else-GGUF") + + def test_the_chosen_publisher_is_said_in_words(self): + # A repository id names the publisher, the parameter count and the + # shape of the weights, and none of that says whether to click it. + box = self.window(cfg.Config()).local_llm + box.repo.setCurrentText(ggml.SUGGESTED_LLM[0]) + self.assertTrue(box.repo_note.text()) + box.repo.setCurrentText("ggml-org/nobody-wrote-a-note-GGUF") + self.assertEqual(box.repo_note.text(), "") + + def test_the_box_says_what_this_machine_will_run_on(self): + box = self.window(cfg.Config()).local_whisper + with mock.patch.object(ggml, "accelerator", return_value="Vulkan"), \ + mock.patch.object(ggml, "total_memory", return_value=32 << 30): + box._show_machine() + self.assertIn("Vulkan", box.machine_label.text()) + self.assertIn("32.0 GB", box.machine_label.text()) + + def test_a_machine_with_no_card_is_told_it_is_on_the_processor(self): + box = self.window(cfg.Config()).local_whisper + with mock.patch.object(ggml, "accelerator", return_value=""), \ + mock.patch.object(ggml, "total_memory", return_value=8 << 30): + box._show_machine() + self.assertIn("processor", box.machine_label.text()) + def test_a_processor_build_where_the_vulkan_one_belongs_says_so(self): # The Vulkan whisper-server is published by hand, and until it is # there the download lands upstream's processor build. Said nowhere,