mirror of
https://github.com/yusufipk/dikte.git
synced 2026-09-11 19:06:11 +00:00
Group the model lists and say which row this machine should take
The two local model boxes handed over a flat list sorted by size and left every choice in it to the reader. For whisper that interleaved the models: large-v3-turbo-q5_0 landed between the two medium quantisations, half a screen from the turbo model it is a copy of. For cleanup it was forty repository ids, half of which answer with nothing at all because what they publish is split across files or larger than the cap, and an empty box read as though the click had not registered. Now each box says what the machine is, groups the list by model, and marks the row to take: - whisper rows are grouped by model, with the quantisations and the English-only files under the model they are a copy of, and every row says its bit depth rather than leaving q5_1 and Q4_K_M and BF16 to be decoded. - the recommendation follows the machine. Under 4 GB it is small-q5_1; with a graphics interface and 15 GB it is large-v3-q5_0, which is worth about two and a half points of word error in the languages that are not English; in between it is turbo, and a processor build where the Vulkan one belongs is not counted as a card. - a row larger than half the memory less a gigabyte says it is too big. - the publisher box holds the five suggestions until the switch beside it is turned on, and a line under it says in words what the chosen one is. - a publisher that answers with nothing says why instead of going blank. - the draft heads (dflash, dspark, eagle3) are no longer offered as models, and neither are the base models that sit beside their tuned twin.
This commit is contained in:
+310
-9
@@ -26,6 +26,7 @@ interface already knows how to show.
|
||||
|
||||
import atexit
|
||||
import collections
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
import hashlib
|
||||
import http.client
|
||||
@@ -98,28 +99,126 @@ NIGHTLY_TAG = "nightly-tag.txt"
|
||||
# hardware and the odd loose file.
|
||||
WHISPER_PREFIX = "ggml-"
|
||||
WHISPER_SUFFIX = ".bin"
|
||||
# The mark on the whisper models trained on English alone. They are half of the
|
||||
# list, and they belong under the model they are a variant of rather than
|
||||
# scattered through it by size.
|
||||
ENGLISH_ONLY = ".en"
|
||||
|
||||
# Full-precision weights, however they are spelled. Several times the memory of
|
||||
# a quantisation of the same model, for a difference dictation and cleanup
|
||||
# cannot see, so nothing here ever points at one.
|
||||
SIXTEEN_BIT = ("bf16", "f16", "fp16")
|
||||
|
||||
# How many bits a weight is stored in, read off the file name. Every one of
|
||||
# these lists spells it differently, `q5_1` and `Q4_K_M` and `MXFP4` and
|
||||
# `BF16`, and the only part of that anybody choosing between two rows needs is
|
||||
# the number. Longest mark first, so `bf16` is not read as `f16`.
|
||||
BIT_DEPTHS = (("mxfp4", 4), ("bf16", 16), ("fp16", 16), ("f16", 16),
|
||||
("q2", 2), ("q3", 3), ("q4", 4), ("q5", 5), ("q6", 6), ("q8", 8))
|
||||
|
||||
# What a GGUF repository holds besides the model: mmproj is the vision half of a
|
||||
# multimodal model, mtp a draft head for speculative decoding. Neither is a model
|
||||
# a server can be started on, and offering them is offering a failure.
|
||||
GGUF_SKIP = ("mmproj", "mtp-")
|
||||
# multimodal model, and mtp, dflash, dspark and eagle3 are draft heads for
|
||||
# speculative decoding. None of them is a model a server can be started on, and
|
||||
# they are the small files in the repository, so a list sorted by size puts them
|
||||
# at the top where they are likeliest to be clicked.
|
||||
GGUF_SKIP = ("mmproj", "mtp-", "dflash-", "dspark-", "eagle3-", "draft-")
|
||||
# Big enough for a 12B at Q4 and far past anything cleanup wants; the point is
|
||||
# to keep a 400 GB frontier model out of a list somebody might click.
|
||||
GGUF_MAX_BYTES = 16 << 30
|
||||
|
||||
# Repositories that carry GGUF files but nothing a cleanup server can be started
|
||||
# on: a vision or audio tower with no text half worth running, a speech model,
|
||||
# and the base models, which continue text rather than following an instruction
|
||||
# and answer a cleanup prompt by carrying on writing the transcript.
|
||||
LLM_REPO_SKIP = ("-Base-GGUF", "-VL-", "-Vision-", "-Omni-", "-Video-",
|
||||
"-TTS-", "parakeet", "test-")
|
||||
|
||||
GB = 1 << 30
|
||||
|
||||
# Suggestions, not a catalogue: the list itself is fetched, and these are only
|
||||
# the rows that float to the top of it. Small instruction-following models,
|
||||
# because cleanup is punctuation and filler words rather than anything that
|
||||
# wants thinking about.
|
||||
# the rows that float to the top of it. Cleanup is punctuation, capitals and
|
||||
# filler words rather than anything that wants thinking about, so what it is
|
||||
# picked on is instruction following at a size a desktop can spare. Gemma 4
|
||||
# scores 94.6 on IFEval at E2B and 96.7 at E4B, and E2B leads here rather than
|
||||
# E4B because two points of instruction following is not worth twice the
|
||||
# weights on a job that runs while somebody waits for their sentence to appear.
|
||||
# SmolLM3 and Gemma 3 are the older pair below them. Qwen3.5 0.8B is for the
|
||||
# machines nothing else fits on; it thinks before it answers, which is what the
|
||||
# Thinking box in the settings window turns off.
|
||||
SUGGESTED_LLM = (
|
||||
"ggml-org/gemma-3-4b-it-GGUF",
|
||||
"ggml-org/gemma-4-E2B-it-GGUF",
|
||||
"ggml-org/gemma-4-E4B-it-GGUF",
|
||||
"ggml-org/gemma-3-4b-it-GGUF",
|
||||
"ggml-org/SmolLM3-3B-GGUF",
|
||||
"ggml-org/Qwen3.5-0.8B-GGUF",
|
||||
)
|
||||
# Roughly what each of those weighs at the quantisation cleanup would run, to
|
||||
# the nearest half gigabyte. Not a catalogue of files: the sizes on the rows
|
||||
# come from the publisher, and this only decides which suggestion is offered
|
||||
# first on a machine that has room for some of them and not others.
|
||||
SUGGESTED_LLM_SIZE = {
|
||||
"ggml-org/gemma-4-E2B-it-GGUF": 3 * GB,
|
||||
"ggml-org/gemma-4-E4B-it-GGUF": 5 * GB,
|
||||
"ggml-org/gemma-3-4b-it-GGUF": 5 * GB // 2,
|
||||
"ggml-org/SmolLM3-3B-GGUF": 2 * GB,
|
||||
"ggml-org/Qwen3.5-0.8B-GGUF": GB // 2,
|
||||
}
|
||||
|
||||
# What each of them is, in the words somebody choosing between them would
|
||||
# use. A repository id says the publisher, the parameter count, the shape of
|
||||
# the weights and nothing at all about whether it is the one to click, and
|
||||
# `ggml-org/gemma-4-E2B-it-GGUF` reads as four pieces of jargon to everybody
|
||||
# who has not been reading model cards all year.
|
||||
SUGGESTED_LLM_NOTE = {
|
||||
"ggml-org/gemma-4-E2B-it-GGUF":
|
||||
"Google Gemma 4, the small one. The default: nothing else this size "
|
||||
"follows an instruction as closely, and cleanup is all instruction.",
|
||||
"ggml-org/gemma-4-E4B-it-GGUF":
|
||||
"The same model one size up. A little more accurate, about twice the "
|
||||
"weights and twice the wait.",
|
||||
"ggml-org/gemma-3-4b-it-GGUF":
|
||||
"The previous Gemma. Still good, and the smallest of the Gemmas here.",
|
||||
"ggml-org/SmolLM3-3B-GGUF":
|
||||
"Hugging Face's own small model, for a machine the Gemmas crowd.",
|
||||
"ggml-org/Qwen3.5-0.8B-GGUF":
|
||||
"The smallest of them, for a machine nothing else fits on. It thinks "
|
||||
"before it answers unless Thinking below is off.",
|
||||
}
|
||||
|
||||
# Turbo at q5_0 is smaller than `small` and better than it, which makes the
|
||||
# usual "start small" advice point at the same file as "start good".
|
||||
# usual "start small" advice point at the same file as "start good". It is
|
||||
# large-v3 with the decoder cut from 32 layers to 4: several times faster, at
|
||||
# one to two points of word error in English and about two and a half in the
|
||||
# other languages.
|
||||
SUGGESTED_WHISPER = "ggml-large-v3-turbo-q5_0.bin"
|
||||
# Those two and a half points back, for twice the file and several times the
|
||||
# work per second. Only suggested where there is a card to do the work and
|
||||
# memory to hold it, because that is where the trade stops costing anything a
|
||||
# person waiting for a dictation would notice.
|
||||
ACCURATE_WHISPER = "ggml-large-v3-q5_0.bin"
|
||||
# What to point at instead on a machine the turbo model would crowd. Same
|
||||
# quantisation ladder, one rung down in size and in accuracy.
|
||||
SMALL_MACHINE_WHISPER = "ggml-small-q5_1.bin"
|
||||
# Under this much system memory, a 600 MB model plus the rest of a desktop is
|
||||
# already tight, so the suggestion drops to the smaller one. Over the other,
|
||||
# the accurate model is the one to point at.
|
||||
SMALL_MACHINE = 4 * GB
|
||||
# Fifteen and not sixteen: what the machine reports is what is left after the
|
||||
# firmware and the graphics have taken their reservations out of it, and a
|
||||
# 16 GB machine answers about 15.4. A threshold written at the number on the
|
||||
# box is one no machine sold as that size ever reaches.
|
||||
ROOMY_MACHINE = 15 * GB
|
||||
# What a model may take of this machine's memory before it is called too big:
|
||||
# half of it, less a gigabyte for the context and the runtime around the
|
||||
# weights. A rule of thumb rather than a measurement, and deliberately a
|
||||
# cautious one, because the failure it is guarding against is a machine that
|
||||
# swaps itself to a standstill rather than a model that refuses to load.
|
||||
MEMORY_SHARE = 0.5
|
||||
MEMORY_OVERHEAD = GB
|
||||
# What is left to offer on a machine too small for the sum above to leave
|
||||
# anything. Enough for the smallest whisper models and for a sub-billion
|
||||
# cleanup model, which is what such a machine can run.
|
||||
MEMORY_FLOOR = GB // 2
|
||||
|
||||
|
||||
class LocalError(Exception):
|
||||
@@ -605,9 +704,190 @@ def _drop_old_versions(program, keep):
|
||||
pass
|
||||
|
||||
|
||||
# --- what this machine can run --------------------------------------------
|
||||
|
||||
|
||||
def total_memory():
|
||||
"""Bytes of memory on this machine, or 0 when it cannot be read.
|
||||
|
||||
Zero is a real answer and not a failure: every caller treats an unknown
|
||||
machine as one big enough for whatever it is looking at, because a wrong
|
||||
"too big" is worse advice than none.
|
||||
"""
|
||||
try:
|
||||
return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
|
||||
except (AttributeError, ValueError, OSError):
|
||||
pass
|
||||
if sys.platform == "darwin":
|
||||
# Not every build of Python on a Mac has SC_PHYS_PAGES in its sysconf
|
||||
# table, and this is the number the system itself is asked for.
|
||||
try:
|
||||
out = subprocess.run(["sysctl", "-n", "hw.memsize"], check=True,
|
||||
capture_output=True, text=True, timeout=5)
|
||||
return int(out.stdout.strip())
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
return 0
|
||||
if sys.platform != "win32":
|
||||
return 0
|
||||
|
||||
class Status(ctypes.Structure):
|
||||
_fields_ = [("dwLength", ctypes.c_ulong),
|
||||
("dwMemoryLoad", ctypes.c_ulong),
|
||||
("ullTotalPhys", ctypes.c_ulonglong),
|
||||
("ullAvailPhys", ctypes.c_ulonglong),
|
||||
("ullTotalPageFile", ctypes.c_ulonglong),
|
||||
("ullAvailPageFile", ctypes.c_ulonglong),
|
||||
("ullTotalVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailExtendedVirtual", ctypes.c_ulonglong)]
|
||||
|
||||
try:
|
||||
status = Status()
|
||||
status.dwLength = ctypes.sizeof(Status)
|
||||
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
|
||||
return int(status.ullTotalPhys)
|
||||
except (AttributeError, OSError, ValueError):
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def accelerator():
|
||||
"""The graphics interface this machine offers, or "".
|
||||
|
||||
The machine's half of the answer only. Whether a card is actually reached
|
||||
also depends on which build landed, and the program line above says that:
|
||||
a processor build ignores the card whatever is installed here. What this
|
||||
is for is the other half, which nothing else on the window says at all.
|
||||
"""
|
||||
if sys.platform == "darwin":
|
||||
return "Metal"
|
||||
return "Vulkan" if _has_vulkan() else ""
|
||||
|
||||
|
||||
def memory_budget(memory=None):
|
||||
"""What a model may weigh on this machine, or 0 when that is unknown.
|
||||
|
||||
Floored rather than allowed to reach zero: on a 2 GB machine the share
|
||||
less the overhead is nothing at all, and a budget of nothing is the same
|
||||
number this returns for a machine it could not read, which would turn the
|
||||
tightest machine there is into the one where everything is offered.
|
||||
"""
|
||||
memory = total_memory() if memory is None else memory
|
||||
if not memory:
|
||||
return 0
|
||||
return max(int(memory * MEMORY_SHARE) - MEMORY_OVERHEAD, MEMORY_FLOOR)
|
||||
|
||||
|
||||
def fits(size, memory=None):
|
||||
"""Whether a model of this size is worth offering on this machine."""
|
||||
budget = memory_budget(memory)
|
||||
return not budget or size <= budget
|
||||
|
||||
|
||||
def suggested_whisper(memory=None, graphics=None):
|
||||
"""The whisper model to point at here, by name.
|
||||
|
||||
Three machines. One with no room, which gets the model that leaves some.
|
||||
One with a card and memory to spare, which gets the accurate model, because
|
||||
the several times the work it is per second is several times a fraction of
|
||||
a second there. Everything in between gets turbo, which is the answer
|
||||
almost every time somebody asks.
|
||||
|
||||
A Vulkan or Metal loader is not proof of a fast card, so the accurate model
|
||||
waits on the memory as well: a machine with 16 GB in it and a driver
|
||||
installed is one that will not notice either way.
|
||||
"""
|
||||
memory = total_memory() if memory is None else memory
|
||||
graphics = accelerator() if graphics is None else graphics
|
||||
if memory and memory < SMALL_MACHINE:
|
||||
return SMALL_MACHINE_WHISPER
|
||||
if graphics and memory >= ROOMY_MACHINE:
|
||||
return ACCURATE_WHISPER
|
||||
return SUGGESTED_WHISPER
|
||||
|
||||
|
||||
def suggested_llm(memory=None):
|
||||
"""The suggested cleanup repositories, the ones that fit here first.
|
||||
|
||||
The order they are written in is the order they are worth having. What
|
||||
this changes is only which of them a machine that cannot hold the best one
|
||||
is shown first, and nothing is dropped: a model that does not fit today
|
||||
fits once something else is closed.
|
||||
"""
|
||||
return sorted(SUGGESTED_LLM,
|
||||
key=lambda repo: not fits(SUGGESTED_LLM_SIZE.get(repo, 0),
|
||||
memory))
|
||||
|
||||
|
||||
def recommended(items, want="", memory=None):
|
||||
"""The one row out of `items` worth pointing at here, or "".
|
||||
|
||||
`want` is taken when it is on offer and fits. Without it, which is the
|
||||
cleanup list, the smallest file that does is taken: q4 is where these
|
||||
lists start, and every rung above it is roughly twice the memory and twice
|
||||
the wait for a difference neither dictation nor cleanup can see. The
|
||||
16-bit weights are left out for the same reason, twice over.
|
||||
"""
|
||||
fitting = [i for i in items if fits(i.size, memory)]
|
||||
if want and any(i.name == want for i in fitting):
|
||||
return want
|
||||
usable = [i for i in fitting
|
||||
if not any(mark in i.name.lower() for mark in SIXTEEN_BIT)]
|
||||
return min(usable, key=lambda i: i.size).name if usable else ""
|
||||
|
||||
|
||||
# --- the models -----------------------------------------------------------
|
||||
|
||||
|
||||
def bit_depth(name):
|
||||
"""The bits per weight the file name says, or 0 when it says nothing."""
|
||||
lowered = name.lower()
|
||||
for mark, bits in BIT_DEPTHS:
|
||||
if mark in lowered:
|
||||
return bits
|
||||
return 0
|
||||
|
||||
|
||||
def whisper_family(name):
|
||||
"""The model a whisper file belongs to: ggml-small.en-q5_1.bin is `small`.
|
||||
|
||||
The list arrives sorted by size and nothing else, which interleaves the
|
||||
families: `large-v3-turbo-q5_0` lands between the two `medium`
|
||||
quantisations, half a screen from the turbo model it is a copy of. Grouping
|
||||
is what puts the choice between models above the choice of quantisation,
|
||||
which is the order somebody actually makes them in.
|
||||
"""
|
||||
stem = name
|
||||
if stem.startswith(WHISPER_PREFIX):
|
||||
stem = stem[len(WHISPER_PREFIX):]
|
||||
if stem.endswith(WHISPER_SUFFIX):
|
||||
stem = stem[:-len(WHISPER_SUFFIX)]
|
||||
head, _, last = stem.rpartition("-")
|
||||
# q5_0, q5_1, q8_0. `turbo` is the other thing a last chunk can be, and it
|
||||
# is part of the model's name rather than a quantisation of it.
|
||||
if head and last.startswith("q") and last[1:].replace("_", "").isdigit():
|
||||
stem = head
|
||||
return stem[:-len(ENGLISH_ONLY)] if stem.endswith(ENGLISH_ONLY) else stem
|
||||
|
||||
|
||||
def whisper_groups(items):
|
||||
"""[(family, [Item])] for a whisper list: one group per model.
|
||||
|
||||
Groups by how big the model gets rather than by a ladder written down
|
||||
here, so a family published next year sorts itself. Inside one, the
|
||||
multilingual files come before the English-only ones and the small
|
||||
quantisations before the large.
|
||||
"""
|
||||
groups = {}
|
||||
for item in items:
|
||||
groups.setdefault(whisper_family(item.name), []).append(item)
|
||||
ordered = sorted(groups.items(),
|
||||
key=lambda pair: (max(i.size for i in pair[1]), pair[0]))
|
||||
return [(family, sorted(files,
|
||||
key=lambda i: (ENGLISH_ONLY in i.name, i.size)))
|
||||
for family, files in ordered]
|
||||
|
||||
|
||||
def whisper_models(refresh=False):
|
||||
"""[hub.Item] for every whisper model on offer, smallest first."""
|
||||
try:
|
||||
@@ -620,10 +900,23 @@ def whisper_models(refresh=False):
|
||||
return sorted(models, key=lambda f: f.size)
|
||||
|
||||
|
||||
def can_clean(repo):
|
||||
"""Whether a repository could hold a model cleanup can be started on.
|
||||
|
||||
By name, because the alternative is a file listing per repository and the
|
||||
list is forty of them. It catches the kinds that are never a cleanup model
|
||||
rather than the ones that are too big, which the file sizes answer exactly
|
||||
once a publisher is chosen.
|
||||
"""
|
||||
lowered = repo.lower()
|
||||
return not any(mark.lower() in lowered for mark in LLM_REPO_SKIP)
|
||||
|
||||
|
||||
def llm_repos(refresh=False):
|
||||
"""Repository ids for the GGUF models on offer, suggestions first."""
|
||||
try:
|
||||
found = [r.id for r in hub.repos(author=LLM_AUTHOR, refresh=refresh)]
|
||||
found = [r.id for r in hub.repos(author=LLM_AUTHOR, refresh=refresh)
|
||||
if can_clean(r.id)]
|
||||
except hub.HubError:
|
||||
# A menu rather than a catalogue: with nothing to show, the suggestions
|
||||
# are still worth showing, and whatever is wrong with the network will
|
||||
@@ -631,6 +924,14 @@ def llm_repos(refresh=False):
|
||||
found = []
|
||||
if not found:
|
||||
return list(SUGGESTED_LLM)
|
||||
# Gemma publishes its base models under the instruction-tuned one's name
|
||||
# with the `-it` taken out, so the two sit next to each other in the list
|
||||
# and the wrong one answers a cleanup prompt by carrying on writing the
|
||||
# transcript. Dropped only where the tuned sibling is here to drop it for.
|
||||
tuned = set(found)
|
||||
found = [r for r in found
|
||||
if not r.endswith("-GGUF")
|
||||
or r[:-len("-GGUF")] + "-it-GGUF" not in tuned]
|
||||
first = [r for r in SUGGESTED_LLM if r in found]
|
||||
return first + [r for r in found if r not in first]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user