From 70bc4c16fad2b933281bc7c46cc3bf6f12095ff6 Mon Sep 17 00:00:00 2001 From: yusufipk Date: Sat, 5 Sep 2026 12:02:28 +0300 Subject: [PATCH 1/3] Give a local model room to think without spending the answer on it llama.cpp counts the thinking towards max_tokens along with the answer it precedes, and the local ceiling was sized for the answer alone. Turning Thinking up therefore came out of the reply rather than being added to it, and on a short dictation the 512 floor is the whole budget, so the model spent it in the think block and came back with nothing to paste. Each rung of the ladder now carries its own budget, doubling from 256 at "minimal" to 8192 at "maximum", added on top of the answer's share rather than taken out of it. The rungs are small because cleanup is punctuation and locally every one of these tokens is also a second of somebody standing in front of the screen. "Off" keeps the old tight ceiling untouched, and an empty setting is given a middling amount, since a template that can think thinks by default and there is no way to ask which kind of model this is. The ceiling is also held under what the server was started with. Above the context it is not a ceiling at all: the runaway it exists to stop would run to the end of the context instead, which on CPU is minutes of waiting. The prompt keeps its share at two characters to the token, which is under any tokeniser's rate for natural language and so reserves too much rather than promising room that is not there. Separately, a reply cut off at somebody's ceiling was returned as if it were whole. Half a sentence looks like a cleaned-up transcript and is not one, so finish_reason is now read in both cleanup and chat. The callers already keep the transcript they started with, which is the better of the two. This one is not local-only: a hosted provider stopping at its own output limit was silently pasted the same way. --- dikte/api.py | 68 +++++++++++++++++++++++++++++++++++++++---- dikte/cleanup.py | 3 ++ dikte/i18n.py | 4 +++ tests/test_api.py | 38 +++++++++++++++++++++++- tests/test_cleanup.py | 46 +++++++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 7 deletions(-) diff --git a/dikte/api.py b/dikte/api.py index 13c4381..58cc30b 100644 --- a/dikte/api.py +++ b/dikte/api.py @@ -518,22 +518,65 @@ def _thinking(payload, provider, reasoning): payload["reasoning"] = {"effort": reasoning, "exclude": True} -def local_ceiling(text): +# Room for the thinking on this machine, one budget per rung of the settings +# ladder. llama.cpp counts the thinking towards max_tokens along with the answer +# it precedes, so a ceiling sized for the answer alone leaves a model that +# thinks nothing to answer with. The rungs double, starting where a small model +# lands when it barely thinks at all: cleanup is punctuation, and locally every +# one of these tokens is also a second of somebody standing in front of the +# screen, so the low rungs are the ones meant to be used. +THINKING_ROOM = { + "minimal": 256, "low": 512, "medium": 1024, + "high": 2048, "xhigh": 4096, "max": 8192, +} +# An empty setting leaves it to the model, and the templates that can think +# think by default. Room for a middling amount of it, since there is no way to +# ask which kind of model this is. +DEFAULT_THINKING_ROOM = THINKING_ROOM["medium"] + + +def local_ceiling(text, reasoning="", context=0, prompt=""): """How much of a reply is worth waiting for from a model on this machine. Cleanup gives back what it was given, near enough, so a reply several times the length of the transcript is a model that has lost the thread rather than one doing the job. A small one will happily repeat the transcript until the context is full, and every one of those tokens is a second of somebody - waiting. A hosted model is left alone: there the same runaway is rare, and a - ceiling would cut the minutes short instead. + waiting, with only the hour-long local timeout underneath. A hosted model is + left alone: there the same runaway is rare, and a ceiling would cut the + minutes short instead. + + The answer's share is the transcript's length in characters spent as a + budget in tokens, so what it really allows is two to four times the + transcript depending on how well the language tokenises. Turkish sits at the + tight end of that and still has room to spare for a reply that is meant to + come back the same length it went in. + + Thinking is added on top of that share rather than taken out of it. Sharing + one budget is what makes turning thinking up quietly cost the answer, and on + a short dictation the 512 floor is the whole budget, so the answer is what + goes missing first. + + `context` is what the server was started with, and the whole of it is the + real limit whatever is asked for here: a ceiling above it is not a ceiling, + because the runaway it exists to stop would run to the end of the context + instead. So the ceiling is held below what the prompt leaves. Two characters + to the token is under any tokeniser's rate for natural language, Turkish + included, which makes the reserve an over-estimate rather than a promise of + room that is not there. """ - return max(512, len(text)) + answer = max(512, len(text)) + if reasoning != "none": + answer += THINKING_ROOM.get(reasoning, DEFAULT_THINKING_ROOM) + context = int(context or 0) + if not context: + return answer + return max(256, min(answer, context - (len(prompt) + len(text)) // 2)) def cleanup(text, api_key, model, system_prompt, reasoning="", base_url=OPENROUTER_URL, timeout=180, provider="openrouter", - service="OpenRouter", aborter=None): + service="OpenRouter", aborter=None, context=0): if not api_key and provider != "local-llm": raise ApiError(t("{service} API key is empty. Add it in Settings.", service=service)) @@ -546,7 +589,8 @@ def cleanup(text, api_key, model, system_prompt, reasoning="", ], } if provider == "local-llm": - payload["max_tokens"] = local_ceiling(text) + payload["max_tokens"] = local_ceiling(text, reasoning, context, + system_prompt) _thinking(payload, provider, reasoning) try: data = _request( @@ -570,6 +614,13 @@ def cleanup(text, api_key, model, system_prompt, reasoning="", raise ApiError(t("The cleanup model spent its whole reply on " "thinking. Set Thinking to \u201cOff\u201d.")) raise ApiError(t("The cleanup model returned an empty reply.")) + if choices[0].get("finish_reason") == "length": + # Cut off at somebody's ceiling: ours locally, the provider's otherwise. + # What came back is a sentence that stops mid-word, and cleanup is meant + # to hand back the whole dictation, so the half is refused rather than + # returned. The callers keep the transcript they started with, which is + # the better of the two. + raise ApiError(t("The cleanup model was cut off before it finished.")) return content @@ -605,6 +656,11 @@ def chat(messages, api_key, model, system_prompt, reasoning="", content = ((choices[0].get("message") or {}).get("content") or "").strip() if not content: raise ApiError(t("The model returned an empty reply.")) + if choices[0].get("finish_reason") == "length": + # An answer that stops mid-sentence reads like a whole one once it has + # been pasted, so it is refused here for the same reason cleanup refuses + # a half transcript. + raise ApiError(t("The model was cut off before it finished.")) return content diff --git a/dikte/cleanup.py b/dikte/cleanup.py index a1bced3..36a3ee9 100644 --- a/dikte/cleanup.py +++ b/dikte/cleanup.py @@ -121,6 +121,9 @@ def _local(text, conf, system_prompt, timeout, aborter=None): base_url=api.serving(ggml.llm), timeout=max(timeout, api.LOCAL_TIMEOUT), provider="local-llm", service=service, aborter=aborter, + # The ceiling is only a ceiling while it sits under what the server + # was started with; above that the context is what stops the reply. + context=ggml.llm.settings()["context"], ) except api.ApiError as exc: # A server that died mid-request would otherwise report only that the diff --git a/dikte/i18n.py b/dikte/i18n.py index 50c497d..6375e55 100644 --- a/dikte/i18n.py +++ b/dikte/i18n.py @@ -959,6 +959,10 @@ TR = { "“Off”.": "Temizleme modeli bütün yanıtını düşünmeye harcadı. Düşünme'yi " "“Kapalı” yap.", + "The cleanup model was cut off before it finished.": + "Temizleme modeli bitiremeden kesildi.", + "The model was cut off before it finished.": + "Model bitiremeden kesildi.", # --- this pass's new messages --------------------------------------- "Audio recorder stopped before receiving sound": diff --git a/tests/test_api.py b/tests/test_api.py index 50b4ef7..4b35774 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -463,6 +463,29 @@ class Cleanup(DikteTest): with fake_urlopen(chat_reply(" ")), self.assertRaises(api.ApiError): api.cleanup("hello", "k", "m", "p") + def test_a_reply_cut_off_at_a_ceiling_is_refused_rather_than_pasted(self): + # Half a sentence looks like a cleaned-up transcript and is not one. The + # caller keeps what it was given, which is the whole dictation. + reply = {"choices": [{"message": {"content": "Hello, and then the"}, + "finish_reason": "length"}]} + with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught: + api.cleanup("hello", "k", "m", "p") + self.assertIn("cut off", str(caught.exception)) + + def test_a_reply_that_stopped_on_its_own_is_kept(self): + reply = {"choices": [{"message": {"content": "Hello."}, + "finish_reason": "stop"}]} + with fake_urlopen(reply): + self.assertEqual(api.cleanup("hello", "k", "m", "p"), "Hello.") + + def test_all_thinking_is_named_before_the_ceiling_it_was_cut_at(self): + """Both are true at once, and only one of them says what to change.""" + reply = {"choices": [{"message": {"content": "", "reasoning": "hmm"}, + "finish_reason": "length"}]} + with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught: + api.cleanup("hello", "k", "m", "p") + self.assertIn("Thinking", str(caught.exception)) + def test_a_rate_limit_is_explained(self): with fake_urlopen(http_error(429)), \ self.assertRaises(api.ApiError) as caught: @@ -471,6 +494,14 @@ class Cleanup(DikteTest): class Chat(DikteTest): + def test_an_answer_cut_off_at_a_ceiling_is_refused_rather_than_pasted(self): + # Half an answer reads like a whole one once it is on the screen. + reply = {"choices": [{"message": {"content": "Booked it for the"}, + "finish_reason": "length"}]} + with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught: + api.chat([{"role": "user", "content": "book it"}], "k", "m", "p") + self.assertIn("cut off", str(caught.exception)) + def test_the_history_is_sent_after_the_system_prompt(self): history = [{"role": "user", "content": "book it"}, {"role": "assistant", "content": "done"}] @@ -616,11 +647,13 @@ if __name__ == "__main__": class FakeServer: """A ggml.Server as far as api.py is concerned.""" - def __init__(self, url="http://127.0.0.1:9999/v1", fails="", log=""): + def __init__(self, url="http://127.0.0.1:9999/v1", fails="", log="", + context=8192): self.url = url self.fails = fails self.log = log self.starts = 0 + self.context = context def serve(self): self.starts += 1 @@ -631,6 +664,9 @@ class FakeServer: def error(self): return self.log + def settings(self): + return {"context": self.context} + LOCAL = api.Target("local", "Local whisper", "", "", "ggml-base.bin") diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index 9de3dc3..245053f 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -411,6 +411,52 @@ class Here(DikteTest): cleanup.run("uh, done", self.conf, "the rules") self.assertEqual(sent_json(calls[0])["max_tokens"], 512) + def test_thinking_is_given_room_of_its_own_rather_than_the_answer_s(self): + # llama.cpp counts the thinking towards the same ceiling, so a rung that + # took its budget out of the answer would leave a short dictation with + # nothing to reply with. On a context roomy enough that the clamp the + # top rung would otherwise meet is not what is being measured. + self.patch_attr(ggml, "llm", FakeServer(context=32768)) + for rung, room in api.THINKING_ROOM.items(): + with self.subTest(rung=rung): + self.conf["local_llm_reasoning"] = rung + with fake_urlopen(chat_reply("Done.")) as calls: + cleanup.run("uh, done", self.conf, "the rules") + self.assertEqual(sent_json(calls[0])["max_tokens"], 512 + room) + + def test_each_rung_of_the_ladder_thinks_longer_than_the_one_below(self): + rungs = [api.THINKING_ROOM[name] for name in + ("minimal", "low", "medium", "high", "xhigh", "max")] + self.assertEqual(rungs, sorted(rungs)) + self.assertEqual(len(set(rungs)), len(rungs)) + + def test_the_models_own_default_is_given_room_to_think_in_too(self): + # Nothing is sent, so a template that thinks will think, and the ceiling + # has to survive that as well. + self.conf["local_llm_reasoning"] = "" + with fake_urlopen(chat_reply("Done.")) as calls: + cleanup.run("uh, done", self.conf, "the rules") + self.assertEqual(sent_json(calls[0])["max_tokens"], + 512 + api.DEFAULT_THINKING_ROOM) + + def test_the_ceiling_stays_under_the_context_the_server_was_started_with(self): + # Above the context there is no ceiling at all: the runaway would run to + # the end of the context instead of stopping where this says. + self.patch_attr(ggml, "llm", FakeServer(context=2048)) + self.conf["local_llm_reasoning"] = "max" + with fake_urlopen(chat_reply("Done.")) as calls: + cleanup.run("uh, done", self.conf, "the rules") + self.assertLess(sent_json(calls[0])["max_tokens"], 2048) + + def test_the_prompt_keeps_its_share_of_a_small_context(self): + self.patch_attr(ggml, "llm", FakeServer(context=2048)) + self.conf["local_llm_reasoning"] = "max" + with fake_urlopen(chat_reply("Done.")) as calls: + cleanup.run("x" * 2000, self.conf, "the rules") + # 2048 less half the characters of prompt and transcript together. + self.assertEqual(sent_json(calls[0])["max_tokens"], + 2048 - (len("the rules") + 2000) // 2) + def test_a_reply_that_was_all_thinking_names_the_setting_that_fixes_it(self): reply = {"choices": [{"message": {"content": "", "reasoning": "hmm"}}]} with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught: From 44db26c45985603f3b57009caee77368bc7f017a Mon Sep 17 00:00:00 2001 From: yusufipk Date: Sat, 5 Sep 2026 12:07:13 +0300 Subject: [PATCH 2/3] Measure a wrapped label against a width it actually has The publisher note is written while the settings window is still being built, when its label is eight pixels wide. Wrapped against that width the sentence came out a hundred and twenty lines tall, and the minimum taken from it did not stay a minimum: QLabel folds the widget's minimum size into its own cached size hints and clears that cache only when the text changes. So the row stood two thousand pixels tall, carrying the model box, the status line and the options under it off the bottom of the window, and picking another publisher was what brought them back. Nothing to measure against yet means nothing to claim yet. The show and the resize come back for it once there is a real width. --- dikte/settings_ui.py | 26 +++++++++++++++++++++----- tests/test_ui.py | 26 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/dikte/settings_ui.py b/dikte/settings_ui.py index 3f32a72..44c99e1 100644 --- a/dikte/settings_ui.py +++ b/dikte/settings_ui.py @@ -181,18 +181,34 @@ class WrappedLabel(QLabel): super().setText(text) self._fit() + def showEvent(self, event): + # Text set while the window was still being built was measured against + # nothing; this is the first moment the width means anything. + super().showEvent(event) + self._fit() + def resizeEvent(self, event): super().resizeEvent(event) self._fit() def _fit(self): + # A label the layout has not placed yet is a handful of pixels wide, + # and wrapping a sentence against that width invents a hundred lines. + # The minimum set from it does not stay a minimum either: QLabel folds + # it into its own cached size hints and clears that cache only when the + # text changes, so the row stands thousands of pixels tall and carries + # the model box and everything under it off the bottom of the window + # until another publisher is picked. Nothing to measure against yet + # means nothing to claim yet, and the show and resize above come back + # for it. + if not self.isVisible() or self.width() <= 0: + return # Measured off the font rather than asked of the label, whose own answer # is floored by the minimum set here a moment ago and so only ever grows. - if self.width() > 0: - wrap = Qt.TextFlag.TextWordWrap | Qt.TextFlag.TextWrapAnywhere - box = QRect(0, 0, self.width(), 0) - self.setMinimumHeight( - self.fontMetrics().boundingRect(box, wrap, self.text()).height()) + wrap = Qt.TextFlag.TextWordWrap | Qt.TextFlag.TextWrapAnywhere + box = QRect(0, 0, self.width(), 0) + self.setMinimumHeight( + self.fontMetrics().boundingRect(box, wrap, self.text()).height()) class WheelGuard(QObject): diff --git a/tests/test_ui.py b/tests/test_ui.py index 1324f33..cc239fb 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -244,6 +244,32 @@ class Settings(DikteTest): label.resize(2000, line) self.assertLessEqual(label.minimumHeight(), line) + def test_a_label_written_before_the_layout_places_it_claims_nothing(self): + # The publisher note is written while the settings window is still + # being built, when the label is a handful of pixels wide. Wrapped + # against that width the sentence became a hundred lines, and the + # minimum taken from it did not stay a minimum: QLabel folds it into + # its own cached size hints and clears that cache only when the text + # changes. The group box stood thousands of pixels tall, with the + # model box and everything under it off the bottom of the window, + # until another publisher was picked. + label = settings_ui.WrappedLabel() + self.addCleanup(label.deleteLater) + line = label.fontMetrics().height() + label.resize(8, line) + label.setText("Google Gemma 4, the small one. The default: nothing " + "else this size follows an instruction as closely, and " + "cleanup is all instruction.") + self.assertEqual(label.minimumHeight(), 0) + # Placed and shown, which is the first width worth measuring against. + # The room the wrapping needs is claimed then, and it is the lines the + # sentence actually takes at this width rather than at the last one. + label.resize(400, line) + label.show() + self.assertGreater(label.minimumHeight(), line) + self.assertLessEqual(label.minimumHeight(), 4 * line) + self.assertLessEqual(label.sizeHint().height(), 4 * line) + def test_saving_without_touching_anything_changes_nothing(self): """Every widget has to load what is stored, or Save writes its default over it. This says so for the whole table at once.""" From 22d2a40341c8cc553100cfe8f513fbe8c68031f1 Mon Sep 17 00:00:00 2001 From: yusufipk Date: Sat, 5 Sep 2026 12:10:58 +0300 Subject: [PATCH 3/3] Count the lines off the font, not off this machine's font The new test pinned the wrapped height at four lines, which is four lines on a Linux runner and four and a half on a Windows one, where the same sentence in the same 400 pixels needs 54 of the box's 48. What the test is actually about is that the height comes from the width the label has now rather than the eight pixels it had while the window was being built, so it measures that width itself and compares against the answer. --- tests/test_ui.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_ui.py b/tests/test_ui.py index cc239fb..f759bf7 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -14,7 +14,7 @@ import unittest from typing import ClassVar from unittest import mock -from PyQt6.QtCore import QPoint, QPointF, Qt +from PyQt6.QtCore import QPoint, QPointF, QRect, Qt from PyQt6.QtGui import QWheelEvent from PyQt6.QtWidgets import QApplication, QMessageBox @@ -263,12 +263,19 @@ class Settings(DikteTest): self.assertEqual(label.minimumHeight(), 0) # Placed and shown, which is the first width worth measuring against. # The room the wrapping needs is claimed then, and it is the lines the - # sentence actually takes at this width rather than at the last one. + # sentence takes at this width rather than at the last one. Counted + # off the font rather than written down here, because how many lines + # 400 pixels hold is a different answer on every machine. label.resize(400, line) label.show() - self.assertGreater(label.minimumHeight(), line) - self.assertLessEqual(label.minimumHeight(), 4 * line) - self.assertLessEqual(label.sizeHint().height(), 4 * line) + wrap = Qt.TextFlag.TextWordWrap | Qt.TextFlag.TextWrapAnywhere + needed = label.fontMetrics().boundingRect( + QRect(0, 0, 400, 0), wrap, label.text()).height() + self.assertGreater(needed, line) # or the sentence never wrapped + self.assertEqual(label.minimumHeight(), needed) + # And the label's own hints are the wrapping at this width too, not + # the hundred lines the eight pixel one asked for. + self.assertLessEqual(label.sizeHint().height(), 3 * needed) def test_saving_without_touching_anything_changes_nothing(self): """Every widget has to load what is stored, or Save writes its default