Merge pull request #78 from yusufipk/claude/yerel-model-thinking-limit-66077e

Give a local model room to think without spending the answer on it

dikte/api.py

@@ -518,22 +518,65 @@ def _thinking(payload, provider, reasoning):
         payload["reasoning"] = {"effort": reasoning, "exclude": True}
 
 
-def local_ceiling(text):
+# Room for the thinking on this machine, one budget per rung of the settings
+# ladder. llama.cpp counts the thinking towards max_tokens along with the answer
+# it precedes, so a ceiling sized for the answer alone leaves a model that
+# thinks nothing to answer with. The rungs double, starting where a small model
+# lands when it barely thinks at all: cleanup is punctuation, and locally every
+# one of these tokens is also a second of somebody standing in front of the
+# screen, so the low rungs are the ones meant to be used.
+THINKING_ROOM = {
+    "minimal": 256, "low": 512, "medium": 1024,
+    "high": 2048, "xhigh": 4096, "max": 8192,
+}
+# An empty setting leaves it to the model, and the templates that can think
+# think by default. Room for a middling amount of it, since there is no way to
+# ask which kind of model this is.
+DEFAULT_THINKING_ROOM = THINKING_ROOM["medium"]
+
+
+def local_ceiling(text, reasoning="", context=0, prompt=""):
     """How much of a reply is worth waiting for from a model on this machine.
 
     Cleanup gives back what it was given, near enough, so a reply several times
     the length of the transcript is a model that has lost the thread rather than
     one doing the job. A small one will happily repeat the transcript until the
     context is full, and every one of those tokens is a second of somebody
-    waiting. A hosted model is left alone: there the same runaway is rare, and a
-    ceiling would cut the minutes short instead.
+    waiting, with only the hour-long local timeout underneath. A hosted model is
+    left alone: there the same runaway is rare, and a ceiling would cut the
+    minutes short instead.
+
+    The answer's share is the transcript's length in characters spent as a
+    budget in tokens, so what it really allows is two to four times the
+    transcript depending on how well the language tokenises. Turkish sits at the
+    tight end of that and still has room to spare for a reply that is meant to
+    come back the same length it went in.
+
+    Thinking is added on top of that share rather than taken out of it. Sharing
+    one budget is what makes turning thinking up quietly cost the answer, and on
+    a short dictation the 512 floor is the whole budget, so the answer is what
+    goes missing first.
+
+    `context` is what the server was started with, and the whole of it is the
+    real limit whatever is asked for here: a ceiling above it is not a ceiling,
+    because the runaway it exists to stop would run to the end of the context
+    instead. So the ceiling is held below what the prompt leaves. Two characters
+    to the token is under any tokeniser's rate for natural language, Turkish
+    included, which makes the reserve an over-estimate rather than a promise of
+    room that is not there.
     """
-    return max(512, len(text))
+    answer = max(512, len(text))
+    if reasoning != "none":
+        answer += THINKING_ROOM.get(reasoning, DEFAULT_THINKING_ROOM)
+    context = int(context or 0)
+    if not context:
+        return answer
+    return max(256, min(answer, context - (len(prompt) + len(text)) // 2))
 
 
 def cleanup(text, api_key, model, system_prompt, reasoning="",
             base_url=OPENROUTER_URL, timeout=180, provider="openrouter",
-            service="OpenRouter", aborter=None):
+            service="OpenRouter", aborter=None, context=0):
     if not api_key and provider != "local-llm":
         raise ApiError(t("{service} API key is empty. Add it in Settings.",
                          service=service))
@@ -546,7 +589,8 @@ def cleanup(text, api_key, model, system_prompt, reasoning="",
         ],
     }
     if provider == "local-llm":
-        payload["max_tokens"] = local_ceiling(text)
+        payload["max_tokens"] = local_ceiling(text, reasoning, context,
+                                              system_prompt)
     _thinking(payload, provider, reasoning)
     try:
         data = _request(
@@ -570,6 +614,13 @@ def cleanup(text, api_key, model, system_prompt, reasoning="",
             raise ApiError(t("The cleanup model spent its whole reply on "
                              "thinking. Set Thinking to \u201cOff\u201d."))
         raise ApiError(t("The cleanup model returned an empty reply."))
+    if choices[0].get("finish_reason") == "length":
+        # Cut off at somebody's ceiling: ours locally, the provider's otherwise.
+        # What came back is a sentence that stops mid-word, and cleanup is meant
+        # to hand back the whole dictation, so the half is refused rather than
+        # returned. The callers keep the transcript they started with, which is
+        # the better of the two.
+        raise ApiError(t("The cleanup model was cut off before it finished."))
     return content
 
 
@@ -605,6 +656,11 @@ def chat(messages, api_key, model, system_prompt, reasoning="",
     content = ((choices[0].get("message") or {}).get("content") or "").strip()
     if not content:
         raise ApiError(t("The model returned an empty reply."))
+    if choices[0].get("finish_reason") == "length":
+        # An answer that stops mid-sentence reads like a whole one once it has
+        # been pasted, so it is refused here for the same reason cleanup refuses
+        # a half transcript.
+        raise ApiError(t("The model was cut off before it finished."))
     return content
 
 

dikte/cleanup.py

@@ -121,6 +121,9 @@ def _local(text, conf, system_prompt, timeout, aborter=None):
             base_url=api.serving(ggml.llm),
             timeout=max(timeout, api.LOCAL_TIMEOUT),
             provider="local-llm", service=service, aborter=aborter,
+            # The ceiling is only a ceiling while it sits under what the server
+            # was started with; above that the context is what stops the reply.
+            context=ggml.llm.settings()["context"],
         )
     except api.ApiError as exc:
         # A server that died mid-request would otherwise report only that the

dikte/i18n.py

@@ -959,6 +959,10 @@ TR = {
     "“Off”.":
         "Temizleme modeli bütün yanıtını düşünmeye harcadı. Düşünme'yi "
         "“Kapalı” yap.",
+    "The cleanup model was cut off before it finished.":
+        "Temizleme modeli bitiremeden kesildi.",
+    "The model was cut off before it finished.":
+        "Model bitiremeden kesildi.",
 
     # --- this pass's new messages ---------------------------------------
     "Audio recorder stopped before receiving sound":

tests/test_api.py

@@ -463,6 +463,29 @@ class Cleanup(DikteTest):
         with fake_urlopen(chat_reply("   ")), self.assertRaises(api.ApiError):
             api.cleanup("hello", "k", "m", "p")
 
+    def test_a_reply_cut_off_at_a_ceiling_is_refused_rather_than_pasted(self):
+        # Half a sentence looks like a cleaned-up transcript and is not one. The
+        # caller keeps what it was given, which is the whole dictation.
+        reply = {"choices": [{"message": {"content": "Hello, and then the"},
+                              "finish_reason": "length"}]}
+        with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught:
+            api.cleanup("hello", "k", "m", "p")
+        self.assertIn("cut off", str(caught.exception))
+
+    def test_a_reply_that_stopped_on_its_own_is_kept(self):
+        reply = {"choices": [{"message": {"content": "Hello."},
+                              "finish_reason": "stop"}]}
+        with fake_urlopen(reply):
+            self.assertEqual(api.cleanup("hello", "k", "m", "p"), "Hello.")
+
+    def test_all_thinking_is_named_before_the_ceiling_it_was_cut_at(self):
+        """Both are true at once, and only one of them says what to change."""
+        reply = {"choices": [{"message": {"content": "", "reasoning": "hmm"},
+                              "finish_reason": "length"}]}
+        with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught:
+            api.cleanup("hello", "k", "m", "p")
+        self.assertIn("Thinking", str(caught.exception))
+
     def test_a_rate_limit_is_explained(self):
         with fake_urlopen(http_error(429)), \
                 self.assertRaises(api.ApiError) as caught:
@@ -471,6 +494,14 @@ class Cleanup(DikteTest):
 
 
 class Chat(DikteTest):
+    def test_an_answer_cut_off_at_a_ceiling_is_refused_rather_than_pasted(self):
+        # Half an answer reads like a whole one once it is on the screen.
+        reply = {"choices": [{"message": {"content": "Booked it for the"},
+                              "finish_reason": "length"}]}
+        with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught:
+            api.chat([{"role": "user", "content": "book it"}], "k", "m", "p")
+        self.assertIn("cut off", str(caught.exception))
+
     def test_the_history_is_sent_after_the_system_prompt(self):
         history = [{"role": "user", "content": "book it"},
                    {"role": "assistant", "content": "done"}]
@@ -616,11 +647,13 @@ if __name__ == "__main__":
 class FakeServer:
     """A ggml.Server as far as api.py is concerned."""
 
-    def __init__(self, url="http://127.0.0.1:9999/v1", fails="", log=""):
+    def __init__(self, url="http://127.0.0.1:9999/v1", fails="", log="",
+                 context=8192):
         self.url = url
         self.fails = fails
         self.log = log
         self.starts = 0
+        self.context = context
 
     def serve(self):
         self.starts += 1
@@ -631,6 +664,9 @@ class FakeServer:
     def error(self):
         return self.log
 
+    def settings(self):
+        return {"context": self.context}
+
 
 LOCAL = api.Target("local", "Local whisper", "", "", "ggml-base.bin")
 

tests/test_cleanup.py

@@ -411,6 +411,52 @@ class Here(DikteTest):
             cleanup.run("uh, done", self.conf, "the rules")
         self.assertEqual(sent_json(calls[0])["max_tokens"], 512)
 
+    def test_thinking_is_given_room_of_its_own_rather_than_the_answer_s(self):
+        # llama.cpp counts the thinking towards the same ceiling, so a rung that
+        # took its budget out of the answer would leave a short dictation with
+        # nothing to reply with. On a context roomy enough that the clamp the
+        # top rung would otherwise meet is not what is being measured.
+        self.patch_attr(ggml, "llm", FakeServer(context=32768))
+        for rung, room in api.THINKING_ROOM.items():
+            with self.subTest(rung=rung):
+                self.conf["local_llm_reasoning"] = rung
+                with fake_urlopen(chat_reply("Done.")) as calls:
+                    cleanup.run("uh, done", self.conf, "the rules")
+                self.assertEqual(sent_json(calls[0])["max_tokens"], 512 + room)
+
+    def test_each_rung_of_the_ladder_thinks_longer_than_the_one_below(self):
+        rungs = [api.THINKING_ROOM[name] for name in
+                 ("minimal", "low", "medium", "high", "xhigh", "max")]
+        self.assertEqual(rungs, sorted(rungs))
+        self.assertEqual(len(set(rungs)), len(rungs))
+
+    def test_the_models_own_default_is_given_room_to_think_in_too(self):
+        # Nothing is sent, so a template that thinks will think, and the ceiling
+        # has to survive that as well.
+        self.conf["local_llm_reasoning"] = ""
+        with fake_urlopen(chat_reply("Done.")) as calls:
+            cleanup.run("uh, done", self.conf, "the rules")
+        self.assertEqual(sent_json(calls[0])["max_tokens"],
+                         512 + api.DEFAULT_THINKING_ROOM)
+
+    def test_the_ceiling_stays_under_the_context_the_server_was_started_with(self):
+        # Above the context there is no ceiling at all: the runaway would run to
+        # the end of the context instead of stopping where this says.
+        self.patch_attr(ggml, "llm", FakeServer(context=2048))
+        self.conf["local_llm_reasoning"] = "max"
+        with fake_urlopen(chat_reply("Done.")) as calls:
+            cleanup.run("uh, done", self.conf, "the rules")
+        self.assertLess(sent_json(calls[0])["max_tokens"], 2048)
+
+    def test_the_prompt_keeps_its_share_of_a_small_context(self):
+        self.patch_attr(ggml, "llm", FakeServer(context=2048))
+        self.conf["local_llm_reasoning"] = "max"
+        with fake_urlopen(chat_reply("Done.")) as calls:
+            cleanup.run("x" * 2000, self.conf, "the rules")
+        # 2048 less half the characters of prompt and transcript together.
+        self.assertEqual(sent_json(calls[0])["max_tokens"],
+                         2048 - (len("the rules") + 2000) // 2)
+
     def test_a_reply_that_was_all_thinking_names_the_setting_that_fixes_it(self):
         reply = {"choices": [{"message": {"content": "", "reasoning": "hmm"}}]}
         with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught: