diff --git a/dikte/api.py b/dikte/api.py index 13c4381..58cc30b 100644 --- a/dikte/api.py +++ b/dikte/api.py @@ -518,22 +518,65 @@ def _thinking(payload, provider, reasoning): payload["reasoning"] = {"effort": reasoning, "exclude": True} -def local_ceiling(text): +# Room for the thinking on this machine, one budget per rung of the settings +# ladder. llama.cpp counts the thinking towards max_tokens along with the answer +# it precedes, so a ceiling sized for the answer alone leaves a model that +# thinks nothing to answer with. The rungs double, starting where a small model +# lands when it barely thinks at all: cleanup is punctuation, and locally every +# one of these tokens is also a second of somebody standing in front of the +# screen, so the low rungs are the ones meant to be used. +THINKING_ROOM = { + "minimal": 256, "low": 512, "medium": 1024, + "high": 2048, "xhigh": 4096, "max": 8192, +} +# An empty setting leaves it to the model, and the templates that can think +# think by default. Room for a middling amount of it, since there is no way to +# ask which kind of model this is. +DEFAULT_THINKING_ROOM = THINKING_ROOM["medium"] + + +def local_ceiling(text, reasoning="", context=0, prompt=""): """How much of a reply is worth waiting for from a model on this machine. Cleanup gives back what it was given, near enough, so a reply several times the length of the transcript is a model that has lost the thread rather than one doing the job. A small one will happily repeat the transcript until the context is full, and every one of those tokens is a second of somebody - waiting. A hosted model is left alone: there the same runaway is rare, and a - ceiling would cut the minutes short instead. + waiting, with only the hour-long local timeout underneath. A hosted model is + left alone: there the same runaway is rare, and a ceiling would cut the + minutes short instead. + + The answer's share is the transcript's length in characters spent as a + budget in tokens, so what it really allows is two to four times the + transcript depending on how well the language tokenises. Turkish sits at the + tight end of that and still has room to spare for a reply that is meant to + come back the same length it went in. + + Thinking is added on top of that share rather than taken out of it. Sharing + one budget is what makes turning thinking up quietly cost the answer, and on + a short dictation the 512 floor is the whole budget, so the answer is what + goes missing first. + + `context` is what the server was started with, and the whole of it is the + real limit whatever is asked for here: a ceiling above it is not a ceiling, + because the runaway it exists to stop would run to the end of the context + instead. So the ceiling is held below what the prompt leaves. Two characters + to the token is under any tokeniser's rate for natural language, Turkish + included, which makes the reserve an over-estimate rather than a promise of + room that is not there. """ - return max(512, len(text)) + answer = max(512, len(text)) + if reasoning != "none": + answer += THINKING_ROOM.get(reasoning, DEFAULT_THINKING_ROOM) + context = int(context or 0) + if not context: + return answer + return max(256, min(answer, context - (len(prompt) + len(text)) // 2)) def cleanup(text, api_key, model, system_prompt, reasoning="", base_url=OPENROUTER_URL, timeout=180, provider="openrouter", - service="OpenRouter", aborter=None): + service="OpenRouter", aborter=None, context=0): if not api_key and provider != "local-llm": raise ApiError(t("{service} API key is empty. Add it in Settings.", service=service)) @@ -546,7 +589,8 @@ def cleanup(text, api_key, model, system_prompt, reasoning="", ], } if provider == "local-llm": - payload["max_tokens"] = local_ceiling(text) + payload["max_tokens"] = local_ceiling(text, reasoning, context, + system_prompt) _thinking(payload, provider, reasoning) try: data = _request( @@ -570,6 +614,13 @@ def cleanup(text, api_key, model, system_prompt, reasoning="", raise ApiError(t("The cleanup model spent its whole reply on " "thinking. Set Thinking to \u201cOff\u201d.")) raise ApiError(t("The cleanup model returned an empty reply.")) + if choices[0].get("finish_reason") == "length": + # Cut off at somebody's ceiling: ours locally, the provider's otherwise. + # What came back is a sentence that stops mid-word, and cleanup is meant + # to hand back the whole dictation, so the half is refused rather than + # returned. The callers keep the transcript they started with, which is + # the better of the two. + raise ApiError(t("The cleanup model was cut off before it finished.")) return content @@ -605,6 +656,11 @@ def chat(messages, api_key, model, system_prompt, reasoning="", content = ((choices[0].get("message") or {}).get("content") or "").strip() if not content: raise ApiError(t("The model returned an empty reply.")) + if choices[0].get("finish_reason") == "length": + # An answer that stops mid-sentence reads like a whole one once it has + # been pasted, so it is refused here for the same reason cleanup refuses + # a half transcript. + raise ApiError(t("The model was cut off before it finished.")) return content diff --git a/dikte/cleanup.py b/dikte/cleanup.py index a1bced3..36a3ee9 100644 --- a/dikte/cleanup.py +++ b/dikte/cleanup.py @@ -121,6 +121,9 @@ def _local(text, conf, system_prompt, timeout, aborter=None): base_url=api.serving(ggml.llm), timeout=max(timeout, api.LOCAL_TIMEOUT), provider="local-llm", service=service, aborter=aborter, + # The ceiling is only a ceiling while it sits under what the server + # was started with; above that the context is what stops the reply. + context=ggml.llm.settings()["context"], ) except api.ApiError as exc: # A server that died mid-request would otherwise report only that the diff --git a/dikte/i18n.py b/dikte/i18n.py index 50c497d..6375e55 100644 --- a/dikte/i18n.py +++ b/dikte/i18n.py @@ -959,6 +959,10 @@ TR = { "“Off”.": "Temizleme modeli bütün yanıtını düşünmeye harcadı. Düşünme'yi " "“Kapalı” yap.", + "The cleanup model was cut off before it finished.": + "Temizleme modeli bitiremeden kesildi.", + "The model was cut off before it finished.": + "Model bitiremeden kesildi.", # --- this pass's new messages --------------------------------------- "Audio recorder stopped before receiving sound": diff --git a/tests/test_api.py b/tests/test_api.py index 50b4ef7..4b35774 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -463,6 +463,29 @@ class Cleanup(DikteTest): with fake_urlopen(chat_reply(" ")), self.assertRaises(api.ApiError): api.cleanup("hello", "k", "m", "p") + def test_a_reply_cut_off_at_a_ceiling_is_refused_rather_than_pasted(self): + # Half a sentence looks like a cleaned-up transcript and is not one. The + # caller keeps what it was given, which is the whole dictation. + reply = {"choices": [{"message": {"content": "Hello, and then the"}, + "finish_reason": "length"}]} + with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught: + api.cleanup("hello", "k", "m", "p") + self.assertIn("cut off", str(caught.exception)) + + def test_a_reply_that_stopped_on_its_own_is_kept(self): + reply = {"choices": [{"message": {"content": "Hello."}, + "finish_reason": "stop"}]} + with fake_urlopen(reply): + self.assertEqual(api.cleanup("hello", "k", "m", "p"), "Hello.") + + def test_all_thinking_is_named_before_the_ceiling_it_was_cut_at(self): + """Both are true at once, and only one of them says what to change.""" + reply = {"choices": [{"message": {"content": "", "reasoning": "hmm"}, + "finish_reason": "length"}]} + with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught: + api.cleanup("hello", "k", "m", "p") + self.assertIn("Thinking", str(caught.exception)) + def test_a_rate_limit_is_explained(self): with fake_urlopen(http_error(429)), \ self.assertRaises(api.ApiError) as caught: @@ -471,6 +494,14 @@ class Cleanup(DikteTest): class Chat(DikteTest): + def test_an_answer_cut_off_at_a_ceiling_is_refused_rather_than_pasted(self): + # Half an answer reads like a whole one once it is on the screen. + reply = {"choices": [{"message": {"content": "Booked it for the"}, + "finish_reason": "length"}]} + with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught: + api.chat([{"role": "user", "content": "book it"}], "k", "m", "p") + self.assertIn("cut off", str(caught.exception)) + def test_the_history_is_sent_after_the_system_prompt(self): history = [{"role": "user", "content": "book it"}, {"role": "assistant", "content": "done"}] @@ -616,11 +647,13 @@ if __name__ == "__main__": class FakeServer: """A ggml.Server as far as api.py is concerned.""" - def __init__(self, url="http://127.0.0.1:9999/v1", fails="", log=""): + def __init__(self, url="http://127.0.0.1:9999/v1", fails="", log="", + context=8192): self.url = url self.fails = fails self.log = log self.starts = 0 + self.context = context def serve(self): self.starts += 1 @@ -631,6 +664,9 @@ class FakeServer: def error(self): return self.log + def settings(self): + return {"context": self.context} + LOCAL = api.Target("local", "Local whisper", "", "", "ggml-base.bin") diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index 9de3dc3..245053f 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -411,6 +411,52 @@ class Here(DikteTest): cleanup.run("uh, done", self.conf, "the rules") self.assertEqual(sent_json(calls[0])["max_tokens"], 512) + def test_thinking_is_given_room_of_its_own_rather_than_the_answer_s(self): + # llama.cpp counts the thinking towards the same ceiling, so a rung that + # took its budget out of the answer would leave a short dictation with + # nothing to reply with. On a context roomy enough that the clamp the + # top rung would otherwise meet is not what is being measured. + self.patch_attr(ggml, "llm", FakeServer(context=32768)) + for rung, room in api.THINKING_ROOM.items(): + with self.subTest(rung=rung): + self.conf["local_llm_reasoning"] = rung + with fake_urlopen(chat_reply("Done.")) as calls: + cleanup.run("uh, done", self.conf, "the rules") + self.assertEqual(sent_json(calls[0])["max_tokens"], 512 + room) + + def test_each_rung_of_the_ladder_thinks_longer_than_the_one_below(self): + rungs = [api.THINKING_ROOM[name] for name in + ("minimal", "low", "medium", "high", "xhigh", "max")] + self.assertEqual(rungs, sorted(rungs)) + self.assertEqual(len(set(rungs)), len(rungs)) + + def test_the_models_own_default_is_given_room_to_think_in_too(self): + # Nothing is sent, so a template that thinks will think, and the ceiling + # has to survive that as well. + self.conf["local_llm_reasoning"] = "" + with fake_urlopen(chat_reply("Done.")) as calls: + cleanup.run("uh, done", self.conf, "the rules") + self.assertEqual(sent_json(calls[0])["max_tokens"], + 512 + api.DEFAULT_THINKING_ROOM) + + def test_the_ceiling_stays_under_the_context_the_server_was_started_with(self): + # Above the context there is no ceiling at all: the runaway would run to + # the end of the context instead of stopping where this says. + self.patch_attr(ggml, "llm", FakeServer(context=2048)) + self.conf["local_llm_reasoning"] = "max" + with fake_urlopen(chat_reply("Done.")) as calls: + cleanup.run("uh, done", self.conf, "the rules") + self.assertLess(sent_json(calls[0])["max_tokens"], 2048) + + def test_the_prompt_keeps_its_share_of_a_small_context(self): + self.patch_attr(ggml, "llm", FakeServer(context=2048)) + self.conf["local_llm_reasoning"] = "max" + with fake_urlopen(chat_reply("Done.")) as calls: + cleanup.run("x" * 2000, self.conf, "the rules") + # 2048 less half the characters of prompt and transcript together. + self.assertEqual(sent_json(calls[0])["max_tokens"], + 2048 - (len("the rules") + 2000) // 2) + def test_a_reply_that_was_all_thinking_names_the_setting_that_fixes_it(self): reply = {"choices": [{"message": {"content": "", "reasoning": "hmm"}}]} with fake_urlopen(reply), self.assertRaises(api.ApiError) as caught: