Merge pull request #63 from sudoeren/auto-language-detect

Detect the spoken language instead of fixing one

README.md

@@ -220,6 +220,11 @@ running.
   written for subtitles, so the lines keep their place and nothing is shortened.
 - **History** of every dictation under Settings → History, with a size limit and
   right-click to delete.
+- **The speech language is detected, not picked.** Auto is the default: whisper
+  on this machine says what it heard, the hosted providers transcribe in
+  whatever language comes in without being told, and the detected language
+  lands in the history and decides which cleanup prompt (Turkish or the
+  language-agnostic one) a run gets. A fixed language still overrides it.
 - **Turkish and English interface**, following the system locale by default.
 
 ## The global shortcuts, and the logout KDE needs

README.tr.md

@@ -214,6 +214,11 @@ olmasını ister.
   yerinde kalır, hiçbir şey kısaltılmaz.
 - **Geçmiş** Ayarlar → Geçmiş sekmesinde; boyut sınırı var, sağ tıklayıp
   silebilirsin.
+- **Konuşma dili seçilmez, algılanır.** Varsayılan otomatiktir: bu makinedeki
+  whisper ne duyduğunu söyler, bulut sağlayıcılar söylenmeden de hangi dilde
+  konuşuluyorsa o dilde yazar; algılanan dil geçmişe düşer ve bir kaydın hangi
+  temizleme promptunu alacağını belirler (Türkçe mi, dile duyarsız olanı mı).
+  Sabit bir dil yine de bunun önüne geçer.
 - **Türkçe ve İngilizce arayüz**, varsayılan olarak sistem dilini izler.
 
 ## Global kısayollar ve KDE'nin istediği oturum kapatma

dikte/api.py

@@ -367,7 +367,8 @@ def local_failure(service, server, exc):
 
 
 def _transcribe_request(target, audio_path, language, prompt, response_format,
-                        granularity=None, timeout=300, aborter=None):
+                        granularity=None, timeout=300, aborter=None,
+                        detect_language=False):
     if target.provider == "local":
         # The timeouts here are sized for a hosted API, where a slow answer is a
         # bill running. Locally the only thing being spent is time.
@@ -379,6 +380,12 @@ def _transcribe_request(target, audio_path, language, prompt, response_format,
     fields = [("model", target.model), ("response_format", response_format)]
     if language and language != "auto":
         fields.append(("language", language))
+    if detect_language:
+        # whisper.cpp was started with -nlp, which keeps the language
+        # probability sweep off every request. Detection is only worth that
+        # sweep for the run that asked for it, so it is switched back on here,
+        # per request, and reported in the verbose_json answer.
+        fields.append(("no_language_probabilities", "false"))
     # OpenRouter takes the hint field and throws it away, so spare it the bytes.
     # The same words still reach the cleanup model as a glossary. whisper.cpp
     # takes it as the initial prompt, the way OpenAI does.
@@ -546,6 +553,44 @@ def transcribe(target, audio_path, language="", prompt="", timeout=300, aborter=
     return text
 
 
+# whisper.cpp reports what it heard as a lowercase full name ("turkish",
+# "english", "german"…); the settings and the cleanup prompt speak in two-letter
+# codes. Only the handful Dikte offers as a fixed choice get a code; anything
+# else is left as the empty string, which the caller reads as "unknown" rather
+# than guessing at a language it has no label for.
+_DETECTED_TO_CODE = {
+    "english": "en", "turkish": "tr", "german": "de",
+    "french": "fr", "spanish": "es", "arabic": "ar",
+}
+
+
+def transcribe_detected(target, audio_path, language="", prompt="", timeout=300,
+                        aborter=None):
+    """(text, code) with the language the model heard.
+
+    The spoken language is only knowable when the transcription model reports
+    it, and only whisper.cpp does: the hosted endpoints accept "auto" but never
+    say what they heard. So detection is asked for exactly where it can be
+    answered, the local server in auto mode, and every other run transcribes
+    as before and hands back an empty code.
+    """
+    if target.provider == "local" and language == "auto":
+        data = _transcribe_request(
+            target, audio_path, language, prompt, "verbose_json",
+            detect_language=True, timeout=timeout, aborter=aborter,
+        )
+        text = _local_text(data.get("text") or "").strip()
+        if not text:
+            raise ApiError(t("Transcript came back empty."))
+        detected = data.get("detected_language")
+        code = _DETECTED_TO_CODE.get(
+            detected.strip().lower(), "") if isinstance(detected, str) else ""
+        return text, code
+    text = transcribe(target, audio_path, language=language, prompt=prompt,
+                      timeout=timeout, aborter=aborter)
+    return text, ""
+
+
 def transcribe_segments(target, audio_path, language="", prompt="", timeout=300,
                         aborter=None):
     """[(start_seconds, end_seconds, text)] using whisper-1's verbose response."""

dikte/app.py

@@ -1084,7 +1084,7 @@ class Dikte:
         if not self._transcripts_pending:
             self._settle(DICTATION, payload)
 
-    def _on_finished(self, _raw, text, warning):
+    def _on_finished(self, _raw, text, warning, speech_language):
         if warning:
             # The text was still pasted, but cleanup did not run. Say so loudly:
             # a rejected key otherwise looks exactly like working dictation.
@@ -1105,9 +1105,10 @@ class Dikte:
                 t("{action}: {preview}", action=action, preview=_preview(text))
             )
         self._transcript_settled({"ok": True, "text": text, "raw": _raw,
-                                  "warning": warning})
+                                  "warning": warning,
+                                  "speech_language": speech_language})
 
-    def _on_ask_finished(self, _raw, text, warning):
+    def _on_ask_finished(self, _raw, text, warning, speech_language):
         agent = assistant.display_name(self.conf)
         if warning:
             # A tool the agent was not allowed to touch otherwise looks exactly
@@ -1128,7 +1129,8 @@ class Dikte:
             )
         self._set_ask_state(IDLE)
         self._settle(ASK, {"ok": True, "answer": text, "question": _raw,
-                           "warning": warning, "agent": agent})
+                           "warning": warning, "agent": agent,
+                           "speech_language": speech_language})
 
     def _on_ask_cancelled(self):
         self.ask_overlay.show_done(t("Stopped."), 2000)

dikte/config.py

@@ -400,7 +400,9 @@ DEFAULTS = {
     # What a timestamped run (subtitles) asks OpenRouter for: not every model
     # there returns segment times. Empty -> openai/whisper-1.
     "openrouter_file_model": "",
-    "language": "tr",
+    # A stored language overrides this default. Hosted providers receive no
+    # language hint in auto mode; local whisper also reports the detected code.
+    "language": "auto",
     "transcribe_prompt": "",
 
     # --- whisper.cpp, on this machine ---------------------------------------
@@ -733,13 +735,23 @@ class Config:
         return self["cleanup_provider"] == "local"
 
     def cleanup_prompt(self, with_timestamps=False, with_speakers=False,
-                       subtitles=False):
-        turkish = i18n.language() == "tr"
+                       subtitles=False, speech=""):
+        """`speech` is the two-letter code of the language that was heard, when
+        the transcription model reported one. The default prompts and the
+        glossary rule only exist in Turkish and English, so a detected Turkish
+        recording gets the Turkish prompt and any other detected language, or
+        none at all, the English one, which is written not to care what
+        language the transcript is in. Nothing else calls this with it, so the
+        interface language keeps deciding everywhere the speech was not asked
+        about."""
+        turkish = (speech == "tr") if speech else i18n.language() == "tr"
         if subtitles:
             prompt = (self["file_cleanup_prompt"].strip()
-                      or default_file_cleanup_prompt())
+                      or (FILE_CLEANUP_PROMPT_TR if turkish
+                          else FILE_CLEANUP_PROMPT_EN))
         else:
-            prompt = self["cleanup_prompt"].strip() or default_cleanup_prompt()
+            prompt = (self["cleanup_prompt"].strip()
+                      or (CLEANUP_PROMPT_TR if turkish else CLEANUP_PROMPT_EN))
         glossary = self["transcribe_prompt"].strip()
         if with_speakers:
             glossary = "\n".join(x for x in (glossary, self.participants()) if x)

dikte/ggml.py

@@ -1547,12 +1547,13 @@ def _whisper_args(settings):
         binary, "-m", str(model),
         "--inference-path", INFERENCE_PATH,
         # Whatever language the request does not name. api.py leaves the field
-        # out when the language is "auto", and the server's own default is
-        # English rather than detection.
+        # out when the language is "auto", and the server's own language is
+        # set here: "auto" makes whisper.cpp detect what it hears.
         "-l", "auto",
         # Stock phrases invented for near-silence come from non-speech tokens,
         # and verbose_json otherwise pays for a language probability sweep
-        # nothing here reads.
+        # nobody asked for. A request that wants the detected language switches
+        # that back on per request.
         "-sns", "-nlp",
     ]
     if int(settings["threads"]) > 0:

dikte/worker.py

@@ -37,7 +37,7 @@ _paste_lock = threading.Lock()
 
 class Pipeline(QObject):
     stage = pyqtSignal(str)          # human-readable progress line
-    finished = pyqtSignal(str, str, str)  # raw transcript, final text, warning
+    finished = pyqtSignal(str, str, str, str)  # raw, final text, warning, language
     failed = pyqtSignal(str)
     cancelled = pyqtSignal()
 
@@ -120,12 +120,24 @@ class Pipeline(QObject):
         try:
             self.stage.emit(t("Transcribing…"))
             target = conf.transcribe_target()
+            # The spoken language is only knowable after the fact, and only the
+            # local server says what it heard: auto mode asks it there, and
+            # every other run (a fixed language, or a hosted provider that
+            # detects but stays silent) transcribes as before.
+            auto = conf["language"] == "auto"
+            if auto:
+                raw, detected = api.transcribe_detected(
+                    target, wav_path, language=conf["language"],
+                    prompt=conf["transcribe_prompt"],
+                )
+            else:
                 raw = api.transcribe(
                     target,
                     wav_path,
                     language=conf["language"],
                     prompt=conf["transcribe_prompt"],
                 )
+                detected = ""
 
             if conf["filter_hallucinations"] and vad.looks_like_hallucination(raw, duration):
                 self._discard(wav_path)
@@ -134,6 +146,10 @@ class Pipeline(QObject):
 
             text = raw
             warning = ""
+            # The language the run actually spoke, reported to the window, the
+            # clipboard path and the history alike: the detected code, or the
+            # configured one when nothing was detected to replace it.
+            speech_language = detected or conf["language"]
             # Remembered rather than re-derived at the history write below: the
             # ask path runs cleanup under a different setting, and the record
             # should say what happened, not what one of the two gates implies.
@@ -145,7 +161,7 @@ class Pipeline(QObject):
                 self.stage.emit(t("Cleaning up…"))
                 cleaned = True
                 try:
-                    text = cleanup.run(raw, conf, conf.cleanup_prompt())
+                    text = cleanup.run(raw, conf, conf.cleanup_prompt(speech=detected))
                 except api.ApiError as exc:
                     # Keep the transcript, but never let the failure pass unseen:
                     # a rejected key would otherwise look like working dictation.
@@ -183,6 +199,7 @@ class Pipeline(QObject):
                 "question": question,
                 "assistant": assistant.provider(conf) if ask else "",
                 "assistant_model": assistant.model(conf) if ask else "",
+                "speech_language": speech_language,
                 "raw": raw,
                 "text": text,
             }
@@ -221,7 +238,7 @@ class Pipeline(QObject):
                     time.sleep(0.35)
                     paste.copy_bytes(previous)
 
-            self.finished.emit(raw, text, warning)
+            self.finished.emit(raw, text, warning, speech_language)
 
         except assistant.Cancelled:
             self.cancelled.emit()

tests/test_api.py

@@ -827,6 +827,40 @@ class TranscribeHere(DikteTest):
             api.transcribe_segments(LOCAL, self.wav)
         self.assertEqual(multipart_fields(calls[0])["model"], "ggml-base.bin")
 
+    # ---- the detected language --------------------------------------------
+
+    def test_auto_mode_asks_whisper_for_the_detected_language(self):
+        # The -nlp the server was started with is switched back on for this one
+        # request, so whisper's verbose_json reports what it heard.
+        reply = {"text": " Merhaba dünya. ", "detected_language": "turkish"}
+        with fake_urlopen(reply) as calls:
+            text, code = api.transcribe_detected(LOCAL, self.wav, language="auto")
+        fields = multipart_fields(calls[0])
+        self.assertEqual(fields["response_format"], "verbose_json")
+        self.assertEqual(fields["no_language_probabilities"], "false")
+        self.assertNotIn("language", fields)
+        self.assertEqual(text, "Merhaba dünya.")
+        self.assertEqual(code, "tr")
+
+    def test_a_fixed_language_reports_no_detection(self):
+        with fake_urlopen({"text": "hello"}) as calls:
+            text, code = api.transcribe_detected(LOCAL, self.wav, language="tr")
+        self.assertNotIn("no_language_probabilities", multipart_fields(calls[0]))
+        self.assertEqual(text, "hello")
+        self.assertEqual(code, "")
+
+    def test_a_detected_language_without_a_code_stays_unknown(self):
+        with fake_urlopen({"text": "hello", "detected_language": "somali"}):
+            _text, code = api.transcribe_detected(LOCAL, self.wav, language="auto")
+        self.assertEqual(code, "")
+
+    def test_a_hosted_auto_run_transcribes_without_detection(self):
+        with fake_urlopen({"text": "hi"}) as calls:
+            text, code = api.transcribe_detected(OPENAI, self.wav, language="auto")
+        self.assertNotIn("no_language_probabilities", multipart_fields(calls[0]))
+        self.assertEqual(text, "hi")
+        self.assertEqual(code, "")
+
 
 class Stopping(unittest.TestCase):
     """The Stop button, from the far end: a request already blocked on a reply.

tests/test_cli.py

@@ -746,6 +746,13 @@ class Replies(DikteTest):
         self.assertEqual(code, 0)
         self.assertEqual(out.strip(), "Book it for Thursday.")
 
+    def test_the_json_answer_carries_the_detected_language(self):
+        code, out, _ = self.run_verb(
+            ["--json", "record"],
+            {"ok": True, "text": "Selam", "speech_language": "tr"})
+        self.assertEqual(code, 0)
+        self.assertEqual(json.loads(out)["speech_language"], "tr")
+
     def test_a_dictation_that_failed(self):
         code, out, err = self.run_verb(["stop", "--wait"],
                                        {"ok": False, "error": "No speech detected"})

tests/test_config.py

@@ -274,6 +274,18 @@ class CleanupPrompt(DikteTest):
     def test_no_glossary_means_no_rule_about_one(self):
         self.assertEqual(cfg.Config().cleanup_prompt(), cfg.CLEANUP_PROMPT_EN)
 
+    def test_a_detected_turkish_recording_gets_the_turkish_prompt(self):
+        """Auto mode learns what was heard, and that decides the prompt rather
+        than the interface language."""
+        self.write_config({"ui_language": "en", "transcribe_prompt": "Paraşüt"})
+        conf = cfg.Config()
+        prompt = conf.cleanup_prompt(speech="tr")
+        self.assertEqual(prompt, cfg.CLEANUP_PROMPT_TR
+                         + cfg.GLOSSARY_RULE_TR.format(glossary="Paraşüt"))
+        self.assertIn("KONUŞMACININ KULLANDIĞI İSİM VE TERİMLER", prompt)
+        self.assertIn("NAMES AND TERMS THE SPEAKER USES",
+                      conf.cleanup_prompt(speech="de"))
+
     def test_subtitles_use_their_own_prompt(self):
         conf = cfg.Config()
         self.assertNotEqual(conf.cleanup_prompt(subtitles=True), conf.cleanup_prompt())

tests/test_worker.py

@@ -35,7 +35,7 @@ class Chain(DikteTest):
                   cleaned="Book it for Thursday.",
                   cleanup_error=None, answer=("Booked.", ""), rms=None,
                   clipboard=b"what was there before", paste_error=None,
-                  focus=None):
+                  detected="en", focus=None):
         pipeline = worker.Pipeline(self.conf)
         done, failures, stages, cancels = [], [], [], []
         pipeline.finished.connect(lambda *args: done.append(args))
@@ -45,14 +45,19 @@ class Chain(DikteTest):
 
         cleanup = (mock.Mock(side_effect=cleanup_error) if cleanup_error
                    else mock.Mock(return_value=cleaned))
+        # Auto mode takes the detection path; a fixed language the plain one.
+        # Both are mocked so the chain runs either way without a server.
+        behavior = {"side_effect": transcribe_error} if transcribe_error \
+            else {"return_value": transcript}
+        detect_behavior = {"side_effect": transcribe_error} if transcribe_error \
+            else {"return_value": (transcript, detected)}
         calls = {}
         # The chain reports its own failures on stderr, which a test run has no
         # use for.
         with contextlib.redirect_stderr(io.StringIO()), \
-                mock.patch.object(
-                    api, "transcribe",
-                    **({"side_effect": transcribe_error} if transcribe_error
-                       else {"return_value": transcript})) as tr, \
+                mock.patch.object(api, "transcribe", **behavior) as tr, \
+                mock.patch.object(api, "transcribe_detected",
+                                  **detect_behavior) as tdet, \
                 mock.patch.object(api, "cleanup", cleanup), \
                 mock.patch.object(assistant, "ask", return_value=answer) as ask_call, \
                 mock.patch.object(paste, "copy") as copy, \
@@ -62,7 +67,8 @@ class Chain(DikteTest):
                                   return_value=clipboard) as read_clipboard, \
                 mock.patch.object(worker.time, "sleep", lambda seconds: None):
             press.side_effect = paste_error
-            calls = {"transcribe": tr, "cleanup": cleanup, "ask": ask_call,
+            calls = {"transcribe": tr, "transcribe_detected": tdet,
+                     "cleanup": cleanup, "ask": ask_call,
                      "copy": copy, "copy_bytes": copy_bytes, "press": press,
                      "read_clipboard": read_clipboard}
             pipeline._work(self.wav, duration,
@@ -77,7 +83,8 @@ class Chain(DikteTest):
         run = self.run_chain()
         self.assertEqual(run["failures"], [])
         self.assertEqual(run["done"][0],
-                         ("uh, book it for Thursday", "Book it for Thursday.", ""))
+                         ("uh, book it for Thursday", "Book it for Thursday.",
+                          "", "en"))
         run["copy"].assert_called_once_with("Book it for Thursday.")
         run["press"].assert_called_once_with(self.conf["paste_shortcut"],
                                              focus=None)
@@ -129,7 +136,7 @@ class Chain(DikteTest):
         self.conf["restore_clipboard"] = True
         run = self.run_chain(paste_error=paste.PasteError("not trusted"))
         self.assertEqual(run["failures"], [])
-        raw, text, warning = run["done"][0]
+        raw, text, warning, _lang = run["done"][0]
         self.assertIn("not trusted", warning)
         run["copy_bytes"].assert_not_called()
 
@@ -182,17 +189,41 @@ class Chain(DikteTest):
         self.assertEqual(run["transcribe"].call_args.kwargs["language"], "tr")
         self.assertEqual(run["transcribe"].call_args.kwargs["prompt"], "Paraşüt")
 
+    def test_auto_mode_asks_for_the_detected_language_and_records_it(self):
+        run = self.run_chain(detected="tr")
+        told = run["transcribe_detected"].call_args.kwargs
+        self.assertEqual(told["language"], "auto")
+        self.assertEqual(cfg.read_history()[0]["speech_language"], "tr")
+        self.assertEqual(run["done"][0][3], "tr")
+        run["transcribe"].assert_not_called()
+
+    def test_the_detected_language_is_told_to_the_cleanup_prompt(self):
+        # The mock stands in for api.cleanup, which the cleanup module calls
+        # with (text, key, model, system_prompt, …); the prompt is the fourth.
+        self.conf["transcribe_prompt"] = "Paraşüt"
+        run = self.run_chain(detected="tr")
+        prompt = run["cleanup"].call_args.args[3]
+        # Turkish was detected, so the Turkish glossary rule is appended.
+        self.assertIn("KONUŞMACININ KULLANDIĞI İSİM VE TERİMLER", prompt)
+
+    def test_a_fixed_language_needs_no_detection(self):
+        self.conf["language"] = "en"
+        run = self.run_chain()
+        run["transcribe"].assert_called_once()
+        run["transcribe_detected"].assert_not_called()
+        self.assertEqual(cfg.read_history()[0]["speech_language"], "en")
+
     # ---- silence and stock phrases ----------------------------------------
 
     def test_room_tone_costs_no_api_call(self):
         run = self.run_chain(rms=[0.00001] * 60)
-        run["transcribe"].assert_not_called()
+        run["transcribe_detected"].assert_not_called()
         self.assertIn("No speech", run["failures"][0])
 
     def test_the_silence_check_can_be_switched_off(self):
         self.conf["skip_silent"] = False
         run = self.run_chain(rms=[0.00001] * 60)
-        run["transcribe"].assert_called_once()
+        run["transcribe_detected"].assert_called_once()
 
     def test_a_stock_phrase_from_a_short_clip_is_thrown_away(self):
         run = self.run_chain(duration=2.0, transcript="Altyazı M.K.")
@@ -208,7 +239,7 @@ class Chain(DikteTest):
 
     def test_a_failed_cleanup_still_pastes_the_transcript(self):
         run = self.run_chain(cleanup_error=api.ApiError("rate limited"))
-        _raw, text, warning = run["done"][0]
+        _raw, text, warning, _lang = run["done"][0]
         self.assertEqual(text, "uh, book it for Thursday")
         self.assertIn("rate limited", warning)
         run["copy"].assert_called_once_with("uh, book it for Thursday")
@@ -220,6 +251,9 @@ class Chain(DikteTest):
         self.assertEqual(cfg.read_history()[0]["cleanup_error"], "bad key")
 
     def test_a_failed_transcription_ends_the_run(self):
+        # This path mocks api.transcribe, so it wants
+        # the plain (fixed-language) transcription.
+        self.conf["language"] = "tr"
         pipeline = worker.Pipeline(self.conf)
         failures = []
         pipeline.failed.connect(failures.append)
@@ -231,6 +265,9 @@ class Chain(DikteTest):
         copy.assert_not_called()
 
     def test_a_clipboard_that_will_not_take_it(self):
+        # This path mocks api.transcribe, so it wants
+        # the plain (fixed-language) transcription.
+        self.conf["language"] = "tr"
         pipeline = worker.Pipeline(self.conf)
         failures = []
         pipeline.failed.connect(failures.append)
@@ -243,6 +280,9 @@ class Chain(DikteTest):
         self.assertIn("wl-copy", failures[0])
 
     def test_an_unexpected_error_is_reported_rather_than_swallowed(self):
+        # This path mocks api.transcribe, so it wants
+        # the plain (fixed-language) transcription.
+        self.conf["language"] = "tr"
         pipeline = worker.Pipeline(self.conf)
         failures = []
         pipeline.failed.connect(failures.append)
@@ -280,6 +320,9 @@ class Chain(DikteTest):
         run["press"].assert_not_called()
 
     def test_a_command_that_was_cancelled(self):
+        # This path mocks api.transcribe, so it wants
+        # the plain (fixed-language) transcription.
+        self.conf["language"] = "tr"
         pipeline = worker.Pipeline(self.conf)
         cancels = []
         pipeline.cancelled.connect(lambda: cancels.append(True))
@@ -289,6 +332,9 @@ class Chain(DikteTest):
         self.assertEqual(cancels, [True])
 
     def test_an_agent_that_is_not_installed(self):
+        # This path mocks api.transcribe, so it wants
+        # the plain (fixed-language) transcription.
+        self.conf["language"] = "tr"
         pipeline = worker.Pipeline(self.conf)
         failures = []
         pipeline.failed.connect(failures.append)