diff --git a/README.md b/README.md index d1b164c..3a2b14e 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,11 @@ running. written for subtitles, so the lines keep their place and nothing is shortened. - **History** of every dictation under Settings → History, with a size limit and right-click to delete. +- **The speech language is detected, not picked.** Auto is the default: whisper + on this machine says what it heard, the hosted providers transcribe in + whatever language comes in without being told, and the detected language + lands in the history and decides which cleanup prompt (Turkish or the + language-agnostic one) a run gets. A fixed language still overrides it. - **Turkish and English interface**, following the system locale by default. ## The global shortcuts, and the logout KDE needs diff --git a/README.tr.md b/README.tr.md index 774ea2f..86a4b82 100644 --- a/README.tr.md +++ b/README.tr.md @@ -214,6 +214,11 @@ olmasını ister. yerinde kalır, hiçbir şey kısaltılmaz. - **Geçmiş** Ayarlar → Geçmiş sekmesinde; boyut sınırı var, sağ tıklayıp silebilirsin. +- **Konuşma dili seçilmez, algılanır.** Varsayılan otomatiktir: bu makinedeki + whisper ne duyduğunu söyler, bulut sağlayıcılar söylenmeden de hangi dilde + konuşuluyorsa o dilde yazar; algılanan dil geçmişe düşer ve bir kaydın hangi + temizleme promptunu alacağını belirler (Türkçe mi, dile duyarsız olanı mı). + Sabit bir dil yine de bunun önüne geçer. - **Türkçe ve İngilizce arayüz**, varsayılan olarak sistem dilini izler. ## Global kısayollar ve KDE'nin istediği oturum kapatma diff --git a/dikte/api.py b/dikte/api.py index 503ee1f..504167e 100644 --- a/dikte/api.py +++ b/dikte/api.py @@ -367,7 +367,8 @@ def local_failure(service, server, exc): def _transcribe_request(target, audio_path, language, prompt, response_format, - granularity=None, timeout=300, aborter=None): + granularity=None, timeout=300, aborter=None, + detect_language=False): if target.provider == "local": # The timeouts here are sized for a hosted API, where a slow answer is a # bill running. Locally the only thing being spent is time. @@ -379,6 +380,12 @@ def _transcribe_request(target, audio_path, language, prompt, response_format, fields = [("model", target.model), ("response_format", response_format)] if language and language != "auto": fields.append(("language", language)) + if detect_language: + # whisper.cpp was started with -nlp, which keeps the language + # probability sweep off every request. Detection is only worth that + # sweep for the run that asked for it, so it is switched back on here, + # per request, and reported in the verbose_json answer. + fields.append(("no_language_probabilities", "false")) # OpenRouter takes the hint field and throws it away, so spare it the bytes. # The same words still reach the cleanup model as a glossary. whisper.cpp # takes it as the initial prompt, the way OpenAI does. @@ -546,6 +553,44 @@ def transcribe(target, audio_path, language="", prompt="", timeout=300, aborter= return text +# whisper.cpp reports what it heard as a lowercase full name ("turkish", +# "english", "german"…); the settings and the cleanup prompt speak in two-letter +# codes. Only the handful Dikte offers as a fixed choice get a code; anything +# else is left as the empty string, which the caller reads as "unknown" rather +# than guessing at a language it has no label for. +_DETECTED_TO_CODE = { + "english": "en", "turkish": "tr", "german": "de", + "french": "fr", "spanish": "es", "arabic": "ar", +} + + +def transcribe_detected(target, audio_path, language="", prompt="", timeout=300, + aborter=None): + """(text, code) with the language the model heard. + + The spoken language is only knowable when the transcription model reports + it, and only whisper.cpp does: the hosted endpoints accept "auto" but never + say what they heard. So detection is asked for exactly where it can be + answered, the local server in auto mode, and every other run transcribes + as before and hands back an empty code. + """ + if target.provider == "local" and language == "auto": + data = _transcribe_request( + target, audio_path, language, prompt, "verbose_json", + detect_language=True, timeout=timeout, aborter=aborter, + ) + text = _local_text(data.get("text") or "").strip() + if not text: + raise ApiError(t("Transcript came back empty.")) + detected = data.get("detected_language") + code = _DETECTED_TO_CODE.get( + detected.strip().lower(), "") if isinstance(detected, str) else "" + return text, code + text = transcribe(target, audio_path, language=language, prompt=prompt, + timeout=timeout, aborter=aborter) + return text, "" + + def transcribe_segments(target, audio_path, language="", prompt="", timeout=300, aborter=None): """[(start_seconds, end_seconds, text)] using whisper-1's verbose response.""" diff --git a/dikte/app.py b/dikte/app.py index e31f5d8..267956f 100644 --- a/dikte/app.py +++ b/dikte/app.py @@ -1084,7 +1084,7 @@ class Dikte: if not self._transcripts_pending: self._settle(DICTATION, payload) - def _on_finished(self, _raw, text, warning): + def _on_finished(self, _raw, text, warning, speech_language): if warning: # The text was still pasted, but cleanup did not run. Say so loudly: # a rejected key otherwise looks exactly like working dictation. @@ -1105,9 +1105,10 @@ class Dikte: t("{action}: {preview}", action=action, preview=_preview(text)) ) self._transcript_settled({"ok": True, "text": text, "raw": _raw, - "warning": warning}) + "warning": warning, + "speech_language": speech_language}) - def _on_ask_finished(self, _raw, text, warning): + def _on_ask_finished(self, _raw, text, warning, speech_language): agent = assistant.display_name(self.conf) if warning: # A tool the agent was not allowed to touch otherwise looks exactly @@ -1128,7 +1129,8 @@ class Dikte: ) self._set_ask_state(IDLE) self._settle(ASK, {"ok": True, "answer": text, "question": _raw, - "warning": warning, "agent": agent}) + "warning": warning, "agent": agent, + "speech_language": speech_language}) def _on_ask_cancelled(self): self.ask_overlay.show_done(t("Stopped."), 2000) diff --git a/dikte/config.py b/dikte/config.py index 86df888..9a7d10a 100644 --- a/dikte/config.py +++ b/dikte/config.py @@ -400,7 +400,9 @@ DEFAULTS = { # What a timestamped run (subtitles) asks OpenRouter for: not every model # there returns segment times. Empty -> openai/whisper-1. "openrouter_file_model": "", - "language": "tr", + # A stored language overrides this default. Hosted providers receive no + # language hint in auto mode; local whisper also reports the detected code. + "language": "auto", "transcribe_prompt": "", # --- whisper.cpp, on this machine --------------------------------------- @@ -733,13 +735,23 @@ class Config: return self["cleanup_provider"] == "local" def cleanup_prompt(self, with_timestamps=False, with_speakers=False, - subtitles=False): - turkish = i18n.language() == "tr" + subtitles=False, speech=""): + """`speech` is the two-letter code of the language that was heard, when + the transcription model reported one. The default prompts and the + glossary rule only exist in Turkish and English, so a detected Turkish + recording gets the Turkish prompt and any other detected language, or + none at all, the English one, which is written not to care what + language the transcript is in. Nothing else calls this with it, so the + interface language keeps deciding everywhere the speech was not asked + about.""" + turkish = (speech == "tr") if speech else i18n.language() == "tr" if subtitles: prompt = (self["file_cleanup_prompt"].strip() - or default_file_cleanup_prompt()) + or (FILE_CLEANUP_PROMPT_TR if turkish + else FILE_CLEANUP_PROMPT_EN)) else: - prompt = self["cleanup_prompt"].strip() or default_cleanup_prompt() + prompt = (self["cleanup_prompt"].strip() + or (CLEANUP_PROMPT_TR if turkish else CLEANUP_PROMPT_EN)) glossary = self["transcribe_prompt"].strip() if with_speakers: glossary = "\n".join(x for x in (glossary, self.participants()) if x) diff --git a/dikte/ggml.py b/dikte/ggml.py index 2160f36..e3301e4 100644 --- a/dikte/ggml.py +++ b/dikte/ggml.py @@ -1547,12 +1547,13 @@ def _whisper_args(settings): binary, "-m", str(model), "--inference-path", INFERENCE_PATH, # Whatever language the request does not name. api.py leaves the field - # out when the language is "auto", and the server's own default is - # English rather than detection. + # out when the language is "auto", and the server's own language is + # set here: "auto" makes whisper.cpp detect what it hears. "-l", "auto", # Stock phrases invented for near-silence come from non-speech tokens, # and verbose_json otherwise pays for a language probability sweep - # nothing here reads. + # nobody asked for. A request that wants the detected language switches + # that back on per request. "-sns", "-nlp", ] if int(settings["threads"]) > 0: diff --git a/dikte/worker.py b/dikte/worker.py index e4fcca8..48fb82b 100644 --- a/dikte/worker.py +++ b/dikte/worker.py @@ -37,7 +37,7 @@ _paste_lock = threading.Lock() class Pipeline(QObject): stage = pyqtSignal(str) # human-readable progress line - finished = pyqtSignal(str, str, str) # raw transcript, final text, warning + finished = pyqtSignal(str, str, str, str) # raw, final text, warning, language failed = pyqtSignal(str) cancelled = pyqtSignal() @@ -120,12 +120,24 @@ class Pipeline(QObject): try: self.stage.emit(t("Transcribing…")) target = conf.transcribe_target() - raw = api.transcribe( - target, - wav_path, - language=conf["language"], - prompt=conf["transcribe_prompt"], - ) + # The spoken language is only knowable after the fact, and only the + # local server says what it heard: auto mode asks it there, and + # every other run (a fixed language, or a hosted provider that + # detects but stays silent) transcribes as before. + auto = conf["language"] == "auto" + if auto: + raw, detected = api.transcribe_detected( + target, wav_path, language=conf["language"], + prompt=conf["transcribe_prompt"], + ) + else: + raw = api.transcribe( + target, + wav_path, + language=conf["language"], + prompt=conf["transcribe_prompt"], + ) + detected = "" if conf["filter_hallucinations"] and vad.looks_like_hallucination(raw, duration): self._discard(wav_path) @@ -134,6 +146,10 @@ class Pipeline(QObject): text = raw warning = "" + # The language the run actually spoke, reported to the window, the + # clipboard path and the history alike: the detected code, or the + # configured one when nothing was detected to replace it. + speech_language = detected or conf["language"] # Remembered rather than re-derived at the history write below: the # ask path runs cleanup under a different setting, and the record # should say what happened, not what one of the two gates implies. @@ -145,7 +161,7 @@ class Pipeline(QObject): self.stage.emit(t("Cleaning up…")) cleaned = True try: - text = cleanup.run(raw, conf, conf.cleanup_prompt()) + text = cleanup.run(raw, conf, conf.cleanup_prompt(speech=detected)) except api.ApiError as exc: # Keep the transcript, but never let the failure pass unseen: # a rejected key would otherwise look like working dictation. @@ -183,6 +199,7 @@ class Pipeline(QObject): "question": question, "assistant": assistant.provider(conf) if ask else "", "assistant_model": assistant.model(conf) if ask else "", + "speech_language": speech_language, "raw": raw, "text": text, } @@ -221,7 +238,7 @@ class Pipeline(QObject): time.sleep(0.35) paste.copy_bytes(previous) - self.finished.emit(raw, text, warning) + self.finished.emit(raw, text, warning, speech_language) except assistant.Cancelled: self.cancelled.emit() diff --git a/tests/test_api.py b/tests/test_api.py index b29ac74..75da6da 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -827,6 +827,40 @@ class TranscribeHere(DikteTest): api.transcribe_segments(LOCAL, self.wav) self.assertEqual(multipart_fields(calls[0])["model"], "ggml-base.bin") + # ---- the detected language -------------------------------------------- + + def test_auto_mode_asks_whisper_for_the_detected_language(self): + # The -nlp the server was started with is switched back on for this one + # request, so whisper's verbose_json reports what it heard. + reply = {"text": " Merhaba dünya. ", "detected_language": "turkish"} + with fake_urlopen(reply) as calls: + text, code = api.transcribe_detected(LOCAL, self.wav, language="auto") + fields = multipart_fields(calls[0]) + self.assertEqual(fields["response_format"], "verbose_json") + self.assertEqual(fields["no_language_probabilities"], "false") + self.assertNotIn("language", fields) + self.assertEqual(text, "Merhaba dünya.") + self.assertEqual(code, "tr") + + def test_a_fixed_language_reports_no_detection(self): + with fake_urlopen({"text": "hello"}) as calls: + text, code = api.transcribe_detected(LOCAL, self.wav, language="tr") + self.assertNotIn("no_language_probabilities", multipart_fields(calls[0])) + self.assertEqual(text, "hello") + self.assertEqual(code, "") + + def test_a_detected_language_without_a_code_stays_unknown(self): + with fake_urlopen({"text": "hello", "detected_language": "somali"}): + _text, code = api.transcribe_detected(LOCAL, self.wav, language="auto") + self.assertEqual(code, "") + + def test_a_hosted_auto_run_transcribes_without_detection(self): + with fake_urlopen({"text": "hi"}) as calls: + text, code = api.transcribe_detected(OPENAI, self.wav, language="auto") + self.assertNotIn("no_language_probabilities", multipart_fields(calls[0])) + self.assertEqual(text, "hi") + self.assertEqual(code, "") + class Stopping(unittest.TestCase): """The Stop button, from the far end: a request already blocked on a reply. diff --git a/tests/test_cli.py b/tests/test_cli.py index a3ffb55..4ab82e1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -746,6 +746,13 @@ class Replies(DikteTest): self.assertEqual(code, 0) self.assertEqual(out.strip(), "Book it for Thursday.") + def test_the_json_answer_carries_the_detected_language(self): + code, out, _ = self.run_verb( + ["--json", "record"], + {"ok": True, "text": "Selam", "speech_language": "tr"}) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out)["speech_language"], "tr") + def test_a_dictation_that_failed(self): code, out, err = self.run_verb(["stop", "--wait"], {"ok": False, "error": "No speech detected"}) diff --git a/tests/test_config.py b/tests/test_config.py index a925579..243c6b2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -274,6 +274,18 @@ class CleanupPrompt(DikteTest): def test_no_glossary_means_no_rule_about_one(self): self.assertEqual(cfg.Config().cleanup_prompt(), cfg.CLEANUP_PROMPT_EN) + def test_a_detected_turkish_recording_gets_the_turkish_prompt(self): + """Auto mode learns what was heard, and that decides the prompt rather + than the interface language.""" + self.write_config({"ui_language": "en", "transcribe_prompt": "Paraşüt"}) + conf = cfg.Config() + prompt = conf.cleanup_prompt(speech="tr") + self.assertEqual(prompt, cfg.CLEANUP_PROMPT_TR + + cfg.GLOSSARY_RULE_TR.format(glossary="Paraşüt")) + self.assertIn("KONUŞMACININ KULLANDIĞI İSİM VE TERİMLER", prompt) + self.assertIn("NAMES AND TERMS THE SPEAKER USES", + conf.cleanup_prompt(speech="de")) + def test_subtitles_use_their_own_prompt(self): conf = cfg.Config() self.assertNotEqual(conf.cleanup_prompt(subtitles=True), conf.cleanup_prompt()) diff --git a/tests/test_worker.py b/tests/test_worker.py index 725ae91..b69f630 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -35,7 +35,7 @@ class Chain(DikteTest): cleaned="Book it for Thursday.", cleanup_error=None, answer=("Booked.", ""), rms=None, clipboard=b"what was there before", paste_error=None, - focus=None): + detected="en", focus=None): pipeline = worker.Pipeline(self.conf) done, failures, stages, cancels = [], [], [], [] pipeline.finished.connect(lambda *args: done.append(args)) @@ -45,14 +45,19 @@ class Chain(DikteTest): cleanup = (mock.Mock(side_effect=cleanup_error) if cleanup_error else mock.Mock(return_value=cleaned)) + # Auto mode takes the detection path; a fixed language the plain one. + # Both are mocked so the chain runs either way without a server. + behavior = {"side_effect": transcribe_error} if transcribe_error \ + else {"return_value": transcript} + detect_behavior = {"side_effect": transcribe_error} if transcribe_error \ + else {"return_value": (transcript, detected)} calls = {} # The chain reports its own failures on stderr, which a test run has no # use for. with contextlib.redirect_stderr(io.StringIO()), \ - mock.patch.object( - api, "transcribe", - **({"side_effect": transcribe_error} if transcribe_error - else {"return_value": transcript})) as tr, \ + mock.patch.object(api, "transcribe", **behavior) as tr, \ + mock.patch.object(api, "transcribe_detected", + **detect_behavior) as tdet, \ mock.patch.object(api, "cleanup", cleanup), \ mock.patch.object(assistant, "ask", return_value=answer) as ask_call, \ mock.patch.object(paste, "copy") as copy, \ @@ -62,7 +67,8 @@ class Chain(DikteTest): return_value=clipboard) as read_clipboard, \ mock.patch.object(worker.time, "sleep", lambda seconds: None): press.side_effect = paste_error - calls = {"transcribe": tr, "cleanup": cleanup, "ask": ask_call, + calls = {"transcribe": tr, "transcribe_detected": tdet, + "cleanup": cleanup, "ask": ask_call, "copy": copy, "copy_bytes": copy_bytes, "press": press, "read_clipboard": read_clipboard} pipeline._work(self.wav, duration, @@ -77,7 +83,8 @@ class Chain(DikteTest): run = self.run_chain() self.assertEqual(run["failures"], []) self.assertEqual(run["done"][0], - ("uh, book it for Thursday", "Book it for Thursday.", "")) + ("uh, book it for Thursday", "Book it for Thursday.", + "", "en")) run["copy"].assert_called_once_with("Book it for Thursday.") run["press"].assert_called_once_with(self.conf["paste_shortcut"], focus=None) @@ -129,7 +136,7 @@ class Chain(DikteTest): self.conf["restore_clipboard"] = True run = self.run_chain(paste_error=paste.PasteError("not trusted")) self.assertEqual(run["failures"], []) - raw, text, warning = run["done"][0] + raw, text, warning, _lang = run["done"][0] self.assertIn("not trusted", warning) run["copy_bytes"].assert_not_called() @@ -182,17 +189,41 @@ class Chain(DikteTest): self.assertEqual(run["transcribe"].call_args.kwargs["language"], "tr") self.assertEqual(run["transcribe"].call_args.kwargs["prompt"], "Paraşüt") + def test_auto_mode_asks_for_the_detected_language_and_records_it(self): + run = self.run_chain(detected="tr") + told = run["transcribe_detected"].call_args.kwargs + self.assertEqual(told["language"], "auto") + self.assertEqual(cfg.read_history()[0]["speech_language"], "tr") + self.assertEqual(run["done"][0][3], "tr") + run["transcribe"].assert_not_called() + + def test_the_detected_language_is_told_to_the_cleanup_prompt(self): + # The mock stands in for api.cleanup, which the cleanup module calls + # with (text, key, model, system_prompt, …); the prompt is the fourth. + self.conf["transcribe_prompt"] = "Paraşüt" + run = self.run_chain(detected="tr") + prompt = run["cleanup"].call_args.args[3] + # Turkish was detected, so the Turkish glossary rule is appended. + self.assertIn("KONUŞMACININ KULLANDIĞI İSİM VE TERİMLER", prompt) + + def test_a_fixed_language_needs_no_detection(self): + self.conf["language"] = "en" + run = self.run_chain() + run["transcribe"].assert_called_once() + run["transcribe_detected"].assert_not_called() + self.assertEqual(cfg.read_history()[0]["speech_language"], "en") + # ---- silence and stock phrases ---------------------------------------- def test_room_tone_costs_no_api_call(self): run = self.run_chain(rms=[0.00001] * 60) - run["transcribe"].assert_not_called() + run["transcribe_detected"].assert_not_called() self.assertIn("No speech", run["failures"][0]) def test_the_silence_check_can_be_switched_off(self): self.conf["skip_silent"] = False run = self.run_chain(rms=[0.00001] * 60) - run["transcribe"].assert_called_once() + run["transcribe_detected"].assert_called_once() def test_a_stock_phrase_from_a_short_clip_is_thrown_away(self): run = self.run_chain(duration=2.0, transcript="Altyazı M.K.") @@ -208,7 +239,7 @@ class Chain(DikteTest): def test_a_failed_cleanup_still_pastes_the_transcript(self): run = self.run_chain(cleanup_error=api.ApiError("rate limited")) - _raw, text, warning = run["done"][0] + _raw, text, warning, _lang = run["done"][0] self.assertEqual(text, "uh, book it for Thursday") self.assertIn("rate limited", warning) run["copy"].assert_called_once_with("uh, book it for Thursday") @@ -220,6 +251,9 @@ class Chain(DikteTest): self.assertEqual(cfg.read_history()[0]["cleanup_error"], "bad key") def test_a_failed_transcription_ends_the_run(self): + # This path mocks api.transcribe, so it wants + # the plain (fixed-language) transcription. + self.conf["language"] = "tr" pipeline = worker.Pipeline(self.conf) failures = [] pipeline.failed.connect(failures.append) @@ -231,6 +265,9 @@ class Chain(DikteTest): copy.assert_not_called() def test_a_clipboard_that_will_not_take_it(self): + # This path mocks api.transcribe, so it wants + # the plain (fixed-language) transcription. + self.conf["language"] = "tr" pipeline = worker.Pipeline(self.conf) failures = [] pipeline.failed.connect(failures.append) @@ -243,6 +280,9 @@ class Chain(DikteTest): self.assertIn("wl-copy", failures[0]) def test_an_unexpected_error_is_reported_rather_than_swallowed(self): + # This path mocks api.transcribe, so it wants + # the plain (fixed-language) transcription. + self.conf["language"] = "tr" pipeline = worker.Pipeline(self.conf) failures = [] pipeline.failed.connect(failures.append) @@ -280,6 +320,9 @@ class Chain(DikteTest): run["press"].assert_not_called() def test_a_command_that_was_cancelled(self): + # This path mocks api.transcribe, so it wants + # the plain (fixed-language) transcription. + self.conf["language"] = "tr" pipeline = worker.Pipeline(self.conf) cancels = [] pipeline.cancelled.connect(lambda: cancels.append(True)) @@ -289,6 +332,9 @@ class Chain(DikteTest): self.assertEqual(cancels, [True]) def test_an_agent_that_is_not_installed(self): + # This path mocks api.transcribe, so it wants + # the plain (fixed-language) transcription. + self.conf["language"] = "tr" pipeline = worker.Pipeline(self.conf) failures = [] pipeline.failed.connect(failures.append)