diff --git a/api.py b/api.py index 1f35e60..775fe31 100644 --- a/api.py +++ b/api.py @@ -309,7 +309,7 @@ def local_failure(service, server, exc): exc.status) -def _transcribe_request(target, wav_path, language, prompt, response_format, +def _transcribe_request(target, audio_path, language, prompt, response_format, granularity=None, timeout=300, aborter=None): if target.provider == "local": # The timeouts here are sized for a hosted API, where a slow answer is a @@ -329,7 +329,7 @@ def _transcribe_request(target, wav_path, language, prompt, response_format, fields.append(("prompt", prompt)) if granularity: fields.append(("timestamp_granularities[]", granularity)) - body, ctype = _multipart(fields, "file", wav_path) + body, ctype = _multipart(fields, "file", audio_path) try: return _request( f"{target.base_url.rstrip('/')}/audio/transcriptions", body, @@ -381,9 +381,9 @@ def _merge_word_splits(segments): return merged -def transcribe(target, wav_path, language="", prompt="", timeout=300, aborter=None): +def transcribe(target, audio_path, language="", prompt="", timeout=300, aborter=None): data = _transcribe_request( - target, wav_path, language, prompt, "json", timeout=timeout, aborter=aborter + target, audio_path, language, prompt, "json", timeout=timeout, aborter=aborter ) text = data.get("text") or "" if target.provider == "local": @@ -394,12 +394,12 @@ def transcribe(target, wav_path, language="", prompt="", timeout=300, aborter=No return text -def transcribe_segments(target, wav_path, language="", prompt="", timeout=300, +def transcribe_segments(target, audio_path, language="", prompt="", timeout=300, aborter=None): """[(start_seconds, end_seconds, text)] using whisper-1's verbose response.""" data = _transcribe_request( target._replace(model=timestamp_model(target.provider, target.model)), - wav_path, language, prompt, "verbose_json", + audio_path, language, prompt, "verbose_json", granularity="segment", timeout=timeout, aborter=aborter, ) segments = data.get("segments") or [] diff --git a/filetranscribe.py b/filetranscribe.py index b336e3a..4c8b4b7 100644 --- a/filetranscribe.py +++ b/filetranscribe.py @@ -1,8 +1,15 @@ """Transcribe an existing audio/video file with the same models. -ffmpeg converts whatever comes in to 16 kHz mono WAV; long files are cut into -chunks that stay under the API's size limit, then stitched back together with -their timestamps shifted into place. +ffmpeg converts whatever comes in to 16 kHz mono WAV, and for a hosted API to +mp3 on top of that. The upload limit is the only reason a file is ever cut up, +and uncompressed audio reaches it after ten minutes where mp3 takes an hour. + +That is worth the encoder, because a cut is not free. Whisper hears in thirty +second windows and decides for itself where one cue ends and the next begins; a +chunk that starts in the middle of a sentence can come back as one cue per +window, twenty seconds of text at a time, for the whole rest of the chunk. So +the file is cut as rarely as the limit allows, what is cut overlaps, and +stitch() drops the half that was heard twice. """ import contextlib @@ -21,7 +28,10 @@ import cleanup import ggml from i18n import t -CHUNK_SECONDS = 600 # 10 min ≈ 19 MB at 16 kHz mono s16 +UPLOAD_LIMIT = 24 * 1024 * 1024 # the APIs take 25 MB; leave the form its room +MP3_BITRATE = "48k" # mono speech at 16 kHz: whisper hears nothing less +OVERLAP_SECONDS = 30 # a whisper window: how far back a chunk starts +WAV_CHUNK_SECONDS = 600 # 19 MB, for the caller that uploads the WAV itself CLEANUP_CHUNK_CHARS = 12000 # keep each cleanup call comfortably small RATE = 16000 MIN_SUBTITLE_SECONDS = 1.5 # how long a cue with no end time of its own stays up @@ -88,21 +98,23 @@ class FileTranscriber(QObject): wav_path = _to_wav(path, workdir, self._abort) self._check() - chunks = split_wav(wav_path, workdir) + target = conf.transcribe_target() + self._local = ggml.whisper if target.provider == "local" else None + chunks = self._chunks(wav_path, workdir, target, timestamps) if len(chunks) > 1: self.progress.emit(t("Splitting into {count} chunks…", count=len(chunks))) - target = conf.transcribe_target() - self._local = ggml.whisper if target.provider == "local" else None pieces = [] segments = [] for index, (chunk_path, offset) in enumerate(chunks, start=1): self._check() self.progress.emit( - t("Transcribing chunk {index}/{count}…", index=index, count=len(chunks)) + t("Transcribing chunk {index}/{count}…", + index=index, count=len(chunks)) + if len(chunks) > 1 else t("Transcribing…") ) if timestamps: - segments.extend( + segments = stitch(segments, [ (start + offset, end + offset, line) for start, end, line in api.transcribe_segments( target, @@ -111,9 +123,7 @@ class FileTranscriber(QObject): prompt=conf["transcribe_prompt"], aborter=self._abort, ) - ) - pieces = [f"[{format_timestamp(start)}] {line}" - for start, _, line in segments] + ]) else: pieces.append(api.transcribe( target, @@ -123,6 +133,9 @@ class FileTranscriber(QObject): aborter=self._abort, )) + if timestamps: + pieces = [f"[{format_timestamp(start)}] {line}" + for start, _, line in segments] text = "\n".join(pieces) if timestamps else " ".join(pieces) if do_cleanup and text: @@ -141,6 +154,28 @@ class FileTranscriber(QObject): if workdir: shutil.rmtree(workdir, ignore_errors=True) + def _chunks(self, wav_path, workdir, target, timestamps): + """[(the file to send, its offset in seconds)], one entry where it can be. + + A server on this machine is handed the WAV as it is: nothing is being + uploaded, so the encoder would cost quality and buy nothing. + """ + if target.provider == "local": + return [(wav_path, 0.0)] + + whole = _to_mp3(wav_path, workdir, "audio.mp3", self._abort) + seconds = chunk_seconds(whole, wav_seconds(wav_path)) + if not seconds: + return [(whole, 0.0)] + + # Only a timestamped run can tell what it has already heard, so only it + # can afford the overlap that keeps a cue off the cut. + self._check() + pieces = split_wav(wav_path, workdir, seconds, + OVERLAP_SECONDS if timestamps else 0) + return [(_to_mp3(piece, workdir, f"chunk-{index:03d}.mp3", self._abort), offset) + for index, (piece, offset) in enumerate(pieces)] + def _cleanup(self, text, timestamps): conf = self.conf self._local = ggml.llm if cleanup.provider(conf) == "local" else None @@ -220,9 +255,32 @@ def _reap(proc): def _to_wav(path, workdir, aborter=None): out = os.path.join(workdir, "audio.wav") + return _ffmpeg(["-i", path, "-vn", "-ac", "1", "-ar", str(RATE), + "-c:a", "pcm_s16le", out], out, aborter) + + +def _to_mp3(wav_path, workdir, name, aborter=None): + """The same audio at a fifth of the size. + + Which is the whole of it: uncompressed, an hour of speech is four uploads + and so three cuts, and every cut is a chance of the model losing the thread + of where its cues should end. Encoded it is one upload and no cuts. The + bitrate is far above what a 16 kHz mono voice has left to lose. + """ + out = os.path.join(workdir, name) + try: + return _ffmpeg(["-i", wav_path, "-c:a", "libmp3lame", "-b:a", MP3_BITRATE, out], + out, aborter) + except api.ApiError: + # An ffmpeg built without the encoder, which is rare and not worth + # failing over: the WAV transcribes just as well, it only has to be cut + # up more often to fit in a request. + return wav_path + + +def _ffmpeg(args, out, aborter=None): proc = subprocess.Popen( - ["ffmpeg", "-nostdin", "-y", "-i", path, "-vn", - "-ac", "1", "-ar", str(RATE), "-c:a", "pcm_s16le", out], + ["ffmpeg", "-nostdin", "-y", *args], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, ) @@ -242,32 +300,80 @@ def _to_wav(path, workdir, aborter=None): return out -def split_wav(wav_path, workdir): - """[(chunk path, offset in seconds)], a single entry for short files.""" +def wav_seconds(wav_path): + with contextlib.closing(wave.open(wav_path, "rb")) as src: + return src.getnframes() / (src.getframerate() or RATE) + + +def chunk_seconds(path, duration): + """How many seconds of this audio fit in one request, or 0 when all of it does. + + Measured rather than worked out: what an encoder makes of an hour of speech + depends on the speech, and the file on disk is the only honest answer. + """ + size = os.path.getsize(path) + if size <= UPLOAD_LIMIT or duration <= 0: + return 0.0 + return max(60.0, duration * UPLOAD_LIMIT / size * 0.95) + + +def split_wav(wav_path, workdir, seconds=WAV_CHUNK_SECONDS, overlap=OVERLAP_SECONDS): + """[(chunk path, offset in seconds)], a single entry for short files. + + Every chunk but the first starts `overlap` seconds inside the one before it, + so the sentence the cut fell in the middle of is heard whole by one of them. + stitch() is what drops the telling that was cut short. + """ with contextlib.closing(wave.open(wav_path, "rb")) as src: rate = src.getframerate() total = src.getnframes() - per_chunk = CHUNK_SECONDS * rate - if total <= per_chunk: + per_chunk = int(seconds * rate) + if per_chunk <= 0 or total <= per_chunk: return [(wav_path, 0.0)] + # Half a chunk is the most an overlap can be and still be an overlap. + step = per_chunk - int(max(0.0, min(overlap, seconds / 2)) * rate) chunks = [] - index = 0 - while True: + position = 0 + while position < total: + # What is left is shorter than the overlap, so the chunk before this + # one already holds all of it. + if chunks and total - position <= per_chunk - step: + break + src.setpos(position) frames = src.readframes(per_chunk) if not frames: break - path = os.path.join(workdir, f"chunk-{index:03d}.wav") + path = os.path.join(workdir, f"chunk-{len(chunks):03d}.wav") with contextlib.closing(wave.open(path, "wb")) as dst: dst.setnchannels(src.getnchannels()) dst.setsampwidth(src.getsampwidth()) dst.setframerate(rate) dst.writeframes(frames) - chunks.append((path, index * CHUNK_SECONDS)) - index += 1 + chunks.append((path, position / rate)) + position += step return chunks +def stitch(collected, incoming): + """Add a chunk's segments to the ones before it, minus what was heard twice. + + The chunks overlap, so the sentence the cut landed in is in both of them: + cut short as the last cue of the chunk before, and whole somewhere in this + one. This chunk's telling of it is the one that stands, and the chunk before + gives way from wherever that telling begins, so that nothing is said twice + and the cues still run forwards. + """ + if not collected: + return list(incoming) + kept = [segment for segment in incoming if segment[1] > collected[-1][0]] + if not kept: + return collected + seam = kept[0][0] + head = [segment for segment in collected if segment[1] <= seam] + return (head or collected[:-1]) + kept + + def split_text(text, timestamps): """Break long text into cleanup-sized blocks, never mid-line.""" if len(text) <= CLEANUP_CHUNK_CHARS: diff --git a/meeting.py b/meeting.py index c178668..5da05d3 100644 --- a/meeting.py +++ b/meeting.py @@ -170,6 +170,7 @@ class MeetingPipeline(QObject): chunk_dir = os.path.join(workdir, speaker) os.makedirs(chunk_dir, exist_ok=True) chunks = filetranscribe.split_wav(path, chunk_dir) + heard = [] for index, (chunk_path, offset) in enumerate(chunks, start=1): self._check() self._say(t("Transcribing {side}: {index}/{count}…", @@ -178,12 +179,15 @@ class MeetingPipeline(QObject): # would cost money to be told so, and can invent a sentence. if self._silent(chunk_path): continue - segments.extend( - (start + offset, end + offset, text, speaker) + # The chunks overlap, so what the cut fell in the middle of is + # in two of them; stitch keeps the one that heard it whole. + heard = filetranscribe.stitch(heard, [ + (start + offset, end + offset, text) for start, end, text in api.transcribe_segments( target, chunk_path, language=language, prompt=hint ) - ) + ]) + segments.extend((start, end, text, speaker) for start, end, text in heard) if not segments: raise api.ApiError(t("Neither side of the recording had any speech in it.")) diff --git a/tests/test_filetranscribe.py b/tests/test_filetranscribe.py index b45a385..ff917aa 100644 --- a/tests/test_filetranscribe.py +++ b/tests/test_filetranscribe.py @@ -7,6 +7,7 @@ made up a stamp nobody recorded. """ import contextlib +import os import time import unittest import wave @@ -133,14 +134,12 @@ class SplitWav(DikteTest): def test_a_long_file_is_cut_at_the_chunk_length(self): path = self.wav(5) - with mock.patch.object(ft, "CHUNK_SECONDS", 2): - chunks = ft.split_wav(path, self.root) + chunks = ft.split_wav(path, self.root, 2, overlap=0) self.assertEqual([offset for _, offset in chunks], [0, 2, 4]) def test_the_chunks_add_up_to_the_original(self): path = self.wav(5) - with mock.patch.object(ft, "CHUNK_SECONDS", 2): - chunks = ft.split_wav(path, self.root) + chunks = ft.split_wav(path, self.root, 2, overlap=0) total = 0 for chunk_path, _ in chunks: with contextlib.closing(wave.open(chunk_path, "rb")) as wav: @@ -150,10 +149,125 @@ class SplitWav(DikteTest): def test_the_chunks_do_not_write_over_each_other(self): path = self.wav(5) - with mock.patch.object(ft, "CHUNK_SECONDS", 2): - chunks = ft.split_wav(path, self.root) + chunks = ft.split_wav(path, self.root, 2, overlap=0) self.assertEqual(len({chunk for chunk, _ in chunks}), len(chunks)) + def test_a_chunk_starts_inside_the_one_before_it(self): + """The cut is what makes whisper lose the thread, so nobody hears only + one side of it.""" + path = self.wav(10) + chunks = ft.split_wav(path, self.root, 4, overlap=1) + self.assertEqual([offset for _, offset in chunks], [0, 3, 6]) + with contextlib.closing(wave.open(chunks[1][0], "rb")) as wav: + self.assertEqual(wav.getnframes(), 4 * 16000) + + def test_an_overlap_is_never_more_than_half_a_chunk(self): + path = self.wav(10) + chunks = ft.split_wav(path, self.root, 4, overlap=60) + self.assertEqual([offset for _, offset in chunks], [0, 2, 4, 6]) + + def test_a_tail_the_chunk_before_already_holds_is_not_cut_again(self): + path = self.wav(9) + chunks = ft.split_wav(path, self.root, 4, overlap=1) + # 0-4, 3-7, 6-9: a fourth starting at 9 would be the last second again. + self.assertEqual([offset for _, offset in chunks], [0, 3, 6]) + + +class Stitch(unittest.TestCase): + def test_the_first_chunk_is_taken_as_it_is(self): + segments = [(0.0, 1.0, "one"), (1.0, 2.0, "two")] + self.assertEqual(ft.stitch([], segments), segments) + + def test_the_cue_the_cut_ran_through_is_replaced(self): + collected = [(0.0, 4.0, "a whole sentence"), (4.0, 5.0, "cut in ha")] + incoming = [(3.0, 4.0, "sentence"), (4.0, 6.0, "cut in half")] + self.assertEqual(ft.stitch(collected, incoming), + [(0.0, 4.0, "a whole sentence"), (4.0, 6.0, "cut in half")]) + + def test_the_chunk_before_gives_way_where_the_new_telling_starts(self): + """The two chunks put the sentence in different cues; whichever way they + fall, nothing is said twice and the cues run forwards.""" + collected = [(0.0, 3.0, "one"), (3.0, 5.0, "two"), (5.0, 6.0, "three cut")] + incoming = [(2.0, 4.5, "one and two"), (4.5, 7.0, "two and three whole")] + stitched = ft.stitch(collected, incoming) + self.assertEqual(stitched, [(0.0, 3.0, "one"), (4.5, 7.0, "two and three whole")]) + for before, after in zip(stitched, stitched[1:]): + self.assertLessEqual(before[1], after[0]) + + def test_a_chunk_with_nothing_in_it_takes_nothing_away(self): + collected = [(0.0, 4.0, "one")] + self.assertEqual(ft.stitch(collected, []), collected) + + def test_a_chunk_that_heard_only_what_was_already_heard_adds_nothing(self): + collected = [(0.0, 4.0, "one"), (4.0, 5.0, "two")] + self.assertEqual(ft.stitch(collected, [(1.0, 2.0, "one")]), collected) + + +class ChunkSeconds(DikteTest): + def file(self, size): + path = self.path("audio.mp3") + with open(path, "wb") as fh: + fh.write(b"\x00" * size) + return path + + def test_a_file_that_fits_is_not_cut_at_all(self): + self.assertEqual(ft.chunk_seconds(self.file(1024), 600), 0.0) + + def test_a_file_over_the_limit_is_cut_by_what_it_measured(self): + # Twice the limit over an hour, so a little under half an hour fits. + seconds = ft.chunk_seconds(self.file(ft.UPLOAD_LIMIT * 2), 3600) + self.assertGreater(seconds, 1500) + self.assertLess(seconds, 1800) + + def test_a_file_with_no_length_is_left_whole(self): + self.assertEqual(ft.chunk_seconds(self.file(ft.UPLOAD_LIMIT * 2), 0), 0.0) + + +class Chunks(DikteTest): + """What each provider is handed, and in how many pieces.""" + + def setUp(self): + super().setUp() + self.wav = make_wav(self.path("audio.wav"), silence(3)) + self.worker = ft.FileTranscriber(self.config()) + + def target(self, provider): + return api.Target(provider, provider, "key", "https://example.test", "whisper-1") + + def test_a_model_on_this_machine_is_handed_the_wav(self): + """Nothing is uploaded, so the encoder would cost quality for nothing.""" + with mock.patch.object(ft, "_to_mp3") as encode: + chunks = self.worker._chunks(self.wav, self.root, self.target("local"), True) + self.assertEqual(chunks, [(self.wav, 0.0)]) + encode.assert_not_called() + + def test_a_hosted_model_is_handed_one_mp3(self): + with mock.patch.object(ft, "_to_mp3", side_effect=lambda p, d, name, *a: + make_wav(os.path.join(d, name), silence(1))): + chunks = self.worker._chunks(self.wav, self.root, + self.target("openrouter"), True) + self.assertEqual(len(chunks), 1) + self.assertTrue(chunks[0][0].endswith("audio.mp3")) + + def test_a_file_too_big_to_upload_is_cut_and_encoded_in_pieces(self): + wav = make_wav(self.path("long.wav"), silence(120)) + + def encode(path, workdir, name, *args): + # The whole file is over the limit; the pieces are not. + size = ft.UPLOAD_LIMIT * 2 if name == "audio.mp3" else 1024 + out = os.path.join(workdir, name) + with open(out, "wb") as fh: + fh.write(b"\x00" * size) + return out + + with mock.patch.object(ft, "_to_mp3", side_effect=encode): + chunks = self.worker._chunks(wav, self.root, + self.target("openrouter"), True) + self.assertGreater(len(chunks), 1) + self.assertEqual(chunks[0][1], 0.0) + for path, _ in chunks: + self.assertTrue(path.endswith(".mp3")) + class Transcriber(DikteTest): """The chain, with ffmpeg and both API calls faked.""" @@ -175,6 +289,8 @@ class Transcriber(DikteTest): return make_wav(self.path("converted.wav"), tone(1.0)) with mock.patch.object(ft, "_to_wav", side_effect=to_wav), \ + mock.patch.object(ft, "_to_mp3", + side_effect=lambda path, *a, **k: path), \ mock.patch.object(ft.shutil, "which", return_value="/usr/bin/ffmpeg"), \ mock.patch.object(api, "transcribe", side_effect=fail or (lambda *a, **k: transcript)), \ @@ -237,6 +353,8 @@ class Transcriber(DikteTest): def test_the_request_is_handed_the_stop_to_watch(self): worker = ft.FileTranscriber(self.conf) with mock.patch.object(ft, "_to_wav", side_effect=lambda *a: self.source), \ + mock.patch.object(ft, "_to_mp3", + side_effect=lambda path, *a, **k: path), \ mock.patch.object(ft.shutil, "which", return_value="/usr/bin/ffmpeg"), \ mock.patch.object(api, "transcribe", return_value="text") as call: worker._work(self.source, False, False)