Merge master: the audio goes up as mp3

api.py

@@ -309,7 +309,7 @@ def local_failure(service, server, exc):
                     exc.status)
 
 
-def _transcribe_request(target, wav_path, language, prompt, response_format,
+def _transcribe_request(target, audio_path, language, prompt, response_format,
                         granularity=None, timeout=300, aborter=None):
     if target.provider == "local":
         # The timeouts here are sized for a hosted API, where a slow answer is a
@@ -329,7 +329,7 @@ def _transcribe_request(target, wav_path, language, prompt, response_format,
         fields.append(("prompt", prompt))
     if granularity:
         fields.append(("timestamp_granularities[]", granularity))
-    body, ctype = _multipart(fields, "file", wav_path)
+    body, ctype = _multipart(fields, "file", audio_path)
     try:
         return _request(
             f"{target.base_url.rstrip('/')}/audio/transcriptions", body,
@@ -381,9 +381,9 @@ def _merge_word_splits(segments):
     return merged
 
 
-def transcribe(target, wav_path, language="", prompt="", timeout=300, aborter=None):
+def transcribe(target, audio_path, language="", prompt="", timeout=300, aborter=None):
     data = _transcribe_request(
-        target, wav_path, language, prompt, "json", timeout=timeout, aborter=aborter
+        target, audio_path, language, prompt, "json", timeout=timeout, aborter=aborter
     )
     text = data.get("text") or ""
     if target.provider == "local":
@@ -394,12 +394,12 @@ def transcribe(target, wav_path, language="", prompt="", timeout=300, aborter=No
     return text
 
 
-def transcribe_segments(target, wav_path, language="", prompt="", timeout=300,
+def transcribe_segments(target, audio_path, language="", prompt="", timeout=300,
                         aborter=None):
     """[(start_seconds, end_seconds, text)] using whisper-1's verbose response."""
     data = _transcribe_request(
         target._replace(model=timestamp_model(target.provider, target.model)),
-        wav_path, language, prompt, "verbose_json",
+        audio_path, language, prompt, "verbose_json",
         granularity="segment", timeout=timeout, aborter=aborter,
     )
     segments = data.get("segments") or []

filetranscribe.py

@@ -1,8 +1,15 @@
 """Transcribe an existing audio/video file with the same models.
 
-ffmpeg converts whatever comes in to 16 kHz mono WAV; long files are cut into
-chunks that stay under the API's size limit, then stitched back together with
-their timestamps shifted into place.
+ffmpeg converts whatever comes in to 16 kHz mono WAV, and for a hosted API to
+mp3 on top of that. The upload limit is the only reason a file is ever cut up,
+and uncompressed audio reaches it after ten minutes where mp3 takes an hour.
+
+That is worth the encoder, because a cut is not free. Whisper hears in thirty
+second windows and decides for itself where one cue ends and the next begins; a
+chunk that starts in the middle of a sentence can come back as one cue per
+window, twenty seconds of text at a time, for the whole rest of the chunk. So
+the file is cut as rarely as the limit allows, what is cut overlaps, and
+stitch() drops the half that was heard twice.
 """
 
 import contextlib
@@ -21,7 +28,10 @@ import cleanup
 import ggml
 from i18n import t
 
-CHUNK_SECONDS = 600          # 10 min ≈ 19 MB at 16 kHz mono s16
+UPLOAD_LIMIT = 24 * 1024 * 1024  # the APIs take 25 MB; leave the form its room
+MP3_BITRATE = "48k"          # mono speech at 16 kHz: whisper hears nothing less
+OVERLAP_SECONDS = 30         # a whisper window: how far back a chunk starts
+WAV_CHUNK_SECONDS = 600      # 19 MB, for the caller that uploads the WAV itself
 CLEANUP_CHUNK_CHARS = 12000  # keep each cleanup call comfortably small
 RATE = 16000
 MIN_SUBTITLE_SECONDS = 1.5   # how long a cue with no end time of its own stays up
@@ -88,21 +98,23 @@ class FileTranscriber(QObject):
             wav_path = _to_wav(path, workdir, self._abort)
             self._check()
 
-            chunks = split_wav(wav_path, workdir)
+            target = conf.transcribe_target()
+            self._local = ggml.whisper if target.provider == "local" else None
+            chunks = self._chunks(wav_path, workdir, target, timestamps)
             if len(chunks) > 1:
                 self.progress.emit(t("Splitting into {count} chunks…", count=len(chunks)))
 
-            target = conf.transcribe_target()
-            self._local = ggml.whisper if target.provider == "local" else None
             pieces = []
             segments = []
             for index, (chunk_path, offset) in enumerate(chunks, start=1):
                 self._check()
                 self.progress.emit(
-                    t("Transcribing chunk {index}/{count}…", index=index, count=len(chunks))
+                    t("Transcribing chunk {index}/{count}…",
+                      index=index, count=len(chunks))
+                    if len(chunks) > 1 else t("Transcribing…")
                 )
                 if timestamps:
-                    segments.extend(
+                    segments = stitch(segments, [
                         (start + offset, end + offset, line)
                         for start, end, line in api.transcribe_segments(
                             target,
@@ -111,9 +123,7 @@ class FileTranscriber(QObject):
                             prompt=conf["transcribe_prompt"],
                             aborter=self._abort,
                         )
-                    )
-                    pieces = [f"[{format_timestamp(start)}] {line}"
-                              for start, _, line in segments]
+                    ])
                 else:
                     pieces.append(api.transcribe(
                         target,
@@ -123,6 +133,9 @@ class FileTranscriber(QObject):
                         aborter=self._abort,
                     ))
 
+            if timestamps:
+                pieces = [f"[{format_timestamp(start)}] {line}"
+                          for start, _, line in segments]
             text = "\n".join(pieces) if timestamps else " ".join(pieces)
 
             if do_cleanup and text:
@@ -141,6 +154,28 @@ class FileTranscriber(QObject):
             if workdir:
                 shutil.rmtree(workdir, ignore_errors=True)
 
+    def _chunks(self, wav_path, workdir, target, timestamps):
+        """[(the file to send, its offset in seconds)], one entry where it can be.
+
+        A server on this machine is handed the WAV as it is: nothing is being
+        uploaded, so the encoder would cost quality and buy nothing.
+        """
+        if target.provider == "local":
+            return [(wav_path, 0.0)]
+
+        whole = _to_mp3(wav_path, workdir, "audio.mp3", self._abort)
+        seconds = chunk_seconds(whole, wav_seconds(wav_path))
+        if not seconds:
+            return [(whole, 0.0)]
+
+        # Only a timestamped run can tell what it has already heard, so only it
+        # can afford the overlap that keeps a cue off the cut.
+        self._check()
+        pieces = split_wav(wav_path, workdir, seconds,
+                           OVERLAP_SECONDS if timestamps else 0)
+        return [(_to_mp3(piece, workdir, f"chunk-{index:03d}.mp3", self._abort), offset)
+                for index, (piece, offset) in enumerate(pieces)]
+
     def _cleanup(self, text, timestamps):
         conf = self.conf
         self._local = ggml.llm if cleanup.provider(conf) == "local" else None
@@ -220,9 +255,32 @@ def _reap(proc):
 
 def _to_wav(path, workdir, aborter=None):
     out = os.path.join(workdir, "audio.wav")
+    return _ffmpeg(["-i", path, "-vn", "-ac", "1", "-ar", str(RATE),
+                    "-c:a", "pcm_s16le", out], out, aborter)
+
+
+def _to_mp3(wav_path, workdir, name, aborter=None):
+    """The same audio at a fifth of the size.
+
+    Which is the whole of it: uncompressed, an hour of speech is four uploads
+    and so three cuts, and every cut is a chance of the model losing the thread
+    of where its cues should end. Encoded it is one upload and no cuts. The
+    bitrate is far above what a 16 kHz mono voice has left to lose.
+    """
+    out = os.path.join(workdir, name)
+    try:
+        return _ffmpeg(["-i", wav_path, "-c:a", "libmp3lame", "-b:a", MP3_BITRATE, out],
+                       out, aborter)
+    except api.ApiError:
+        # An ffmpeg built without the encoder, which is rare and not worth
+        # failing over: the WAV transcribes just as well, it only has to be cut
+        # up more often to fit in a request.
+        return wav_path
+
+
+def _ffmpeg(args, out, aborter=None):
     proc = subprocess.Popen(
-        ["ffmpeg", "-nostdin", "-y", "-i", path, "-vn",
-         "-ac", "1", "-ar", str(RATE), "-c:a", "pcm_s16le", out],
+        ["ffmpeg", "-nostdin", "-y", *args],
         stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         text=True,
     )
@@ -242,32 +300,80 @@ def _to_wav(path, workdir, aborter=None):
     return out
 
 
-def split_wav(wav_path, workdir):
-    """[(chunk path, offset in seconds)], a single entry for short files."""
+def wav_seconds(wav_path):
+    with contextlib.closing(wave.open(wav_path, "rb")) as src:
+        return src.getnframes() / (src.getframerate() or RATE)
+
+
+def chunk_seconds(path, duration):
+    """How many seconds of this audio fit in one request, or 0 when all of it does.
+
+    Measured rather than worked out: what an encoder makes of an hour of speech
+    depends on the speech, and the file on disk is the only honest answer.
+    """
+    size = os.path.getsize(path)
+    if size <= UPLOAD_LIMIT or duration <= 0:
+        return 0.0
+    return max(60.0, duration * UPLOAD_LIMIT / size * 0.95)
+
+
+def split_wav(wav_path, workdir, seconds=WAV_CHUNK_SECONDS, overlap=OVERLAP_SECONDS):
+    """[(chunk path, offset in seconds)], a single entry for short files.
+
+    Every chunk but the first starts `overlap` seconds inside the one before it,
+    so the sentence the cut fell in the middle of is heard whole by one of them.
+    stitch() is what drops the telling that was cut short.
+    """
     with contextlib.closing(wave.open(wav_path, "rb")) as src:
         rate = src.getframerate()
         total = src.getnframes()
-        per_chunk = CHUNK_SECONDS * rate
-        if total <= per_chunk:
+        per_chunk = int(seconds * rate)
+        if per_chunk <= 0 or total <= per_chunk:
             return [(wav_path, 0.0)]
 
+        # Half a chunk is the most an overlap can be and still be an overlap.
+        step = per_chunk - int(max(0.0, min(overlap, seconds / 2)) * rate)
         chunks = []
-        index = 0
-        while True:
+        position = 0
+        while position < total:
+            # What is left is shorter than the overlap, so the chunk before this
+            # one already holds all of it.
+            if chunks and total - position <= per_chunk - step:
+                break
+            src.setpos(position)
             frames = src.readframes(per_chunk)
             if not frames:
                 break
-            path = os.path.join(workdir, f"chunk-{index:03d}.wav")
+            path = os.path.join(workdir, f"chunk-{len(chunks):03d}.wav")
             with contextlib.closing(wave.open(path, "wb")) as dst:
                 dst.setnchannels(src.getnchannels())
                 dst.setsampwidth(src.getsampwidth())
                 dst.setframerate(rate)
                 dst.writeframes(frames)
-            chunks.append((path, index * CHUNK_SECONDS))
-            index += 1
+            chunks.append((path, position / rate))
+            position += step
         return chunks
 
 
+def stitch(collected, incoming):
+    """Add a chunk's segments to the ones before it, minus what was heard twice.
+
+    The chunks overlap, so the sentence the cut landed in is in both of them:
+    cut short as the last cue of the chunk before, and whole somewhere in this
+    one. This chunk's telling of it is the one that stands, and the chunk before
+    gives way from wherever that telling begins, so that nothing is said twice
+    and the cues still run forwards.
+    """
+    if not collected:
+        return list(incoming)
+    kept = [segment for segment in incoming if segment[1] > collected[-1][0]]
+    if not kept:
+        return collected
+    seam = kept[0][0]
+    head = [segment for segment in collected if segment[1] <= seam]
+    return (head or collected[:-1]) + kept
+
+
 def split_text(text, timestamps):
     """Break long text into cleanup-sized blocks, never mid-line."""
     if len(text) <= CLEANUP_CHUNK_CHARS:

meeting.py

@@ -170,6 +170,7 @@ class MeetingPipeline(QObject):
             chunk_dir = os.path.join(workdir, speaker)
             os.makedirs(chunk_dir, exist_ok=True)
             chunks = filetranscribe.split_wav(path, chunk_dir)
+            heard = []
             for index, (chunk_path, offset) in enumerate(chunks, start=1):
                 self._check()
                 self._say(t("Transcribing {side}: {index}/{count}…",
@@ -178,12 +179,15 @@ class MeetingPipeline(QObject):
                 # would cost money to be told so, and can invent a sentence.
                 if self._silent(chunk_path):
                     continue
-                segments.extend(
-                    (start + offset, end + offset, text, speaker)
+                # The chunks overlap, so what the cut fell in the middle of is
+                # in two of them; stitch keeps the one that heard it whole.
+                heard = filetranscribe.stitch(heard, [
+                    (start + offset, end + offset, text)
                     for start, end, text in api.transcribe_segments(
                         target, chunk_path, language=language, prompt=hint
                     )
-                )
+                ])
+            segments.extend((start, end, text, speaker) for start, end, text in heard)
         if not segments:
             raise api.ApiError(t("Neither side of the recording had any speech in it."))
 

tests/test_filetranscribe.py

@@ -7,6 +7,7 @@ made up a stamp nobody recorded.
 """
 
 import contextlib
+import os
 import time
 import unittest
 import wave
@@ -133,14 +134,12 @@ class SplitWav(DikteTest):
 
     def test_a_long_file_is_cut_at_the_chunk_length(self):
         path = self.wav(5)
-        with mock.patch.object(ft, "CHUNK_SECONDS", 2):
-            chunks = ft.split_wav(path, self.root)
+        chunks = ft.split_wav(path, self.root, 2, overlap=0)
         self.assertEqual([offset for _, offset in chunks], [0, 2, 4])
 
     def test_the_chunks_add_up_to_the_original(self):
         path = self.wav(5)
-        with mock.patch.object(ft, "CHUNK_SECONDS", 2):
-            chunks = ft.split_wav(path, self.root)
+        chunks = ft.split_wav(path, self.root, 2, overlap=0)
         total = 0
         for chunk_path, _ in chunks:
             with contextlib.closing(wave.open(chunk_path, "rb")) as wav:
@@ -150,10 +149,125 @@ class SplitWav(DikteTest):
 
     def test_the_chunks_do_not_write_over_each_other(self):
         path = self.wav(5)
-        with mock.patch.object(ft, "CHUNK_SECONDS", 2):
-            chunks = ft.split_wav(path, self.root)
+        chunks = ft.split_wav(path, self.root, 2, overlap=0)
         self.assertEqual(len({chunk for chunk, _ in chunks}), len(chunks))
 
+    def test_a_chunk_starts_inside_the_one_before_it(self):
+        """The cut is what makes whisper lose the thread, so nobody hears only
+        one side of it."""
+        path = self.wav(10)
+        chunks = ft.split_wav(path, self.root, 4, overlap=1)
+        self.assertEqual([offset for _, offset in chunks], [0, 3, 6])
+        with contextlib.closing(wave.open(chunks[1][0], "rb")) as wav:
+            self.assertEqual(wav.getnframes(), 4 * 16000)
+
+    def test_an_overlap_is_never_more_than_half_a_chunk(self):
+        path = self.wav(10)
+        chunks = ft.split_wav(path, self.root, 4, overlap=60)
+        self.assertEqual([offset for _, offset in chunks], [0, 2, 4, 6])
+
+    def test_a_tail_the_chunk_before_already_holds_is_not_cut_again(self):
+        path = self.wav(9)
+        chunks = ft.split_wav(path, self.root, 4, overlap=1)
+        # 0-4, 3-7, 6-9: a fourth starting at 9 would be the last second again.
+        self.assertEqual([offset for _, offset in chunks], [0, 3, 6])
+
+
+class Stitch(unittest.TestCase):
+    def test_the_first_chunk_is_taken_as_it_is(self):
+        segments = [(0.0, 1.0, "one"), (1.0, 2.0, "two")]
+        self.assertEqual(ft.stitch([], segments), segments)
+
+    def test_the_cue_the_cut_ran_through_is_replaced(self):
+        collected = [(0.0, 4.0, "a whole sentence"), (4.0, 5.0, "cut in ha")]
+        incoming = [(3.0, 4.0, "sentence"), (4.0, 6.0, "cut in half")]
+        self.assertEqual(ft.stitch(collected, incoming),
+                         [(0.0, 4.0, "a whole sentence"), (4.0, 6.0, "cut in half")])
+
+    def test_the_chunk_before_gives_way_where_the_new_telling_starts(self):
+        """The two chunks put the sentence in different cues; whichever way they
+        fall, nothing is said twice and the cues run forwards."""
+        collected = [(0.0, 3.0, "one"), (3.0, 5.0, "two"), (5.0, 6.0, "three cut")]
+        incoming = [(2.0, 4.5, "one and two"), (4.5, 7.0, "two and three whole")]
+        stitched = ft.stitch(collected, incoming)
+        self.assertEqual(stitched, [(0.0, 3.0, "one"), (4.5, 7.0, "two and three whole")])
+        for before, after in zip(stitched, stitched[1:]):
+            self.assertLessEqual(before[1], after[0])
+
+    def test_a_chunk_with_nothing_in_it_takes_nothing_away(self):
+        collected = [(0.0, 4.0, "one")]
+        self.assertEqual(ft.stitch(collected, []), collected)
+
+    def test_a_chunk_that_heard_only_what_was_already_heard_adds_nothing(self):
+        collected = [(0.0, 4.0, "one"), (4.0, 5.0, "two")]
+        self.assertEqual(ft.stitch(collected, [(1.0, 2.0, "one")]), collected)
+
+
+class ChunkSeconds(DikteTest):
+    def file(self, size):
+        path = self.path("audio.mp3")
+        with open(path, "wb") as fh:
+            fh.write(b"\x00" * size)
+        return path
+
+    def test_a_file_that_fits_is_not_cut_at_all(self):
+        self.assertEqual(ft.chunk_seconds(self.file(1024), 600), 0.0)
+
+    def test_a_file_over_the_limit_is_cut_by_what_it_measured(self):
+        # Twice the limit over an hour, so a little under half an hour fits.
+        seconds = ft.chunk_seconds(self.file(ft.UPLOAD_LIMIT * 2), 3600)
+        self.assertGreater(seconds, 1500)
+        self.assertLess(seconds, 1800)
+
+    def test_a_file_with_no_length_is_left_whole(self):
+        self.assertEqual(ft.chunk_seconds(self.file(ft.UPLOAD_LIMIT * 2), 0), 0.0)
+
+
+class Chunks(DikteTest):
+    """What each provider is handed, and in how many pieces."""
+
+    def setUp(self):
+        super().setUp()
+        self.wav = make_wav(self.path("audio.wav"), silence(3))
+        self.worker = ft.FileTranscriber(self.config())
+
+    def target(self, provider):
+        return api.Target(provider, provider, "key", "https://example.test", "whisper-1")
+
+    def test_a_model_on_this_machine_is_handed_the_wav(self):
+        """Nothing is uploaded, so the encoder would cost quality for nothing."""
+        with mock.patch.object(ft, "_to_mp3") as encode:
+            chunks = self.worker._chunks(self.wav, self.root, self.target("local"), True)
+        self.assertEqual(chunks, [(self.wav, 0.0)])
+        encode.assert_not_called()
+
+    def test_a_hosted_model_is_handed_one_mp3(self):
+        with mock.patch.object(ft, "_to_mp3", side_effect=lambda p, d, name, *a:
+                               make_wav(os.path.join(d, name), silence(1))):
+            chunks = self.worker._chunks(self.wav, self.root,
+                                         self.target("openrouter"), True)
+        self.assertEqual(len(chunks), 1)
+        self.assertTrue(chunks[0][0].endswith("audio.mp3"))
+
+    def test_a_file_too_big_to_upload_is_cut_and_encoded_in_pieces(self):
+        wav = make_wav(self.path("long.wav"), silence(120))
+
+        def encode(path, workdir, name, *args):
+            # The whole file is over the limit; the pieces are not.
+            size = ft.UPLOAD_LIMIT * 2 if name == "audio.mp3" else 1024
+            out = os.path.join(workdir, name)
+            with open(out, "wb") as fh:
+                fh.write(b"\x00" * size)
+            return out
+
+        with mock.patch.object(ft, "_to_mp3", side_effect=encode):
+            chunks = self.worker._chunks(wav, self.root,
+                                         self.target("openrouter"), True)
+        self.assertGreater(len(chunks), 1)
+        self.assertEqual(chunks[0][1], 0.0)
+        for path, _ in chunks:
+            self.assertTrue(path.endswith(".mp3"))
+
 
 class Transcriber(DikteTest):
     """The chain, with ffmpeg and both API calls faked."""
@@ -175,6 +289,8 @@ class Transcriber(DikteTest):
             return make_wav(self.path("converted.wav"), tone(1.0))
 
         with mock.patch.object(ft, "_to_wav", side_effect=to_wav), \
+                mock.patch.object(ft, "_to_mp3",
+                                  side_effect=lambda path, *a, **k: path), \
                 mock.patch.object(ft.shutil, "which", return_value="/usr/bin/ffmpeg"), \
                 mock.patch.object(api, "transcribe",
                                   side_effect=fail or (lambda *a, **k: transcript)), \
@@ -237,6 +353,8 @@ class Transcriber(DikteTest):
     def test_the_request_is_handed_the_stop_to_watch(self):
         worker = ft.FileTranscriber(self.conf)
         with mock.patch.object(ft, "_to_wav", side_effect=lambda *a: self.source), \
+                mock.patch.object(ft, "_to_mp3",
+                                  side_effect=lambda path, *a, **k: path), \
                 mock.patch.object(ft.shutil, "which", return_value="/usr/bin/ffmpeg"), \
                 mock.patch.object(api, "transcribe", return_value="text") as call:
             worker._work(self.source, False, False)