Send the audio as mp3, and stop cutting a file that fits in one request

Whisper hears in thirty second windows and decides for itself where one cue
ends and the next begins. A chunk that starts in the middle of a sentence can
answer with one cue per window, twenty seconds of text at a time, for the whole
rest of the chunk: a twenty five minute recording was fine until 20:00, which
was where the second cut fell, and ran on in paragraphs from there. Sending the
same audio in one request instead of three gives cues of two and a half seconds
throughout.

The cuts were only ever there for the upload limit, and we were the ones
walking into it: ffmpeg opened a 24 MB m4a into 48 MB of uncompressed WAV, over
the 25 MB the APIs take, so the file had to be cut every ten minutes. As mp3 it
is 9 MB, and an hour of speech goes in one request. A server on this machine is
still handed the WAV, where nothing is uploaded and the encoder would only cost
quality. How long a chunk may be is now measured from the encoded file rather
than assumed from a bitrate.

Where a file still has to be cut, the chunks overlap by a whisper window and
stitch() drops the telling that was cut short, keeping the one that heard the
sentence whole. Meetings, which upload the WAV itself and so still cut every
ten minutes, get the same stitching.

api.py

@@ -309,7 +309,7 @@ def local_failure(service, server, exc):
                     exc.status)
 
 
-def _transcribe_request(target, wav_path, language, prompt, response_format,
+def _transcribe_request(target, audio_path, language, prompt, response_format,
                         granularity=None, timeout=300, aborter=None):
     if target.provider == "local":
         # The timeouts here are sized for a hosted API, where a slow answer is a
@@ -329,7 +329,7 @@ def _transcribe_request(target, wav_path, language, prompt, response_format,
         fields.append(("prompt", prompt))
     if granularity:
         fields.append(("timestamp_granularities[]", granularity))
-    body, ctype = _multipart(fields, "file", wav_path)
+    body, ctype = _multipart(fields, "file", audio_path)
     try:
         return _request(
             f"{target.base_url.rstrip('/')}/audio/transcriptions", body,
@@ -381,9 +381,9 @@ def _merge_word_splits(segments):
     return merged
 
 
-def transcribe(target, wav_path, language="", prompt="", timeout=300, aborter=None):
+def transcribe(target, audio_path, language="", prompt="", timeout=300, aborter=None):
     data = _transcribe_request(
-        target, wav_path, language, prompt, "json", timeout=timeout, aborter=aborter
+        target, audio_path, language, prompt, "json", timeout=timeout, aborter=aborter
     )
     text = data.get("text") or ""
     if target.provider == "local":
@@ -394,12 +394,12 @@ def transcribe(target, wav_path, language="", prompt="", timeout=300, aborter=No
     return text
 
 
-def transcribe_segments(target, wav_path, language="", prompt="", timeout=300,
+def transcribe_segments(target, audio_path, language="", prompt="", timeout=300,
                         aborter=None):
     """[(start_seconds, end_seconds, text)] using whisper-1's verbose response."""
     data = _transcribe_request(
         target._replace(model=timestamp_model(target.provider, target.model)),
-        wav_path, language, prompt, "verbose_json",
+        audio_path, language, prompt, "verbose_json",
         granularity="segment", timeout=timeout, aborter=aborter,
     )
     segments = data.get("segments") or []

filetranscribe.py

@@ -1,8 +1,15 @@
 """Transcribe an existing audio/video file with the same models.
 
-ffmpeg converts whatever comes in to 16 kHz mono WAV; long files are cut into
+ffmpeg converts whatever comes in to 16 kHz mono WAV, and for a hosted API to
-chunks that stay under the API's size limit, then stitched back together with
+mp3 on top of that. The upload limit is the only reason a file is ever cut up,
-their timestamps shifted into place.
+and uncompressed audio reaches it after ten minutes where mp3 takes an hour.
+
+That is worth the encoder, because a cut is not free. Whisper hears in thirty
+second windows and decides for itself where one cue ends and the next begins; a
+chunk that starts in the middle of a sentence can come back as one cue per
+window, twenty seconds of text at a time, for the whole rest of the chunk. So
+the file is cut as rarely as the limit allows, what is cut overlaps, and
+stitch() drops the half that was heard twice.
 """
 
 import contextlib
@@ -21,7 +28,10 @@ import cleanup
 import ggml
 from i18n import t
 
-CHUNK_SECONDS = 600          # 10 min ≈ 19 MB at 16 kHz mono s16
+UPLOAD_LIMIT = 24 * 1024 * 1024  # the APIs take 25 MB; leave the form its room
+MP3_BITRATE = "48k"          # mono speech at 16 kHz: whisper hears nothing less
+OVERLAP_SECONDS = 30         # a whisper window: how far back a chunk starts
+WAV_CHUNK_SECONDS = 600      # 19 MB, for the caller that uploads the WAV itself
 CLEANUP_CHUNK_CHARS = 12000  # keep each cleanup call comfortably small
 RATE = 16000
 MIN_SUBTITLE_SECONDS = 1.5   # how long a cue with no end time of its own stays up
@@ -88,21 +98,23 @@ class FileTranscriber(QObject):
             wav_path = _to_wav(path, workdir, self._abort)
             self._check()
 
-            chunks = split_wav(wav_path, workdir)
+            target = conf.transcribe_target()
+            self._local = ggml.whisper if target.provider == "local" else None
+            chunks = self._chunks(wav_path, workdir, target, timestamps)
             if len(chunks) > 1:
                 self.progress.emit(t("Splitting into {count} chunks…", count=len(chunks)))
 
-            target = conf.transcribe_target()
-            self._local = ggml.whisper if target.provider == "local" else None
             pieces = []
             segments = []
             for index, (chunk_path, offset) in enumerate(chunks, start=1):
                 self._check()
                 self.progress.emit(
-                    t("Transcribing chunk {index}/{count}…", index=index, count=len(chunks))
+                    t("Transcribing chunk {index}/{count}…",
+                      index=index, count=len(chunks))
+                    if len(chunks) > 1 else t("Transcribing…")
                 )
                 if timestamps:
-                    segments.extend(
+                    segments = stitch(segments, [
                         (start + offset, end + offset, line)
                         for start, end, line in api.transcribe_segments(
                             target,
@@ -111,9 +123,7 @@ class FileTranscriber(QObject):
                             prompt=conf["transcribe_prompt"],
                             aborter=self._abort,
                         )
-                    )
+                    ])
-                    pieces = [f"[{format_timestamp(start)}] {line}"
-                              for start, _, line in segments]
                 else:
                     pieces.append(api.transcribe(
                         target,
@@ -123,6 +133,9 @@ class FileTranscriber(QObject):
                         aborter=self._abort,
                     ))
 
+            if timestamps:
+                pieces = [f"[{format_timestamp(start)}] {line}"
+                          for start, _, line in segments]
             text = "\n".join(pieces) if timestamps else " ".join(pieces)
 
             if do_cleanup and text:
@@ -141,6 +154,28 @@ class FileTranscriber(QObject):
             if workdir:
                 shutil.rmtree(workdir, ignore_errors=True)
 
+    def _chunks(self, wav_path, workdir, target, timestamps):
+        """[(the file to send, its offset in seconds)], one entry where it can be.
+
+        A server on this machine is handed the WAV as it is: nothing is being
+        uploaded, so the encoder would cost quality and buy nothing.
+        """
+        if target.provider == "local":
+            return [(wav_path, 0.0)]
+
+        whole = _to_mp3(wav_path, workdir, "audio.mp3", self._abort)
+        seconds = chunk_seconds(whole, wav_seconds(wav_path))
+        if not seconds:
+            return [(whole, 0.0)]
+
+        # Only a timestamped run can tell what it has already heard, so only it
+        # can afford the overlap that keeps a cue off the cut.
+        self._check()
+        pieces = split_wav(wav_path, workdir, seconds,
+                           OVERLAP_SECONDS if timestamps else 0)
+        return [(_to_mp3(piece, workdir, f"chunk-{index:03d}.mp3", self._abort), offset)
+                for index, (piece, offset) in enumerate(pieces)]
+
     def _cleanup(self, text, timestamps):
         conf = self.conf
         self._local = ggml.llm if cleanup.provider(conf) == "local" else None
@@ -220,9 +255,32 @@ def _reap(proc):
 
 def _to_wav(path, workdir, aborter=None):
     out = os.path.join(workdir, "audio.wav")
+    return _ffmpeg(["-i", path, "-vn", "-ac", "1", "-ar", str(RATE),
+                    "-c:a", "pcm_s16le", out], out, aborter)
+
+
+def _to_mp3(wav_path, workdir, name, aborter=None):
+    """The same audio at a fifth of the size.
+
+    Which is the whole of it: uncompressed, an hour of speech is four uploads
+    and so three cuts, and every cut is a chance of the model losing the thread
+    of where its cues should end. Encoded it is one upload and no cuts. The
+    bitrate is far above what a 16 kHz mono voice has left to lose.
+    """
+    out = os.path.join(workdir, name)
+    try:
+        return _ffmpeg(["-i", wav_path, "-c:a", "libmp3lame", "-b:a", MP3_BITRATE, out],
+                       out, aborter)
+    except api.ApiError:
+        # An ffmpeg built without the encoder, which is rare and not worth
+        # failing over: the WAV transcribes just as well, it only has to be cut
+        # up more often to fit in a request.
+        return wav_path
+
+
+def _ffmpeg(args, out, aborter=None):
     proc = subprocess.Popen(
-        ["ffmpeg", "-nostdin", "-y", "-i", path, "-vn",
+        ["ffmpeg", "-nostdin", "-y", *args],
-         "-ac", "1", "-ar", str(RATE), "-c:a", "pcm_s16le", out],
         stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         text=True,
     )
@@ -242,32 +300,80 @@ def _to_wav(path, workdir, aborter=None):
     return out
 
 
-def split_wav(wav_path, workdir):
+def wav_seconds(wav_path):
-    """[(chunk path, offset in seconds)], a single entry for short files."""
+    with contextlib.closing(wave.open(wav_path, "rb")) as src:
+        return src.getnframes() / (src.getframerate() or RATE)
+
+
+def chunk_seconds(path, duration):
+    """How many seconds of this audio fit in one request, or 0 when all of it does.
+
+    Measured rather than worked out: what an encoder makes of an hour of speech
+    depends on the speech, and the file on disk is the only honest answer.
+    """
+    size = os.path.getsize(path)
+    if size <= UPLOAD_LIMIT or duration <= 0:
+        return 0.0
+    return max(60.0, duration * UPLOAD_LIMIT / size * 0.95)
+
+
+def split_wav(wav_path, workdir, seconds=WAV_CHUNK_SECONDS, overlap=OVERLAP_SECONDS):
+    """[(chunk path, offset in seconds)], a single entry for short files.
+
+    Every chunk but the first starts `overlap` seconds inside the one before it,
+    so the sentence the cut fell in the middle of is heard whole by one of them.
+    stitch() is what drops the telling that was cut short.
+    """
     with contextlib.closing(wave.open(wav_path, "rb")) as src:
         rate = src.getframerate()
         total = src.getnframes()
-        per_chunk = CHUNK_SECONDS * rate
+        per_chunk = int(seconds * rate)
-        if total <= per_chunk:
+        if per_chunk <= 0 or total <= per_chunk:
             return [(wav_path, 0.0)]
 
+        # Half a chunk is the most an overlap can be and still be an overlap.
+        step = per_chunk - int(max(0.0, min(overlap, seconds / 2)) * rate)
         chunks = []
-        index = 0
+        position = 0
-        while True:
+        while position < total:
+            # What is left is shorter than the overlap, so the chunk before this
+            # one already holds all of it.
+            if chunks and total - position <= per_chunk - step:
+                break
+            src.setpos(position)
             frames = src.readframes(per_chunk)
             if not frames:
                 break
-            path = os.path.join(workdir, f"chunk-{index:03d}.wav")
+            path = os.path.join(workdir, f"chunk-{len(chunks):03d}.wav")
             with contextlib.closing(wave.open(path, "wb")) as dst:
                 dst.setnchannels(src.getnchannels())
                 dst.setsampwidth(src.getsampwidth())
                 dst.setframerate(rate)
                 dst.writeframes(frames)
-            chunks.append((path, index * CHUNK_SECONDS))
+            chunks.append((path, position / rate))
-            index += 1
+            position += step
         return chunks
 
 
+def stitch(collected, incoming):
+    """Add a chunk's segments to the ones before it, minus what was heard twice.
+
+    The chunks overlap, so the sentence the cut landed in is in both of them:
+    cut short as the last cue of the chunk before, and whole somewhere in this
+    one. This chunk's telling of it is the one that stands, and the chunk before
+    gives way from wherever that telling begins, so that nothing is said twice
+    and the cues still run forwards.
+    """
+    if not collected:
+        return list(incoming)
+    kept = [segment for segment in incoming if segment[1] > collected[-1][0]]
+    if not kept:
+        return collected
+    seam = kept[0][0]
+    head = [segment for segment in collected if segment[1] <= seam]
+    return (head or collected[:-1]) + kept
+
+
 def split_text(text, timestamps):
     """Break long text into cleanup-sized blocks, never mid-line."""
     if len(text) <= CLEANUP_CHUNK_CHARS:

meeting.py

@@ -170,6 +170,7 @@ class MeetingPipeline(QObject):
             chunk_dir = os.path.join(workdir, speaker)
             os.makedirs(chunk_dir, exist_ok=True)
             chunks = filetranscribe.split_wav(path, chunk_dir)
+            heard = []
             for index, (chunk_path, offset) in enumerate(chunks, start=1):
                 self._check()
                 self._say(t("Transcribing {side}: {index}/{count}…",
@@ -178,12 +179,15 @@ class MeetingPipeline(QObject):
                 # would cost money to be told so, and can invent a sentence.
                 if self._silent(chunk_path):
                     continue
-                segments.extend(
+                # The chunks overlap, so what the cut fell in the middle of is
-                    (start + offset, end + offset, text, speaker)
+                # in two of them; stitch keeps the one that heard it whole.
+                heard = filetranscribe.stitch(heard, [
+                    (start + offset, end + offset, text)
                     for start, end, text in api.transcribe_segments(
                         target, chunk_path, language=language, prompt=hint
                     )
-                )
+                ])
+            segments.extend((start, end, text, speaker) for start, end, text in heard)
         if not segments:
             raise api.ApiError(t("Neither side of the recording had any speech in it."))
 

tests/test_filetranscribe.py

@@ -7,6 +7,7 @@ made up a stamp nobody recorded.
 """
 
 import contextlib
+import os
 import time
 import unittest
 import wave
@@ -133,14 +134,12 @@ class SplitWav(DikteTest):
 
     def test_a_long_file_is_cut_at_the_chunk_length(self):
         path = self.wav(5)
-        with mock.patch.object(ft, "CHUNK_SECONDS", 2):
+        chunks = ft.split_wav(path, self.root, 2, overlap=0)
-            chunks = ft.split_wav(path, self.root)
         self.assertEqual([offset for _, offset in chunks], [0, 2, 4])
 
     def test_the_chunks_add_up_to_the_original(self):
         path = self.wav(5)
-        with mock.patch.object(ft, "CHUNK_SECONDS", 2):
+        chunks = ft.split_wav(path, self.root, 2, overlap=0)
-            chunks = ft.split_wav(path, self.root)
         total = 0
         for chunk_path, _ in chunks:
             with contextlib.closing(wave.open(chunk_path, "rb")) as wav:
@@ -150,10 +149,125 @@ class SplitWav(DikteTest):
 
     def test_the_chunks_do_not_write_over_each_other(self):
         path = self.wav(5)
-        with mock.patch.object(ft, "CHUNK_SECONDS", 2):
+        chunks = ft.split_wav(path, self.root, 2, overlap=0)
-            chunks = ft.split_wav(path, self.root)
         self.assertEqual(len({chunk for chunk, _ in chunks}), len(chunks))
 
+    def test_a_chunk_starts_inside_the_one_before_it(self):
+        """The cut is what makes whisper lose the thread, so nobody hears only
+        one side of it."""
+        path = self.wav(10)
+        chunks = ft.split_wav(path, self.root, 4, overlap=1)
+        self.assertEqual([offset for _, offset in chunks], [0, 3, 6])
+        with contextlib.closing(wave.open(chunks[1][0], "rb")) as wav:
+            self.assertEqual(wav.getnframes(), 4 * 16000)
+
+    def test_an_overlap_is_never_more_than_half_a_chunk(self):
+        path = self.wav(10)
+        chunks = ft.split_wav(path, self.root, 4, overlap=60)
+        self.assertEqual([offset for _, offset in chunks], [0, 2, 4, 6])
+
+    def test_a_tail_the_chunk_before_already_holds_is_not_cut_again(self):
+        path = self.wav(9)
+        chunks = ft.split_wav(path, self.root, 4, overlap=1)
+        # 0-4, 3-7, 6-9: a fourth starting at 9 would be the last second again.
+        self.assertEqual([offset for _, offset in chunks], [0, 3, 6])
+
+
+class Stitch(unittest.TestCase):
+    def test_the_first_chunk_is_taken_as_it_is(self):
+        segments = [(0.0, 1.0, "one"), (1.0, 2.0, "two")]
+        self.assertEqual(ft.stitch([], segments), segments)
+
+    def test_the_cue_the_cut_ran_through_is_replaced(self):
+        collected = [(0.0, 4.0, "a whole sentence"), (4.0, 5.0, "cut in ha")]
+        incoming = [(3.0, 4.0, "sentence"), (4.0, 6.0, "cut in half")]
+        self.assertEqual(ft.stitch(collected, incoming),
+                         [(0.0, 4.0, "a whole sentence"), (4.0, 6.0, "cut in half")])
+
+    def test_the_chunk_before_gives_way_where_the_new_telling_starts(self):
+        """The two chunks put the sentence in different cues; whichever way they
+        fall, nothing is said twice and the cues run forwards."""
+        collected = [(0.0, 3.0, "one"), (3.0, 5.0, "two"), (5.0, 6.0, "three cut")]
+        incoming = [(2.0, 4.5, "one and two"), (4.5, 7.0, "two and three whole")]
+        stitched = ft.stitch(collected, incoming)
+        self.assertEqual(stitched, [(0.0, 3.0, "one"), (4.5, 7.0, "two and three whole")])
+        for before, after in zip(stitched, stitched[1:]):
+            self.assertLessEqual(before[1], after[0])
+
+    def test_a_chunk_with_nothing_in_it_takes_nothing_away(self):
+        collected = [(0.0, 4.0, "one")]
+        self.assertEqual(ft.stitch(collected, []), collected)
+
+    def test_a_chunk_that_heard_only_what_was_already_heard_adds_nothing(self):
+        collected = [(0.0, 4.0, "one"), (4.0, 5.0, "two")]
+        self.assertEqual(ft.stitch(collected, [(1.0, 2.0, "one")]), collected)
+
+
+class ChunkSeconds(DikteTest):
+    def file(self, size):
+        path = self.path("audio.mp3")
+        with open(path, "wb") as fh:
+            fh.write(b"\x00" * size)
+        return path
+
+    def test_a_file_that_fits_is_not_cut_at_all(self):
+        self.assertEqual(ft.chunk_seconds(self.file(1024), 600), 0.0)
+
+    def test_a_file_over_the_limit_is_cut_by_what_it_measured(self):
+        # Twice the limit over an hour, so a little under half an hour fits.
+        seconds = ft.chunk_seconds(self.file(ft.UPLOAD_LIMIT * 2), 3600)
+        self.assertGreater(seconds, 1500)
+        self.assertLess(seconds, 1800)
+
+    def test_a_file_with_no_length_is_left_whole(self):
+        self.assertEqual(ft.chunk_seconds(self.file(ft.UPLOAD_LIMIT * 2), 0), 0.0)
+
+
+class Chunks(DikteTest):
+    """What each provider is handed, and in how many pieces."""
+
+    def setUp(self):
+        super().setUp()
+        self.wav = make_wav(self.path("audio.wav"), silence(3))
+        self.worker = ft.FileTranscriber(self.config())
+
+    def target(self, provider):
+        return api.Target(provider, provider, "key", "https://example.test", "whisper-1")
+
+    def test_a_model_on_this_machine_is_handed_the_wav(self):
+        """Nothing is uploaded, so the encoder would cost quality for nothing."""
+        with mock.patch.object(ft, "_to_mp3") as encode:
+            chunks = self.worker._chunks(self.wav, self.root, self.target("local"), True)
+        self.assertEqual(chunks, [(self.wav, 0.0)])
+        encode.assert_not_called()
+
+    def test_a_hosted_model_is_handed_one_mp3(self):
+        with mock.patch.object(ft, "_to_mp3", side_effect=lambda p, d, name, *a:
+                               make_wav(os.path.join(d, name), silence(1))):
+            chunks = self.worker._chunks(self.wav, self.root,
+                                         self.target("openrouter"), True)
+        self.assertEqual(len(chunks), 1)
+        self.assertTrue(chunks[0][0].endswith("audio.mp3"))
+
+    def test_a_file_too_big_to_upload_is_cut_and_encoded_in_pieces(self):
+        wav = make_wav(self.path("long.wav"), silence(120))
+
+        def encode(path, workdir, name, *args):
+            # The whole file is over the limit; the pieces are not.
+            size = ft.UPLOAD_LIMIT * 2 if name == "audio.mp3" else 1024
+            out = os.path.join(workdir, name)
+            with open(out, "wb") as fh:
+                fh.write(b"\x00" * size)
+            return out
+
+        with mock.patch.object(ft, "_to_mp3", side_effect=encode):
+            chunks = self.worker._chunks(wav, self.root,
+                                         self.target("openrouter"), True)
+        self.assertGreater(len(chunks), 1)
+        self.assertEqual(chunks[0][1], 0.0)
+        for path, _ in chunks:
+            self.assertTrue(path.endswith(".mp3"))
+
 
 class Transcriber(DikteTest):
     """The chain, with ffmpeg and both API calls faked."""
@@ -175,6 +289,8 @@ class Transcriber(DikteTest):
             return make_wav(self.path("converted.wav"), tone(1.0))
 
         with mock.patch.object(ft, "_to_wav", side_effect=to_wav), \
+                mock.patch.object(ft, "_to_mp3",
+                                  side_effect=lambda path, *a, **k: path), \
                 mock.patch.object(ft.shutil, "which", return_value="/usr/bin/ffmpeg"), \
                 mock.patch.object(api, "transcribe",
                                   side_effect=fail or (lambda *a, **k: transcript)), \
@@ -237,6 +353,8 @@ class Transcriber(DikteTest):
     def test_the_request_is_handed_the_stop_to_watch(self):
         worker = ft.FileTranscriber(self.conf)
         with mock.patch.object(ft, "_to_wav", side_effect=lambda *a: self.source), \
+                mock.patch.object(ft, "_to_mp3",
+                                  side_effect=lambda path, *a, **k: path), \
                 mock.patch.object(ft.shutil, "which", return_value="/usr/bin/ffmpeg"), \
                 mock.patch.object(api, "transcribe", return_value="text") as call:
             worker._work(self.source, False, False)