mirror of
https://github.com/yusufipk/dikte.git
synced 2026-09-11 10:56:10 +00:00
yusufipk
395b52d685
Whisper hears in thirty second windows and decides for itself where one cue ends and the next begins. A chunk that starts in the middle of a sentence can answer with one cue per window, twenty seconds of text at a time, for the whole rest of the chunk: a twenty five minute recording was fine until 20:00, which was where the second cut fell, and ran on in paragraphs from there. Sending the same audio in one request instead of three gives cues of two and a half seconds throughout. The cuts were only ever there for the upload limit, and we were the ones walking into it: ffmpeg opened a 24 MB m4a into 48 MB of uncompressed WAV, over the 25 MB the APIs take, so the file had to be cut every ten minutes. As mp3 it is 9 MB, and an hour of speech goes in one request. A server on this machine is still handed the WAV, where nothing is uploaded and the encoder would only cost quality. How long a chunk may be is now measured from the encoded file rather than assumed from a bitrate. Where a file still has to be cut, the chunks overlap by a whisper window and stitch() drops the telling that was cut short, keeping the one that heard the sentence whole. Meetings, which upload the WAV itself and so still cut every ten minutes, get the same stitching.
393 lines
15 KiB
Python
393 lines
15 KiB
Python
"""Transcribe an existing audio/video file with the same models.
|
|
|
|
ffmpeg converts whatever comes in to 16 kHz mono WAV, and for a hosted API to
|
|
mp3 on top of that. The upload limit is the only reason a file is ever cut up,
|
|
and uncompressed audio reaches it after ten minutes where mp3 takes an hour.
|
|
|
|
That is worth the encoder, because a cut is not free. Whisper hears in thirty
|
|
second windows and decides for itself where one cue ends and the next begins; a
|
|
chunk that starts in the middle of a sentence can come back as one cue per
|
|
window, twenty seconds of text at a time, for the whole rest of the chunk. So
|
|
the file is cut as rarely as the limit allows, what is cut overlaps, and
|
|
stitch() drops the half that was heard twice.
|
|
"""
|
|
|
|
import contextlib
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import wave
|
|
|
|
from PyQt6.QtCore import QObject, pyqtSignal
|
|
|
|
import api
|
|
import cleanup
|
|
import ggml
|
|
from i18n import t
|
|
|
|
UPLOAD_LIMIT = 24 * 1024 * 1024 # the APIs take 25 MB; leave the form its room
|
|
MP3_BITRATE = "48k" # mono speech at 16 kHz: whisper hears nothing less
|
|
OVERLAP_SECONDS = 30 # a whisper window: how far back a chunk starts
|
|
WAV_CHUNK_SECONDS = 600 # 19 MB, for the caller that uploads the WAV itself
|
|
CLEANUP_CHUNK_CHARS = 12000 # keep each cleanup call comfortably small
|
|
RATE = 16000
|
|
MIN_SUBTITLE_SECONDS = 1.5 # how long a cue with no end time of its own stays up
|
|
|
|
# The [mm:ss] or [h:mm:ss] prefix a timestamped line starts with.
|
|
STAMP_RE = re.compile(r"^\[(?:(\d+):)?(\d{1,2}):(\d{2})\]\s*")
|
|
|
|
|
|
# What a stopped run comes back with, wherever it was stopped: the request that
|
|
# was cut off raises it from api, and the steps in between raise it themselves.
|
|
Cancelled = api.Aborted
|
|
|
|
|
|
class FileTranscriber(QObject):
|
|
progress = pyqtSignal(str)
|
|
finished = pyqtSignal(str, list) # text, [(start, end, text)] when timestamped
|
|
failed = pyqtSignal(str)
|
|
|
|
def __init__(self, conf, parent=None):
|
|
super().__init__(parent)
|
|
self.conf = conf
|
|
self._thread = None
|
|
self._abort = api.Aborter()
|
|
# The server on this machine the work is with, when it is with one.
|
|
self._local = None
|
|
|
|
@property
|
|
def busy(self):
|
|
return self._thread is not None and self._thread.is_alive()
|
|
|
|
def start(self, path, timestamps, do_cleanup):
|
|
if self.busy:
|
|
return
|
|
self._abort = api.Aborter() # the last one is spent
|
|
self._thread = threading.Thread(
|
|
target=self._work, args=(path, timestamps, do_cleanup), daemon=True
|
|
)
|
|
self._thread.start()
|
|
|
|
def stop(self):
|
|
"""Cut the run off where it stands, rather than at the next step."""
|
|
self._abort.abort()
|
|
# Closing the socket is nothing to a server on this machine: it is a
|
|
# process of ours, and it would grind on to the end of the chunk with
|
|
# nobody left to hand the answer to. Stopping it is what stops the
|
|
# work; the next run starts it again. Killing waits on the process, so
|
|
# not on the thread the window is drawn from.
|
|
local = self._local
|
|
if local is not None:
|
|
threading.Thread(target=local.stop, daemon=True).start()
|
|
|
|
def _check(self):
|
|
self._abort.check()
|
|
|
|
def _work(self, path, timestamps, do_cleanup):
|
|
conf = self.conf
|
|
workdir = None
|
|
try:
|
|
if not shutil.which("ffmpeg"):
|
|
raise api.ApiError(t("ffmpeg not found. Install it to transcribe files."))
|
|
|
|
workdir = tempfile.mkdtemp(prefix="dikte-file-")
|
|
self.progress.emit(t("Converting audio…"))
|
|
wav_path = _to_wav(path, workdir, self._abort)
|
|
self._check()
|
|
|
|
target = conf.transcribe_target()
|
|
self._local = ggml.whisper if target.provider == "local" else None
|
|
chunks = self._chunks(wav_path, workdir, target, timestamps)
|
|
if len(chunks) > 1:
|
|
self.progress.emit(t("Splitting into {count} chunks…", count=len(chunks)))
|
|
|
|
pieces = []
|
|
segments = []
|
|
for index, (chunk_path, offset) in enumerate(chunks, start=1):
|
|
self._check()
|
|
self.progress.emit(
|
|
t("Transcribing chunk {index}/{count}…",
|
|
index=index, count=len(chunks))
|
|
if len(chunks) > 1 else t("Transcribing…")
|
|
)
|
|
if timestamps:
|
|
segments = stitch(segments, [
|
|
(start + offset, end + offset, line)
|
|
for start, end, line in api.transcribe_segments(
|
|
target,
|
|
chunk_path,
|
|
language=conf["language"],
|
|
prompt=conf["transcribe_prompt"],
|
|
aborter=self._abort,
|
|
)
|
|
])
|
|
else:
|
|
pieces.append(api.transcribe(
|
|
target,
|
|
chunk_path,
|
|
language=conf["language"],
|
|
prompt=conf["transcribe_prompt"],
|
|
aborter=self._abort,
|
|
))
|
|
|
|
if timestamps:
|
|
pieces = [f"[{format_timestamp(start)}] {line}"
|
|
for start, _, line in segments]
|
|
text = "\n".join(pieces) if timestamps else " ".join(pieces)
|
|
|
|
if do_cleanup and text:
|
|
self._check()
|
|
self.progress.emit(t("Cleaning up…"))
|
|
text = self._cleanup(text, timestamps)
|
|
|
|
self.finished.emit(text, segments)
|
|
|
|
except Cancelled:
|
|
self.progress.emit(t("Stopped."))
|
|
except (api.ApiError, OSError, subprocess.SubprocessError, wave.Error) as exc:
|
|
self.failed.emit(str(exc))
|
|
finally:
|
|
self._local = None
|
|
if workdir:
|
|
shutil.rmtree(workdir, ignore_errors=True)
|
|
|
|
def _chunks(self, wav_path, workdir, target, timestamps):
|
|
"""[(the file to send, its offset in seconds)], one entry where it can be.
|
|
|
|
A server on this machine is handed the WAV as it is: nothing is being
|
|
uploaded, so the encoder would cost quality and buy nothing.
|
|
"""
|
|
if target.provider == "local":
|
|
return [(wav_path, 0.0)]
|
|
|
|
whole = _to_mp3(wav_path, workdir, "audio.mp3", self._abort)
|
|
seconds = chunk_seconds(whole, wav_seconds(wav_path))
|
|
if not seconds:
|
|
return [(whole, 0.0)]
|
|
|
|
# Only a timestamped run can tell what it has already heard, so only it
|
|
# can afford the overlap that keeps a cue off the cut.
|
|
self._check()
|
|
pieces = split_wav(wav_path, workdir, seconds,
|
|
OVERLAP_SECONDS if timestamps else 0)
|
|
return [(_to_mp3(piece, workdir, f"chunk-{index:03d}.mp3", self._abort), offset)
|
|
for index, (piece, offset) in enumerate(pieces)]
|
|
|
|
def _cleanup(self, text, timestamps):
|
|
conf = self.conf
|
|
self._local = ggml.llm if cleanup.provider(conf) == "local" else None
|
|
prompt = conf.cleanup_prompt(with_timestamps=timestamps, subtitles=True)
|
|
out = []
|
|
for block in split_text(text, timestamps):
|
|
self._check()
|
|
out.append(cleanup.run(block, conf, prompt, aborter=self._abort))
|
|
return ("\n" if timestamps else "\n\n").join(out)
|
|
|
|
|
|
def format_timestamp(seconds):
|
|
seconds = int(seconds)
|
|
hours, rest = divmod(seconds, 3600)
|
|
minutes, secs = divmod(rest, 60)
|
|
return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes:02d}:{secs:02d}"
|
|
|
|
|
|
def srt_timestamp(seconds):
|
|
millis = int(round(max(seconds, 0.0) * 1000))
|
|
hours, rest = divmod(millis, 3600000)
|
|
minutes, rest = divmod(rest, 60000)
|
|
secs, millis = divmod(rest, 1000)
|
|
return f"{hours:02d}:{minutes:02d}:{secs:02d},{millis:03d}"
|
|
|
|
|
|
def to_srt(text, segments):
|
|
"""Turn the timestamped transcript into SRT cues.
|
|
|
|
The text is the authority on wording, so cleanup edits survive; the segments
|
|
are the authority on timing. They meet at the [mm:ss] prefix, which cleanup
|
|
is told to leave alone: a line's whole-second stamp finds the segment it came
|
|
from, and with it the fractional start and the end time whisper reported. A
|
|
line whose stamp finds nothing runs until the next line starts.
|
|
"""
|
|
cues = []
|
|
for line in text.splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
match = STAMP_RE.match(line)
|
|
body = line[match.end():].strip() if match else line
|
|
if not match:
|
|
if cues and body: # a wrapped line belongs to the cue above it
|
|
cues[-1][2] += " " + body
|
|
continue
|
|
if not body:
|
|
continue
|
|
hours, minutes, secs = (int(g or 0) for g in match.groups())
|
|
cues.append([hours * 3600 + minutes * 60 + secs, None, body])
|
|
|
|
timing = {}
|
|
for start, end, _ in segments:
|
|
timing.setdefault(int(start), (start, end))
|
|
for cue in cues:
|
|
cue[0], cue[1] = timing.get(cue[0], (float(cue[0]), 0.0))
|
|
for index, cue in enumerate(cues):
|
|
following = cues[index + 1][0] if index + 1 < len(cues) else 0.0
|
|
if following > cue[0]:
|
|
cue[1] = min(cue[1], following) if cue[1] > cue[0] else following
|
|
elif cue[1] <= cue[0]:
|
|
cue[1] = cue[0] + MIN_SUBTITLE_SECONDS
|
|
|
|
blocks = [
|
|
f"{number}\n{srt_timestamp(start)} --> {srt_timestamp(end)}\n{body}"
|
|
for number, (start, end, body) in enumerate(cues, start=1)
|
|
]
|
|
return "\n\n".join(blocks) + "\n" if blocks else ""
|
|
|
|
|
|
def _reap(proc):
|
|
"""Leave nothing running behind a conversion that did not finish."""
|
|
if proc.poll() is None:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def _to_wav(path, workdir, aborter=None):
|
|
out = os.path.join(workdir, "audio.wav")
|
|
return _ffmpeg(["-i", path, "-vn", "-ac", "1", "-ar", str(RATE),
|
|
"-c:a", "pcm_s16le", out], out, aborter)
|
|
|
|
|
|
def _to_mp3(wav_path, workdir, name, aborter=None):
|
|
"""The same audio at a fifth of the size.
|
|
|
|
Which is the whole of it: uncompressed, an hour of speech is four uploads
|
|
and so three cuts, and every cut is a chance of the model losing the thread
|
|
of where its cues should end. Encoded it is one upload and no cuts. The
|
|
bitrate is far above what a 16 kHz mono voice has left to lose.
|
|
"""
|
|
out = os.path.join(workdir, name)
|
|
try:
|
|
return _ffmpeg(["-i", wav_path, "-c:a", "libmp3lame", "-b:a", MP3_BITRATE, out],
|
|
out, aborter)
|
|
except api.ApiError:
|
|
# An ffmpeg built without the encoder, which is rare and not worth
|
|
# failing over: the WAV transcribes just as well, it only has to be cut
|
|
# up more often to fit in a request.
|
|
return wav_path
|
|
|
|
|
|
def _ffmpeg(args, out, aborter=None):
|
|
proc = subprocess.Popen(
|
|
["ffmpeg", "-nostdin", "-y", *args],
|
|
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
text=True,
|
|
)
|
|
# A two hour film is a minute of ffmpeg, which is a minute of a Stop button
|
|
# doing nothing unless the abort reaches the process itself.
|
|
with contextlib.ExitStack() as stack:
|
|
stack.callback(_reap, proc)
|
|
if aborter is not None:
|
|
stack.enter_context(aborter.holding(proc.kill))
|
|
_stdout, stderr = proc.communicate()
|
|
if aborter is not None:
|
|
aborter.check()
|
|
if proc.returncode != 0 or not os.path.exists(out):
|
|
tail = (stderr or "").strip().splitlines()
|
|
raise api.ApiError(t("Could not read the file: {error}",
|
|
error=tail[-1] if tail else proc.returncode))
|
|
return out
|
|
|
|
|
|
def wav_seconds(wav_path):
|
|
with contextlib.closing(wave.open(wav_path, "rb")) as src:
|
|
return src.getnframes() / (src.getframerate() or RATE)
|
|
|
|
|
|
def chunk_seconds(path, duration):
|
|
"""How many seconds of this audio fit in one request, or 0 when all of it does.
|
|
|
|
Measured rather than worked out: what an encoder makes of an hour of speech
|
|
depends on the speech, and the file on disk is the only honest answer.
|
|
"""
|
|
size = os.path.getsize(path)
|
|
if size <= UPLOAD_LIMIT or duration <= 0:
|
|
return 0.0
|
|
return max(60.0, duration * UPLOAD_LIMIT / size * 0.95)
|
|
|
|
|
|
def split_wav(wav_path, workdir, seconds=WAV_CHUNK_SECONDS, overlap=OVERLAP_SECONDS):
|
|
"""[(chunk path, offset in seconds)], a single entry for short files.
|
|
|
|
Every chunk but the first starts `overlap` seconds inside the one before it,
|
|
so the sentence the cut fell in the middle of is heard whole by one of them.
|
|
stitch() is what drops the telling that was cut short.
|
|
"""
|
|
with contextlib.closing(wave.open(wav_path, "rb")) as src:
|
|
rate = src.getframerate()
|
|
total = src.getnframes()
|
|
per_chunk = int(seconds * rate)
|
|
if per_chunk <= 0 or total <= per_chunk:
|
|
return [(wav_path, 0.0)]
|
|
|
|
# Half a chunk is the most an overlap can be and still be an overlap.
|
|
step = per_chunk - int(max(0.0, min(overlap, seconds / 2)) * rate)
|
|
chunks = []
|
|
position = 0
|
|
while position < total:
|
|
# What is left is shorter than the overlap, so the chunk before this
|
|
# one already holds all of it.
|
|
if chunks and total - position <= per_chunk - step:
|
|
break
|
|
src.setpos(position)
|
|
frames = src.readframes(per_chunk)
|
|
if not frames:
|
|
break
|
|
path = os.path.join(workdir, f"chunk-{len(chunks):03d}.wav")
|
|
with contextlib.closing(wave.open(path, "wb")) as dst:
|
|
dst.setnchannels(src.getnchannels())
|
|
dst.setsampwidth(src.getsampwidth())
|
|
dst.setframerate(rate)
|
|
dst.writeframes(frames)
|
|
chunks.append((path, position / rate))
|
|
position += step
|
|
return chunks
|
|
|
|
|
|
def stitch(collected, incoming):
|
|
"""Add a chunk's segments to the ones before it, minus what was heard twice.
|
|
|
|
The chunks overlap, so the sentence the cut landed in is in both of them:
|
|
cut short as the last cue of the chunk before, and whole somewhere in this
|
|
one. This chunk's telling of it is the one that stands, and the chunk before
|
|
gives way from wherever that telling begins, so that nothing is said twice
|
|
and the cues still run forwards.
|
|
"""
|
|
if not collected:
|
|
return list(incoming)
|
|
kept = [segment for segment in incoming if segment[1] > collected[-1][0]]
|
|
if not kept:
|
|
return collected
|
|
seam = kept[0][0]
|
|
head = [segment for segment in collected if segment[1] <= seam]
|
|
return (head or collected[:-1]) + kept
|
|
|
|
|
|
def split_text(text, timestamps):
|
|
"""Break long text into cleanup-sized blocks, never mid-line."""
|
|
if len(text) <= CLEANUP_CHUNK_CHARS:
|
|
return [text]
|
|
separator = "\n" if timestamps else " "
|
|
blocks, current = [], ""
|
|
for part in text.split(separator):
|
|
candidate = f"{current}{separator}{part}" if current else part
|
|
if len(candidate) > CLEANUP_CHUNK_CHARS and current:
|
|
blocks.append(current)
|
|
current = part
|
|
else:
|
|
current = candidate
|
|
if current:
|
|
blocks.append(current)
|
|
return blocks
|