yusufipek/dikte
1
0
Fork 0
mirror of https://github.com/yusufipk/dikte.git synced 2026-09-11 10:56:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Give the memory back when a local model has been sitting unused

Browse Source 
A whisper.cpp or llama.cpp server started for one dictation stayed loaded
until Dikte quit. On this machine that is 1.3 GB of VRAM for large-v3 plus
whatever the cleanup LLM takes, held all day between dictations that last
seconds.

Each Server now carries an idle window. A watcher thread per launch stops the
server once nothing has asked it anything for that long, and the next request
loads it again through serve(), which already starts what is not running.
Settings has one checkbox and one number for both servers, on by default at ten
minutes, and it only appears for a machine that runs a model here. The tray menu
says which models are loaded and offers to unload them now.

Two things the clock alone gets wrong, both held off by a count of requests in
flight:

  * A file or a meeting is one address lookup and then minutes of work, which
    to a clock started at the lookup looks exactly like a model nobody wants.
    api.py and cleanup.py hold the count for the length of the request.

  * The count must survive the start it triggered. cleanup._local takes the
    hold and only then asks for the address, so a cold start happens inside it;
    neither serve() nor _stop_now() resets the count any more.

Unloading by hand runs on the interface's thread, so it asks for the start lock
rather than waiting on it: a model still being read in is refused, the way one
in the middle of a request is, instead of freezing the window for as long as
the load takes.
This commit is contained in:
yusufipek
2026-09-05 12:15:14 +03:00
parent b13b08fc38
commit 825f089fe9
13 changed files with 444 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_api.py
+10
View File
@@ -9,6 +9,7 @@ is blocked on, and a faked urlopen has no socket to cut, so those tests talk to
a server of their own on the loopback interface.
"""
import contextlib
import http.server
import json
import os
@@ -621,6 +622,7 @@ class FakeServer:
self.fails = fails
self.log = log
self.starts = 0
self.held = 0
def serve(self):
self.starts += 1
@@ -628,6 +630,14 @@ class FakeServer:
raise ggml.LocalError(self.fails)
return self.url
@contextlib.contextmanager
def busy(self):
self.held += 1
try:
yield
finally:
self.held -= 1
def error(self):
return self.log
tests/test_config.py
+21
View File
@@ -689,3 +689,24 @@ class ReadyToRun(DikteTest):
self.assertEqual(ggml.whisper.settings()["threads"], 4)
self.assertFalse(ggml.whisper.settings()["gpu"])
self.assertEqual(ggml.llm.settings()["context"], 4096)
def test_the_idle_window_is_in_seconds(self):
conf = self.config(local_idle_unload=True, local_idle_minutes=15)
self.assertEqual(conf.idle_seconds(), 900)
def test_an_unchecked_box_keeps_the_model(self):
conf = self.config(local_idle_unload=False, local_idle_minutes=15)
self.assertEqual(conf.idle_seconds(), 0)
def test_a_window_of_no_minutes_is_still_a_window(self):
"""The spin box will not go below one; a config edited by hand can."""
conf = self.config(local_idle_unload=True, local_idle_minutes=0)
self.assertEqual(conf.idle_seconds(), 60)
def test_both_servers_are_told_the_window(self):
conf = self.config(local_idle_unload=True, local_idle_minutes=3)
self.addCleanup(ggml.llm.set_idle, 0)
self.addCleanup(ggml.whisper.set_idle, 0)
conf.apply_local()
self.assertEqual(ggml.whisper.idle, 180)
self.assertEqual(ggml.llm.idle, 180)
tests/test_ggml.py
+121 -1
View File
@@ -783,7 +783,9 @@ STAND_IN = textwrap.dedent("""
""")
class Servers(Local):
class ServerCase(Local):
"""The stand-in server and the fixture around it, with no tests of its own."""
def setUp(self):
super().setUp()
self.path("data").mkdir(parents=True, exist_ok=True)
@@ -806,6 +808,8 @@ class Servers(Local):
self.addCleanup(made.stop)
return made
class Servers(ServerCase):
def test_a_started_server_hands_back_its_address(self):
server = self.server()
url = server.serve()
@@ -1037,6 +1041,122 @@ class Servers(Local):
self.assertFalse(server.sweep()) # and the pid file went with it
class IdleUnload(ServerCase):
"""Giving the memory back when nothing has asked anything for a while."""
IDLE = 0.3
def setUp(self):
super().setUp()
# The real check runs every five seconds against a window of minutes.
# Both are scaled down here; what is being tested is the decision, and
# nothing in it reads the clock in units of its own.
self.patch_attr(ggml, "IDLE_CHECK_SECONDS", 0.05)
def idle_server(self, seconds=None, **settings):
server = self.server(**settings)
server.set_idle(self.IDLE if seconds is None else seconds)
return server
def wait_for(self, predicate, timeout=5.0):
"""True as soon as `predicate` holds, False once the wait runs out."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate():
return True
time.sleep(0.02)
return False
def test_a_model_nobody_is_using_is_unloaded(self):
server = self.idle_server()
server.serve()
self.assertTrue(self.wait_for(lambda: not server.running))
def test_the_default_is_to_keep_it(self):
"""A server nobody set a window on stays until something stops it."""
server = self.server()
server.serve()
self.assertFalse(self.wait_for(lambda: not server.running, timeout=0.6))
def test_a_window_of_zero_keeps_it_too(self):
server = self.idle_server(0)
server.serve()
self.assertFalse(self.wait_for(lambda: not server.running, timeout=0.6))
def test_a_request_in_flight_holds_the_model(self):
"""A file is one address lookup and then minutes of work: the clock
alone would call that idle and unload it mid-transcription."""
server = self.idle_server()
server.serve()
with server.busy():
self.assertFalse(
self.wait_for(lambda: not server.running, timeout=self.IDLE * 3))
self.assertTrue(self.wait_for(lambda: not server.running))
def test_asking_for_the_address_puts_the_window_back(self):
server = self.idle_server()
first = server.serve()
for _ in range(4):
time.sleep(self.IDLE / 2)
self.assertEqual(server.serve(), first) # never restarted
self.assertTrue(server.running)
def test_the_next_request_loads_it_again(self):
server = self.idle_server()
first = server.serve()
self.assertTrue(self.wait_for(lambda: not server.running))
second = server.serve()
self.assertTrue(server.running)
self.assertNotEqual(second, first) # a new process, a new port
def test_the_watcher_of_a_stopped_server_does_not_touch_the_next_one(self):
server = self.idle_server()
server.serve()
server.stop()
server.set_idle(0)
server.serve()
self.assertFalse(self.wait_for(lambda: not server.running, timeout=0.6))
def test_unloading_by_hand_does_not_wait_for_the_window(self):
server = self.idle_server(0)
server.serve()
self.assertTrue(server.unload())
self.assertFalse(server.running)
def test_a_hold_taken_before_the_start_survives_it(self):
"""The local cleanup takes the hold and only then asks for the address,
so the start it triggers must not be what drops the hold."""
server = self.idle_server()
with server.busy():
server.serve()
self.assertFalse(
self.wait_for(lambda: not server.running, timeout=self.IDLE * 3))
self.assertTrue(self.wait_for(lambda: not server.running))
def test_unloading_is_refused_while_the_model_is_still_loading(self):
"""It runs on the interface's thread, and a start holds its lock for as
long as the load takes: waiting there would freeze the whole window."""
server = self.idle_server(0, extra=["--wait", "0.6"])
thread = threading.Thread(target=server.serve)
thread.start()
try:
began = time.monotonic()
self.assertFalse(server.unload())
self.assertLess(time.monotonic() - began, 0.2)
finally:
thread.join(timeout=10)
def test_unloading_is_refused_while_a_request_is_in_flight(self):
server = self.idle_server(0)
server.serve()
with server.busy():
self.assertFalse(server.unload())
self.assertTrue(server.running)
def test_unloading_nothing_is_not_a_refusal(self):
self.assertTrue(self.server().unload())
class Arguments(Local):
"""What the two command lines say, since neither program is here to say it."""
tests/test_ui.py
+22
View File
@@ -83,6 +83,8 @@ CHANGED = {
"local_llm_gpu": False,
"local_llm_preload": True,
"local_llm_reasoning": "low",
"local_idle_unload": False,
"local_idle_minutes": 45,
"cleanup_prompt": "Only fix the punctuation.",
"file_cleanup_prompt": "Keep the stamps where they are.",
"transcribe_prompt": "Paraşüt, OpenFrame",
@@ -1701,6 +1703,26 @@ class LocalModels(DikteTest):
# Its own thinking box, because the two default to opposite things.
self.assertFalse(window.cleanup_form.isRowVisible(window.cleanup_reasoning))
def test_the_idle_unload_is_offered_to_whoever_runs_a_model_here(self):
for transcriber, cleaner in (("local", "openrouter"),
("openai", "local"),
("local", "local")):
with self.subTest(transcriber=transcriber, cleaner=cleaner):
window = self.window(self.config(transcribe_provider=transcriber,
cleanup_provider=cleaner))
self.assertTrue(window.local_box.isVisibleTo(window))
def test_a_machine_that_runs_neither_is_not_asked_about_memory(self):
window = self.window(self.config(transcribe_provider="openai",
cleanup_provider="openrouter"))
self.assertFalse(window.local_box.isVisibleTo(window))
def test_the_minutes_follow_the_checkbox(self):
window = self.window(self.config(local_idle_unload=False))
self.assertFalse(window.local_idle_minutes.isEnabled())
window.local_idle_unload.setChecked(True)
self.assertTrue(window.local_idle_minutes.isEnabled())
def test_each_cleaner_brings_its_own_model_row_and_no_other(self):
window = self.window(cfg.Config())
rows = {"openrouter": window.cleanup_model_row,