Restart under high swap after unloading resident Ollama models

This commit is contained in:
Andrew Emelyaneko
2026-10-31 21:59:18 +03:00
parent 961efd9b40
commit d2415425fc
2 changed files with 29 additions and 27 deletions
+11 -19
View File
@@ -357,7 +357,7 @@ class Runtime:
state = self.state.snapshot()
threshold = common.threshold_exceeded(self.config, stats) if stats else False
would_restart = threshold
recommendation = "restart" if threshold and not busy else ("defer" if threshold and busy else "none")
recommendation = "restart" if threshold else "none"
return {
"platform": SYSTEM,
"config": {
@@ -447,30 +447,22 @@ class Runtime:
self.state.mutate(lambda data: {**data, "pending_since": None})
return actions
busy = RemoteOllama(self.config).is_busy()
if busy:
now = common.now_utc()
threshold = True
pending = self.state.snapshot().get("pending_since")
if pending is None:
self.state.mutate(lambda data: {**data, "pending_since": common.format_iso(now)})
elif common.parse_iso(pending) is not None and (now - common.parse_iso(pending)).total_seconds() >= int(
self.config["restart_force_after_seconds"]
):
pass
if not self.restart_allowed(force=False)[0]:
return {"restarted": False, "deferred": True, "message": "busy and cooldown/limit active"}
if not self.restart_allowed(force=True)[0]:
return {"restarted": False, "deferred": True, "message": "restart limit active"}
# A resident model is not treated as an active request for recovery. `restart()` first
# unloads resident models and then restarts the service.
try:
result = self.restart(force=False, reason="threshold")
result = self.restart(force=True, reason="threshold")
return {"restarted": True, "deferred": False, "message": result}
except BusyError:
return {"restarted": False, "deferred": True, "message": "busy; restart deferred"}
# Not busy.
if not self.restart_allowed(force=False)[0]:
return {"restarted": False, "deferred": False, "message": self.restart_allowed(force=False)[1]}
try:
result = self.restart(force=False, reason="threshold")
return {"restarted": True, "deferred": False, "message": result}
except BusyError:
return {"restarted": False, "deferred": True, "message": "busy; restart deferred"}
except RestartInProgressError as exc:
return {"restarted": False, "deferred": True, "message": str(exc)}
except Exception as exc:
return {"restarted": False, "deferred": False, "message": "swap check failed: %s" % exc}
class CooldownError(RuntimeError):
+14 -4
View File
@@ -37,15 +37,15 @@ def test_check_once_restarts_when_threshold_exceeded(tmp_path):
assert result["restarted"] is True
def test_check_once_defers_when_busy(tmp_path):
def test_check_once_restarts_under_load(tmp_path):
config = make_config(tmp_path)
runtime = a.Runtime(config)
fake_stats = c.SwapStats(100, 40, 60)
with mock.patch.object(runtime.sampler, "sample", return_value=fake_stats), \
mock.patch.object(a.RemoteOllama, "is_busy", return_value=True):
mock.patch.object(a.RemoteOllama, "is_busy", return_value=True), \
mock.patch.object(runtime.controller, "restart", return_value={"restarted": True}):
result = runtime.check_once()
assert result["restarted"] is False
assert result["deferred"] is True
assert result["restarted"] is True
def test_restart_rejects_busy_without_force(tmp_path):
@@ -104,3 +104,13 @@ def test_http_status_and_restart(tmp_path):
assert data["ok"] is True
httpd.shutdown()
httpd.server_close()
def test_state_store_mutate_persists(tmp_path):
state_path = tmp_path / "state.json"
store = a.StateStore(str(state_path))
assert store.mutate(lambda data: {**data, "last_restart": "2026-10-31T20:00:00+00:00"})[
"last_restart"
] == "2026-10-31T20:00:00+00:00"
reloaded = a.StateStore(str(state_path))
assert reloaded.snapshot()["last_restart"] == "2026-10-31T20:00:00+00:00"