Restart under high swap after unloading resident Ollama models
This commit is contained in:
+15
-23
@@ -357,7 +357,7 @@ class Runtime:
|
|||||||
state = self.state.snapshot()
|
state = self.state.snapshot()
|
||||||
threshold = common.threshold_exceeded(self.config, stats) if stats else False
|
threshold = common.threshold_exceeded(self.config, stats) if stats else False
|
||||||
would_restart = threshold
|
would_restart = threshold
|
||||||
recommendation = "restart" if threshold and not busy else ("defer" if threshold and busy else "none")
|
recommendation = "restart" if threshold else "none"
|
||||||
return {
|
return {
|
||||||
"platform": SYSTEM,
|
"platform": SYSTEM,
|
||||||
"config": {
|
"config": {
|
||||||
@@ -447,30 +447,22 @@ class Runtime:
|
|||||||
self.state.mutate(lambda data: {**data, "pending_since": None})
|
self.state.mutate(lambda data: {**data, "pending_since": None})
|
||||||
return actions
|
return actions
|
||||||
busy = RemoteOllama(self.config).is_busy()
|
busy = RemoteOllama(self.config).is_busy()
|
||||||
if busy:
|
now = common.now_utc()
|
||||||
now = common.now_utc()
|
threshold = True
|
||||||
pending = self.state.snapshot().get("pending_since")
|
pending = self.state.snapshot().get("pending_since")
|
||||||
if pending is None:
|
if pending is None:
|
||||||
self.state.mutate(lambda data: {**data, "pending_since": common.format_iso(now)})
|
self.state.mutate(lambda data: {**data, "pending_since": common.format_iso(now)})
|
||||||
elif common.parse_iso(pending) is not None and (now - common.parse_iso(pending)).total_seconds() >= int(
|
if not self.restart_allowed(force=True)[0]:
|
||||||
self.config["restart_force_after_seconds"]
|
return {"restarted": False, "deferred": True, "message": "restart limit active"}
|
||||||
):
|
# A resident model is not treated as an active request for recovery. `restart()` first
|
||||||
pass
|
# unloads resident models and then restarts the service.
|
||||||
if not self.restart_allowed(force=False)[0]:
|
|
||||||
return {"restarted": False, "deferred": True, "message": "busy and cooldown/limit active"}
|
|
||||||
try:
|
|
||||||
result = self.restart(force=False, reason="threshold")
|
|
||||||
return {"restarted": True, "deferred": False, "message": result}
|
|
||||||
except BusyError:
|
|
||||||
return {"restarted": False, "deferred": True, "message": "busy; restart deferred"}
|
|
||||||
# Not busy.
|
|
||||||
if not self.restart_allowed(force=False)[0]:
|
|
||||||
return {"restarted": False, "deferred": False, "message": self.restart_allowed(force=False)[1]}
|
|
||||||
try:
|
try:
|
||||||
result = self.restart(force=False, reason="threshold")
|
result = self.restart(force=True, reason="threshold")
|
||||||
return {"restarted": True, "deferred": False, "message": result}
|
return {"restarted": True, "deferred": False, "message": result}
|
||||||
except BusyError:
|
except RestartInProgressError as exc:
|
||||||
return {"restarted": False, "deferred": True, "message": "busy; restart deferred"}
|
return {"restarted": False, "deferred": True, "message": str(exc)}
|
||||||
|
except Exception as exc:
|
||||||
|
return {"restarted": False, "deferred": False, "message": "swap check failed: %s" % exc}
|
||||||
|
|
||||||
|
|
||||||
class CooldownError(RuntimeError):
|
class CooldownError(RuntimeError):
|
||||||
|
|||||||
@@ -37,15 +37,15 @@ def test_check_once_restarts_when_threshold_exceeded(tmp_path):
|
|||||||
assert result["restarted"] is True
|
assert result["restarted"] is True
|
||||||
|
|
||||||
|
|
||||||
def test_check_once_defers_when_busy(tmp_path):
|
def test_check_once_restarts_under_load(tmp_path):
|
||||||
config = make_config(tmp_path)
|
config = make_config(tmp_path)
|
||||||
runtime = a.Runtime(config)
|
runtime = a.Runtime(config)
|
||||||
fake_stats = c.SwapStats(100, 40, 60)
|
fake_stats = c.SwapStats(100, 40, 60)
|
||||||
with mock.patch.object(runtime.sampler, "sample", return_value=fake_stats), \
|
with mock.patch.object(runtime.sampler, "sample", return_value=fake_stats), \
|
||||||
mock.patch.object(a.RemoteOllama, "is_busy", return_value=True):
|
mock.patch.object(a.RemoteOllama, "is_busy", return_value=True), \
|
||||||
|
mock.patch.object(runtime.controller, "restart", return_value={"restarted": True}):
|
||||||
result = runtime.check_once()
|
result = runtime.check_once()
|
||||||
assert result["restarted"] is False
|
assert result["restarted"] is True
|
||||||
assert result["deferred"] is True
|
|
||||||
|
|
||||||
|
|
||||||
def test_restart_rejects_busy_without_force(tmp_path):
|
def test_restart_rejects_busy_without_force(tmp_path):
|
||||||
@@ -104,3 +104,13 @@ def test_http_status_and_restart(tmp_path):
|
|||||||
assert data["ok"] is True
|
assert data["ok"] is True
|
||||||
httpd.shutdown()
|
httpd.shutdown()
|
||||||
httpd.server_close()
|
httpd.server_close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_state_store_mutate_persists(tmp_path):
|
||||||
|
state_path = tmp_path / "state.json"
|
||||||
|
store = a.StateStore(str(state_path))
|
||||||
|
assert store.mutate(lambda data: {**data, "last_restart": "2026-10-31T20:00:00+00:00"})[
|
||||||
|
"last_restart"
|
||||||
|
] == "2026-10-31T20:00:00+00:00"
|
||||||
|
reloaded = a.StateStore(str(state_path))
|
||||||
|
assert reloaded.snapshot()["last_restart"] == "2026-10-31T20:00:00+00:00"
|
||||||
|
|||||||
Reference in New Issue
Block a user