Add ollama-plagin-swap remote swap control plugin

This commit is contained in:
Andrew Emelyaneko
2026-10-31 19:27:59 +03:00
commit be7932441c
13 changed files with 1549 additions and 0 deletions
+5
View File
@@ -0,0 +1,5 @@
{
"dependencies": {
"@opencode-ai/plugin": "latest"
}
}
+126
View File
@@ -0,0 +1,126 @@
// ollama-plagin-swap: low-overhead remote Ollama swap monitor for OpenCode.
const fs = require("node:fs")
const os = require("node:os")
const path = require("node:path")
const CLIENT_CONFIG = path.join(os.homedir(), ".config", "ollama-plagin-swap", "clients.json")
const STARTED_KEY = Symbol.for("ollamaPlaginSwapStarted")
function readClientConfig() {
try {
const raw = fs.readFileSync(CLIENT_CONFIG, "utf8")
const parsed = JSON.parse(raw)
if (Array.isArray(parsed?.controls)) return parsed
if (Array.isArray(parsed)) return { controls: parsed, interval_seconds: 60 }
return { controls: [], interval_seconds: 60 }
} catch {
return { controls: [], interval_seconds: 60 }
}
}
function inferControls(ctx) {
const configured = readClientConfig()
const controls = [...(configured.controls || [])]
const token = process.env.OLLAMA_SWAP_TOKEN || null
const interval = Number(configured.interval_seconds || process.env.OLLAMA_SWAP_INTERVAL || 60)
// Infer agent endpoints from OpenCode providers that point at an Ollama server.
try {
const configPath = path.join(ctx.directory || process.cwd(), "opencode.json")
const raw = fs.readFileSync(configPath, "utf8")
const config = JSON.parse(raw)
for (const provider of Object.values(config?.provider || {})) {
const baseURL = provider?.options?.baseURL
if (typeof baseURL !== "string") continue
const match = baseURL.match(/^https?:\/\/([^:/]+)(?::\d+)?/)
if (!match) continue
const host = match[1]
if (controls.some((item) => item.agent_url?.includes(host))) continue
controls.push({
name: provider?.name || host,
host,
agent_url: "http://" + host + ":8731",
token,
ollama_base_url: baseURL.replace(/\/v1\/?$/, ""),
})
}
} catch {
// OpenCode config is optional; explicit clients.json is enough.
}
return { controls, interval_seconds: interval > 0 ? interval : 60 }
}
async function fetchStatus(control) {
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(), 5000)
try {
const response = await fetch(control.agent_url.replace(/\/$/, "") + "/status", {
headers: control.token ? { Authorization: "Bearer " + control.token } : {},
signal: controller.signal,
})
if (!response.ok) throw new Error("HTTP " + response.status)
return await response.json()
} finally {
clearTimeout(timer)
}
}
export const OllamaSwapPlugin = async (ctx) => {
if (globalThis[STARTED_KEY]) {
return {}
}
globalThis[STARTED_KEY] = true
const { controls, interval_seconds } = inferControls(ctx)
async function pollOnce(client) {
for (const control of controls) {
try {
const status = await fetchStatus(control)
const summary =
control.name +
": swap " +
(status?.swap?.swap_used_mb ?? "?") +
"/" +
(status?.swap?.swap_total_mb ?? "?") +
" MB (" +
(status?.swap?.swap_percent ?? "?") +
"%), recommendation=" +
(status?.recommendation ?? "unknown")
await client.app.log({
body: {
service: "ollama-plagin-swap",
level: status?.threshold_exceeded ? "warn" : "info",
message: summary,
extra: { control: control.name, status },
},
})
} catch (error) {
await client.app.log({
body: {
service: "ollama-plagin-swap",
level: "warn",
message: "agent unreachable: " + control.name + " (" + String(error) + ")",
extra: { control: control.name },
},
})
}
}
}
const timer = setInterval(() => {
pollOnce(ctx.client).catch(() => {})
}, interval_seconds * 1000)
return {
event: async ({ event }) => {
if (event.type === "server.connected") {
await pollOnce(ctx.client)
}
if (event.type === "session.deleted") {
clearInterval(timer)
}
},
}
}
+99
View File
@@ -0,0 +1,99 @@
import { type Plugin, tool } from "@opencode-ai/plugin"
import { readFile } from "node:fs/promises"
import { homedir } from "node:os"
import path from "node:path"
interface Control {
name: string
agent_url: string
token?: string | null
}
async function controls() {
const file = path.join(homedir(), ".config", "ollama-plagin-swap", "clients.json")
try {
const parsed = JSON.parse(await readFile(file, "utf8"))
return Array.isArray(parsed?.controls) ? parsed.controls : []
} catch {
return []
}
}
async function request(control: Control, method: string, pathname: string, body?: unknown) {
const response = await fetch(control.agent_url.replace(/\/$/, "") + pathname, {
method,
headers: {
...(control.token ? { Authorization: "Bearer " + control.token } : {}),
...(body ? { "Content-Type": "application/json" } : {}),
},
body: body ? JSON.stringify(body) : undefined,
signal: AbortSignal.timeout(8000),
})
const text = await response.text()
let parsed: unknown
try {
parsed = JSON.parse(text)
} catch {
parsed = text
}
if (!response.ok) throw new Error(control.name + ": HTTP " + response.status + " " + text)
return parsed
}
export const OllamaSwapPlugin: Plugin = async (ctx) => {
return {
event: async ({ event }) => {
if (event.type !== "server.connected") return
for (const control of await controls()) {
try {
const status = await request(control, "GET", "/status")
await ctx.client.app.log({
body: {
service: "ollama-plagin-swap",
level: status?.threshold_exceeded ? "warn" : "info",
message: `${control.name}: swap ${status?.swap?.swap_used_mb}/${status?.swap?.swap_total_mb} MB (${status?.swap?.swap_percent}%), recommendation=${status?.recommendation}`,
extra: { control: control.name, status },
},
})
} catch (error) {
await ctx.client.app.log({
body: {
service: "ollama-plagin-swap",
level: "warn",
message: `${control.name}: agent unreachable (${String(error)})`,
},
})
}
}
},
tool: {
ollama_swap_status: tool({
description: "Read Ollama swap status from configured remote agents",
args: {},
async execute() {
const result = []
for (const control of await controls()) {
try {
result.push({ name: control.name, status: await request(control, "GET", "/status") })
} catch (error) {
result.push({ name: control.name, error: String(error) })
}
}
return JSON.stringify(result, null, 2)
},
}),
ollama_swap_restart: tool({
description: "Restart a remote Ollama service after its swap threshold is exceeded",
args: {
name: tool.schema.string().describe("Agent name from clients.json"),
force: tool.schema.boolean().optional().describe("Force restart even if models are active"),
},
async execute(args) {
const control = (await controls()).find((item) => item.name === args.name)
if (!control) throw new Error("unknown agent: " + args.name)
return JSON.stringify(await request(control, "POST", "/restart", { force: Boolean(args.force), reason: "opencode" }), null, 2)
},
}),
},
}
}
+129
View File
@@ -0,0 +1,129 @@
# ollama-plagin-swap
Удалённый контроль swap для серверов Ollama, используемых через API OpenCode.
## Что устанавливается
- `src/ollama_swap_agent.py` и `src/ollama_swap_common.py`: Python-агент для macOS и Ubuntu.
- `.opencode/plugins/ollama-swap.js`: клиентский плагин OpenCode.
- `.opencode/plugins/ollama-swap.ts`: необязательный TypeScript-плагин с пользовательскими инструментами OpenCode.
- Агент предоставляет `GET /health`, `GET /status` и `POST /restart`.
## Установка клиента на текущий macOS
```sh
./scripts/install-client.sh
```
Устанавливаются файлы:
```text
~/.config/opencode/plugins/ollama-swap.js
~/.config/ollama-plagin-swap/clients.json
```
Отредактируйте `clients.json` и укажите URL и токен удалённого агента Ollama.
## Установка агента на сервере Ollama
Подключитесь по SSH к серверу `10.8.0.62` под пользователем с правами администратора/root. Пример:
```sh
ssh admin@10.8.0.62
```
### Сервер Ollama на macOS
Выполните на сервере Ollama:
```sh
mkdir -p ~/src/ollama-plagin-swap
# Сначала скопируйте файлы репозитория на сервер, затем:
cd ~/src/ollama-plagin-swap
./scripts/install-agent.sh
```
Установщик создаёт файлы:
```text
~/.local/share/ollama-plagin-swap/ollama_swap_agent.py
~/.local/share/ollama-plagin-swap/ollama_swap_common.py
~/.config/ollama-plagin-swap/config.json
~/Library/LaunchAgents/com.ollama.swap-agent.plist
```
Измените токен в `~/.config/ollama-plagin-swap/config.json`, затем перезапустите агент:
```sh
launchctl unload ~/Library/LaunchAgents/com.ollama.swap-agent.plist 2>/dev/null || true
launchctl load -w ~/Library/LaunchAgents/com.ollama.swap-agent.plist
```
Сам сервис Ollama агент перезапускает командами:
```sh
launchctl unload ~/Library/LaunchAgents/com.ollama.serve.plist 2>/dev/null
launchctl load -w ~/Library/LaunchAgents/com.ollama.serve.plist
```
### Сервер Ollama на Ubuntu
Выполните на сервере Ollama под root:
```sh
cd /opt/ollama-plagin-swap
sudo ./scripts/install-agent.sh
```
Установщик создаёт файлы:
```text
/opt/ollama-plagin-swap/ollama_swap_agent.py
/opt/ollama-plagin-swap/ollama_swap_common.py
/etc/ollama-plagin-swap/config.json
/etc/systemd/system/ollama-swap-agent.service
```
Измените `/etc/ollama-plagin-swap/config.json`, затем выполните:
```sh
sudo systemctl daemon-reload
sudo systemctl enable --now ollama-swap-agent
```
При необходимости откройте доступ с клиентского IP:
```sh
sudo ufw allow from <ip-клиента-opencode> to any port 8731 proto tcp
```
Сам сервис Ollama агент перезапускает командой:
```sh
sudo systemctl restart ollama
```
## Проверка агента на сервере
```sh
curl -sS -H "Authorization: Bearer <token>" http://10.8.0.62:8731/health
curl -sS -H "Authorization: Bearer <token>" http://10.8.0.62:8731/status
```
Ручной запрос на перезапуск:
```sh
curl -sS -X POST \
-H "Authorization: Bearer <token>" \
-H "Content-Type: application/json" \
-d '{"reason":"manual","force":false}' \
http://10.8.0.62:8731/restart
```
## Локальные проверки перед установкой
```sh
/usr/bin/python3 -m py_compile src/*.py tests/*.py
/opt/homebrew/bin/pytest -q
node --check .opencode/plugins/ollama-swap.js
```
+12
View File
@@ -0,0 +1,12 @@
{
"interval_seconds": 60,
"controls": [
{
"name": "ollama-mac-62",
"host": "10.8.0.62",
"agent_url": "http://10.8.0.62:8731",
"token": "replace-with-agent-token",
"ollama_base_url": "http://10.8.0.62:11434"
}
]
}
+15
View File
@@ -0,0 +1,15 @@
{
"bind_host": "10.8.0.62",
"port": 8731,
"token": "replace-with-strong-token",
"allow_insecure": false,
"poll_interval_seconds": 60,
"swap_threshold_percent": 25,
"cooldown_seconds": 600,
"max_restarts_per_hour": 2,
"restart_force_after_seconds": 300,
"health_url": "http://127.0.0.1:11434/api/tags",
"ollama_ps_url": "http://127.0.0.1:11434/api/ps",
"launch_agent_plist": "~/Library/LaunchAgents/com.ollama.serve.plist",
"linux_service": "ollama"
}
Executable
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
"${ROOT}/scripts/install-agent.sh"
"${ROOT}/scripts/install-client.sh"
echo "ollama-plagin-swap installation complete."
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
OS="$(uname -s)"
PYTHON="$(command -v python3 || true)"
if [ -z "${PYTHON}" ]; then
echo "python3 is required" >&2
exit 1
fi
case "${OS}" in
Darwin)
APP_DIR="${HOME}/.local/share/ollama-plagin-swap"
CONFIG_DIR="${HOME}/.config/ollama-plagin-swap"
STATE_DIR="${HOME}/.local/state/ollama-plagin-swap"
AGENT="${APP_DIR}/ollama_swap_agent.py"
COMMON="${APP_DIR}/ollama_swap_common.py"
CONFIG="${CONFIG_DIR}/config.json"
PLIST="${HOME}/Library/LaunchAgents/com.ollama.swap-agent.plist"
mkdir -p "${APP_DIR}" "${CONFIG_DIR}" "${STATE_DIR}" "${HOME}/Library/LaunchAgents"
install -m 0755 "${ROOT}/src/ollama_swap_agent.py" "${AGENT}"
install -m 0755 "${ROOT}/src/ollama_swap_common.py" "${COMMON}"
if [ ! -f "${CONFIG}" ]; then
cat > "${CONFIG}" <<'JSON'
{
"bind_host": "0.0.0.0",
"port": 8731,
"token": "replace-with-strong-token",
"allow_insecure": false,
"poll_interval_seconds": 60,
"swap_threshold_percent": 25,
"cooldown_seconds": 600,
"max_restarts_per_hour": 2,
"restart_force_after_seconds": 300,
"health_url": "http://127.0.0.1:11434/api/tags",
"ollama_ps_url": "http://127.0.0.1:11434/api/ps",
"launch_agent_plist": "~/Library/LaunchAgents/com.ollama.serve.plist",
"linux_service": "ollama"
}
JSON
fi
cat > "${PLIST}" <<PLIST
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.ollama.swap-agent</string>
<key>ProgramArguments</key>
<array>
<string>${PYTHON}</string>
<string>${AGENT}</string>
<string>serve</string>
<string>--config</string>
<string>${CONFIG}</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
<key>StandardOutPath</key>
<string>${STATE_DIR}/agent.log</string>
<key>StandardErrorPath</key>
<string>${STATE_DIR}/agent.log</string>
</dict>
</plist>
PLIST
launchctl unload "${PLIST}" >/dev/null 2>&1 || true
launchctl load -w "${PLIST}"
echo "Installed and started macOS agent."
;;
Linux)
if [ "$(id -u)" -ne 0 ]; then
echo "Run this installer as root on Ubuntu (for example: sudo $0)" >&2
exit 1
fi
APP_DIR="/opt/ollama-plagin-swap"
CONFIG_DIR="/etc/ollama-plagin-swap"
STATE_DIR="/var/lib/ollama-plagin-swap"
AGENT="${APP_DIR}/ollama_swap_agent.py"
COMMON="${APP_DIR}/ollama_swap_common.py"
CONFIG="${CONFIG_DIR}/config.json"
UNIT="/etc/systemd/system/ollama-swap-agent.service"
mkdir -p "${APP_DIR}" "${CONFIG_DIR}" "${STATE_DIR}"
install -m 0755 "${ROOT}/src/ollama_swap_agent.py" "${AGENT}"
install -m 0755 "${ROOT}/src/ollama_swap_common.py" "${COMMON}"
if [ ! -f "${CONFIG}" ]; then
cat > "${CONFIG}" <<'JSON'
{
"bind_host": "0.0.0.0",
"port": 8731,
"token": "replace-with-strong-token",
"allow_insecure": false,
"poll_interval_seconds": 60,
"swap_threshold_percent": 25,
"cooldown_seconds": 600,
"max_restarts_per_hour": 2,
"restart_force_after_seconds": 300,
"health_url": "http://127.0.0.1:11434/api/tags",
"ollama_ps_url": "http://127.0.0.1:11434/api/ps",
"launch_agent_plist": "~/Library/LaunchAgents/com.ollama.serve.plist",
"linux_service": "ollama"
}
JSON
fi
cat > "${UNIT}" <<UNIT
[Unit]
Description=ollama-plagin-swap remote agent
After=network-online.target ollama.service
Wants=network-online.target
[Service]
Type=simple
ExecStart=${PYTHON} ${AGENT} serve --config ${CONFIG}
Restart=always
RestartSec=5
User=root
StandardOutput=append:${STATE_DIR}/agent.log
StandardError=append:${STATE_DIR}/agent.log
[Install]
WantedBy=multi-user.target
UNIT
systemctl daemon-reload
systemctl enable --now ollama-swap-agent
echo "Installed and started Ubuntu agent."
;;
*)
echo "Unsupported OS: ${OS}" >&2
exit 1
;;
esac
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
CONFIG_DIR="${HOME}/.config/ollama-plagin-swap"
OPENCODE_PLUGIN_DIR="${HOME}/.config/opencode/plugins"
CLIENT_CONFIG="${CONFIG_DIR}/clients.json"
mkdir -p "${CONFIG_DIR}" "${OPENCODE_PLUGIN_DIR}"
install -m 0755 "${ROOT}/.opencode/plugins/ollama-swap.js" "${OPENCODE_PLUGIN_DIR}/ollama-swap.js"
if [ ! -f "${CLIENT_CONFIG}" ]; then
cat > "${CLIENT_CONFIG}" <<'JSON'
{
"interval_seconds": 60,
"controls": [
{
"name": "ollama-remote",
"host": "10.8.0.62",
"agent_url": "http://10.8.0.62:8731",
"token": "replace-with-agent-token",
"ollama_base_url": "http://10.8.0.62:11434"
}
]
}
JSON
fi
echo "Installed OpenCode client plugin to ${OPENCODE_PLUGIN_DIR}/ollama-swap.js"
echo "Edit agent endpoints and tokens in ${CLIENT_CONFIG}"
+564
View File
@@ -0,0 +1,564 @@
#!/usr/bin/env python3
"""Remote swap-control agent for ollama-plagin-swap."""
from __future__ import annotations
import argparse
import http.server
import json
import os
import platform
import socketserver
import subprocess
import threading
import time
import urllib.error
import urllib.request
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
import ollama_swap_common as common
SYSTEM = platform.system().lower()
HTTP_OK = 200
class StateStore:
def __init__(self, path: str):
self.path = Path(path).expanduser()
self.lock = threading.Lock()
self.data = {
"last_restart": None,
"restart_times": [],
"pending_since": None,
"restart_in_progress": False,
}
self._load()
def _load(self) -> None:
try:
if self.path.exists():
with self.path.open("r", encoding="utf-8") as handle:
loaded = json.load(handle)
if isinstance(loaded, dict):
self.data.update(loaded)
except (OSError, json.JSONDecodeError):
pass
def _save(self) -> None:
common.ensure_parent(str(self.path))
tmp = self.path.with_suffix(self.path.suffix + ".tmp")
with tmp.open("w", encoding="utf-8") as handle:
json.dump(self.data, handle, indent=2, sort_keys=True)
tmp.replace(self.path)
def snapshot(self) -> Dict[str, Any]:
with self.lock:
return {
"last_restart": self.data.get("last_restart"),
"restart_times": list(self.data.get("restart_times", [])),
"pending_since": self.data.get("pending_since"),
"restart_in_progress": bool(self.data.get("restart_in_progress")),
}
def mutate(self, updater) -> Dict[str, Any]:
with self.lock:
result = updater(self.data)
self._save()
return result
class SwapSampler:
def __init__(self, config: Dict[str, Any]):
self.config = config
def sample(self) -> common.SwapStats:
if SYSTEM == "darwin":
output = subprocess.run(
["sysctl", "-n", "vm.swapusage"],
check=True,
capture_output=True,
text=True,
timeout=5,
).stdout
return common.parse_sysctl_swap(output)
if SYSTEM == "linux":
try:
with open("/proc/meminfo", "r", encoding="utf-8") as handle:
return common.parse_proc_meminfo(handle.read())
except OSError:
output = subprocess.run(
["free", "-b"],
check=True,
capture_output=True,
text=True,
timeout=5,
).stdout
return common.parse_free_output(output)
raise RuntimeError("unsupported platform: %s" % SYSTEM)
@staticmethod
def service_active(config: Dict[str, Any]) -> Optional[bool]:
try:
if SYSTEM == "darwin":
plist = config["launch_agent_plist"]
result = subprocess.run(
["launchctl", "list"],
capture_output=True,
text=True,
timeout=5,
)
return "com.ollama" in result.stdout or result.returncode == 0
output = subprocess.run(
["systemctl", "is-active", str(config["linux_service"])],
capture_output=True,
text=True,
timeout=5,
)
return output.stdout.strip() == "active"
except (OSError, subprocess.SubprocessError):
return None
class RemoteOllama:
def __init__(self, config: Dict[str, Any]):
self.config = config
def ps(self) -> Dict[str, Any]:
return self._get(str(self.config["ollama_ps_url"]))
def health(self) -> bool:
try:
self._get(str(self.config["health_url"]))
return True
except Exception:
return False
def running_models(self) -> List[Any]:
data = self.ps()
models = data.get("models")
if isinstance(models, list):
return models
return []
def is_busy(self) -> bool:
timeout = int(self.config["busy_check_timeout_seconds"])
# Reuse configured timeout by performing a short request.
try:
with urllib.request.urlopen(
str(self.config["ollama_ps_url"]), timeout=timeout
) as response:
data = json.loads(response.read().decode("utf-8"))
models = data.get("models") if isinstance(data, dict) else None
return bool(models)
except Exception:
return False
@staticmethod
def _get(url: str) -> Dict[str, Any]:
with urllib.request.urlopen(url, timeout=5) as response:
raw = response.read().decode("utf-8")
data = common.safe_json_loads(raw)
if not isinstance(data, dict):
raise ValueError("ollama response is not a JSON object")
return data
class ServiceController:
def __init__(self, config: Dict[str, Any], state: StateStore):
self.config = config
self.state = state
def restart(self, force: bool = False, reason: str = "threshold") -> Dict[str, Any]:
stats_before = None
busy = False
try:
busy = RemoteOllama(self.config).is_busy()
except Exception:
busy = False
if busy and not force:
raise BusyError("ollama has running models; restart deferred")
started = common.now_utc()
with self.state.lock:
if self.state.data.get("restart_in_progress"):
raise RestartInProgressError("restart already in progress")
self.state.data["restart_in_progress"] = True
self.state._save()
try:
if SYSTEM == "darwin":
plist = str(self.config["launch_agent_plist"])
subprocess.run(
["launchctl", "unload", plist],
check=False,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
subprocess.run(
["launchctl", "load", "-w", plist],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
elif SYSTEM == "linux":
subprocess.run(
["systemctl", "restart", str(self.config["linux_service"])],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
else:
raise RuntimeError("unsupported platform: %s" % SYSTEM)
health_ok = self._wait_for_health(int(self.config["restart_health_timeout_seconds"]))
finished = common.now_utc()
times = list(self.state.snapshot()["restart_times"]) + [common.format_iso(finished)]
# Keep last hour of history.
cutoff = finished - timedelta(hours=1)
times = [t for t in times if (common.parse_iso(t) or finished) >= cutoff]
self.state.mutate(
lambda data: {
**data,
"last_restart": common.format_iso(finished),
"restart_times": times,
"pending_since": None,
"restart_in_progress": False,
}
)
return {
"restarted": True,
"reason": reason,
"busy_before": busy,
"health_ok": health_ok,
"duration_seconds": round((finished - started).total_seconds(), 2),
}
except Exception:
self.state.mutate(
lambda data: {**data, "restart_in_progress": False}
)
raise
def _wait_for_health(self, timeout: int) -> bool:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if RemoteOllama(self.config).health():
return True
time.sleep(2)
return False
class BusyError(RuntimeError):
pass
class RestartInProgressError(RuntimeError):
pass
class Runtime:
def __init__(self, config: Dict[str, Any]):
self.config = config
self.sampler = SwapSampler(config)
self.state = StateStore(str(config["state_file"]))
self.controller = ServiceController(config, self.state)
self.lock = threading.Lock()
self._pending_lock = threading.Lock()
def status(self) -> Dict[str, Any]:
try:
stats = self.sampler.sample()
stats_dict = stats.to_dict()
stats_ok = True
stats_error = None
except Exception as exc:
stats = None
stats_dict = None
stats_ok = False
stats_error = str(exc)
busy = None
try:
busy = RemoteOllama(self.config).is_busy()
except Exception:
busy = None
state = self.state.snapshot()
threshold = common.threshold_exceeded(self.config, stats) if stats else False
would_restart = threshold
recommendation = "restart" if threshold and not busy else ("defer" if threshold and busy else "none")
return {
"platform": SYSTEM,
"config": {
"bind_host": self.config["bind_host"],
"port": self.config["port"],
"swap_threshold_percent": self.config.get("swap_threshold_percent"),
"swap_threshold_mb": self.config.get("swap_threshold_mb"),
"cooldown_seconds": self.config.get("cooldown_seconds"),
"max_restarts_per_hour": self.config.get("max_restarts_per_hour"),
"poll_interval_seconds": self.config.get("poll_interval_seconds"),
},
"swap": stats_dict,
"swap_ok": stats_ok,
"swap_error": stats_error,
"threshold_exceeded": threshold,
"ollama_busy": busy,
"recommendation": recommendation,
"would_restart": would_restart,
"service_active": SwapSampler.service_active(self.config),
"running_models": self._running_models(),
"state": state,
"command_preview": common.service_command_preview(self.config),
}
def _running_models(self) -> List[Any]:
try:
return RemoteOllama(self.config).running_models()
except Exception:
return []
def restart_allowed(self, force: bool = False) -> tuple[bool, str]:
state = self.state.snapshot()
times = state["restart_times"]
now = common.now_utc()
if not force:
cooldown = int(self.config["cooldown_seconds"])
last = common.parse_iso(state["last_restart"])
if last is not None and now - last < timedelta(seconds=cooldown):
remaining = int(cooldown - (now - last).total_seconds())
return False, "cooldown active: %d seconds remaining" % remaining
max_hour = int(self.config["max_restarts_per_hour"])
cutoff = now - timedelta(hours=1)
if common.count_recent(times, cutoff) >= max_hour:
return False, "hourly restart limit reached"
return True, "ok"
def restart(self, force: bool = False, reason: str = "manual") -> Dict[str, Any]:
allowed, message = self.restart_allowed(force=force)
if not allowed:
raise CooldownError(message)
effective_force = force
stats = None
try:
stats = self.sampler.sample()
busy = RemoteOllama(self.config).is_busy()
except Exception:
busy = False
pending = self.state.snapshot().get("pending_since")
now = common.now_utc()
if (
not force
and stats is not None
and busy
and common.threshold_exceeded(self.config, stats)
and pending is not None
):
pending_dt = common.parse_iso(pending)
if pending_dt is not None and (now - pending_dt).total_seconds() >= int(
self.config["restart_force_after_seconds"]
):
effective_force = True
if busy and not effective_force:
# Keep deferred restart state for the daemon.
self.state.mutate(lambda data: {**data, "pending_since": common.format_iso(now)})
raise BusyError("ollama has running models; restart deferred")
if stats is not None and common.threshold_exceeded(self.config, stats) and not pending:
self.state.mutate(lambda data: {**data, "pending_since": common.format_iso(now)})
return self.controller.restart(force=effective_force, reason=reason)
def check_once(self) -> Dict[str, Any]:
actions = {"restarted": False, "deferred": False, "message": "threshold not exceeded"}
try:
stats = self.sampler.sample()
except Exception as exc:
return {"restarted": False, "deferred": False, "message": "swap sample failed: %s" % exc}
if not common.threshold_exceeded(self.config, stats):
self.state.mutate(lambda data: {**data, "pending_since": None})
return actions
busy = RemoteOllama(self.config).is_busy()
if busy:
now = common.now_utc()
pending = self.state.snapshot().get("pending_since")
if pending is None:
self.state.mutate(lambda data: {**data, "pending_since": common.format_iso(now)})
elif common.parse_iso(pending) is not None and (now - common.parse_iso(pending)).total_seconds() >= int(
self.config["restart_force_after_seconds"]
):
pass
if not self.restart_allowed(force=False)[0]:
return {"restarted": False, "deferred": True, "message": "busy and cooldown/limit active"}
try:
result = self.restart(force=False, reason="threshold")
return {"restarted": True, "deferred": False, "message": result}
except BusyError:
return {"restarted": False, "deferred": True, "message": "busy; restart deferred"}
# Not busy.
if not self.restart_allowed(force=False)[0]:
return {"restarted": False, "deferred": False, "message": self.restart_allowed(force=False)[1]}
try:
result = self.restart(force=False, reason="threshold")
return {"restarted": True, "deferred": False, "message": result}
except BusyError:
return {"restarted": False, "deferred": True, "message": "busy; restart deferred"}
class CooldownError(RuntimeError):
pass
class JsonHandler(http.server.BaseHTTPRequestHandler):
runtime: "Runtime" = None # type: ignore
def _send(self, status: int, payload: Any) -> None:
body = json.dumps(payload, indent=2).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _authorized(self) -> bool:
token = self.runtime.config.get("token")
if not token:
return True
header = self.headers.get("Authorization", "")
expected = "Bearer " + str(token)
return header == expected
def do_GET(self) -> None: # noqa: N802
if not self._authorized():
self._send(401, {"error": "unauthorized"})
return
if self.path.split("?", 1)[0] == "/health":
self._send(200, {"status": "ok", "platform": SYSTEM})
return
if self.path.split("?", 1)[0] == "/status":
self._send(200, self.runtime.status())
return
self._send(404, {"error": "not found"})
def do_POST(self) -> None: # noqa: N802
if not self._authorized():
self._send(401, {"error": "unauthorized"})
return
path = self.path.split("?", 1)[0]
if path != "/restart":
self._send(404, {"error": "not found"})
return
length = int(self.headers.get("Content-Length", "0") or "0")
raw = self.rfile.read(length) if length else b"{}"
payload = common.safe_json_loads(raw.decode("utf-8", errors="replace")) or {}
force = bool(payload.get("force", False))
reason = str(payload.get("reason", "manual"))
try:
result = self.runtime.restart(force=force, reason=reason)
self._send(200, {"ok": True, "result": result})
except CooldownError as exc:
self._send(429, {"ok": False, "error": str(exc)})
except BusyError as exc:
self._send(409, {"ok": False, "error": str(exc)})
except RuntimeError as exc:
self._send(503, {"ok": False, "error": str(exc)})
except Exception as exc:
self._send(500, {"ok": False, "error": str(exc)})
def log_message(self, format: str, *args: Any) -> None:
return
class ReusableServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
daemon_threads = True
allow_reuse_address = True
def serve(config: Dict[str, Any]) -> None:
runtime = Runtime(config)
JsonHandler.runtime = runtime
host = str(config["bind_host"])
port = int(config["port"])
if host not in common.LOCALHOSTS and not config.get("token") and not config.get("allow_insecure"):
raise ValueError("refusing to bind to non-localhost without a token")
httpd = ReusableServer((host, port), JsonHandler)
print("ollama-plagin-swap agent listening on http://%s:%d" % (host, port), flush=True)
def watch() -> None:
while True:
try:
result = runtime.check_once()
print("swap check: %s" % result, flush=True)
except Exception as exc:
print("swap check failed: %s" % exc, flush=True)
time.sleep(int(config["poll_interval_seconds"]))
thread = threading.Thread(target=watch, name="swap-watcher", daemon=True)
thread.start()
httpd.serve_forever()
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="ollama-plagin-swap remote agent")
parser.add_argument("--config", help="path to JSON configuration")
subparsers = parser.add_subparsers(dest="command")
subparsers.add_parser("serve", help="run HTTP agent and watcher")
status_parser = subparsers.add_parser("status", help="print one status report")
status_parser.add_argument("--json", action="store_true")
status_parser.add_argument("--dry-run", action="store_true")
check_parser = subparsers.add_parser("check", help="run one threshold check")
check_parser.add_argument("--dry-run", action="store_true")
restart_parser = subparsers.add_parser("restart-now", help="restart ollama service now")
restart_parser.add_argument("--force", action="store_true")
restart_parser.add_argument("--reason", default="manual")
restart_parser.add_argument("--dry-run", action="store_true")
return parser
def main(argv: Optional[List[str]] = None) -> int:
parser = build_parser()
args = parser.parse_args(argv)
config = common.load_config(args.config)
runtime = Runtime(config)
command = args.command or "serve"
if command == "serve":
serve(config)
return 0
if command == "status":
report = runtime.status()
if getattr(args, "dry_run", False):
report["dry_run"] = True
if getattr(args, "json", False):
print(json.dumps(report, indent=2))
else:
print("swap: %s" % report.get("swap"))
print("threshold exceeded: %s" % report.get("threshold_exceeded"))
print("ollama busy: %s" % report.get("ollama_busy"))
print("recommendation: %s" % report.get("recommendation"))
print("last restart: %s" % report.get("state", {}).get("last_restart"))
return 0
if command == "check":
if getattr(args, "dry_run", False):
report = runtime.status()
print(json.dumps({"dry_run": True, "would_restart": report.get("would_restart"), "status": report}, indent=2))
return 0
print(json.dumps(runtime.check_once(), indent=2))
return 0
if command == "restart-now":
if getattr(args, "dry_run", False):
allowed, message = runtime.restart_allowed(force=bool(args.force))
print(json.dumps({"dry_run": True, "allowed": allowed, "message": message, "command": common.service_command_preview(config)}, indent=2))
return 0
try:
print(json.dumps(runtime.restart(force=bool(args.force), reason=str(args.reason)), indent=2))
return 0
except Exception as exc:
print(json.dumps({"ok": False, "error": str(exc)}, indent=2))
return 1
parser.print_help()
return 2
if __name__ == "__main__":
raise SystemExit(main())
+254
View File
@@ -0,0 +1,254 @@
"""Shared configuration, models, parsing, and policy helpers for ollama-plagin-swap."""
from __future__ import annotations
import copy
import json
import os
import platform
import re
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
DEFAULT_CONFIG = {
"bind_host": "127.0.0.1",
"port": 8731,
"token": None,
"allow_insecure": False,
"poll_interval_seconds": 60,
"swap_threshold_percent": 25,
"swap_threshold_mb": None,
"cooldown_seconds": 600,
"max_restarts_per_hour": 2,
"restart_force_after_seconds": 300,
"busy_check_timeout_seconds": 3,
"health_timeout_seconds": 5,
"restart_health_timeout_seconds": 180,
"health_url": "http://127.0.0.1:11434/api/tags",
"ollama_ps_url": "http://127.0.0.1:11434/api/ps",
"launch_agent_plist": "~/Library/LaunchAgents/com.ollama.serve.plist",
"linux_service": "ollama",
"state_file": None,
}
LOCALHOSTS = {"127.0.0.1", "localhost", "::1"}
@dataclass
class SwapStats:
total_mb: float
used_mb: float
free_mb: float
@property
def percent(self) -> float:
if self.total_mb <= 0:
return 0.0
return self.used_mb / self.total_mb * 100.0
def to_dict(self) -> dict[str, Any]:
return {
"swap_total_mb": round(self.total_mb, 2),
"swap_used_mb": round(self.used_mb, 2),
"swap_free_mb": round(self.free_mb, 2),
"swap_percent": round(self.percent, 2),
}
def default_config_path() -> Path:
env = os.environ.get("OLLAMA_SWAP_CONFIG")
if env:
return Path(env).expanduser()
if platform.system().lower() == "linux":
system = Path("/etc/ollama-plagin-swap/config.json")
if system.exists():
return system
return Path("~/.config/ollama-plagin-swap/config.json").expanduser()
def default_state_path() -> Path:
base = os.environ.get("OLLAMA_SWAP_STATE_DIR")
root = Path(base).expanduser() if base else Path("~/.local/state/ollama-plagin-swap").expanduser()
root.mkdir(parents=True, exist_ok=True)
return root / "state.json"
def merge_config(user: Optional[dict[str, Any]] = None) -> dict[str, Any]:
result = copy.deepcopy(DEFAULT_CONFIG)
if user:
result.update(user)
if result.get("state_file") is None:
result["state_file"] = str(default_state_path())
if result.get("launch_agent_plist"):
result["launch_agent_plist"] = str(Path(result["launch_agent_plist"]).expanduser())
return result
def load_config(path: Optional[str] = None) -> dict[str, Any]:
config_path = Path(path).expanduser() if path else default_config_path()
data: dict[str, Any] = {}
if config_path.exists() and config_path.stat().st_size > 0:
with config_path.open("r", encoding="utf-8") as handle:
loaded = json.load(handle)
if not isinstance(loaded, dict):
raise ValueError("configuration must be a JSON object")
data = loaded
config = merge_config(data)
validate_config(config)
return config
def validate_config(config: dict[str, Any]) -> None:
if not isinstance(config.get("port"), int) or not (1 <= int(config["port"]) <= 65535):
raise ValueError("port must be an integer between 1 and 65535")
if config.get("poll_interval_seconds", 0) <= 0:
raise ValueError("poll_interval_seconds must be positive")
threshold = config.get("swap_threshold_percent")
if threshold is not None and not (0 <= float(threshold) <= 100):
raise ValueError("swap_threshold_percent must be between 0 and 100")
if config.get("cooldown_seconds", 0) < 0:
raise ValueError("cooldown_seconds must be non-negative")
if config.get("max_restarts_per_hour", 0) < 0:
raise ValueError("max_restarts_per_hour must be non-negative")
host = str(config.get("bind_host", ""))
if not host:
raise ValueError("bind_host is required")
token = config.get("token")
if host not in LOCALHOSTS and not token and not config.get("allow_insecure"):
raise ValueError("a token is required when binding to a non-localhost address")
absolute = config.get("swap_threshold_mb")
if absolute is not None and float(absolute) < 0:
raise ValueError("swap_threshold_mb must be non-negative")
def _suffix_to_mb(value: float, suffix: str) -> float:
suffix = (suffix or "").upper()
multipliers = {
"": 1.0 / 1024.0 / 1024.0,
"B": 1.0 / 1024.0 / 1024.0,
"K": 1.0 / 1024.0,
"M": 1.0,
"G": 1024.0,
"T": 1024.0 * 1024.0,
}
if suffix not in multipliers:
raise ValueError("unknown size suffix: %s" % suffix)
return value * multipliers[suffix]
def parse_sysctl_swap(output: str) -> SwapStats:
output = output.replace(",", ".")
matches = re.findall(r"(\w+)\s*=\s*([0-9]+(?:\.[0-9]+)?)\s*([KMGTPE]?)", output)
values = {}
for name, number, suffix in matches:
values[name.lower()] = _suffix_to_mb(float(number), suffix)
if "total" not in values:
raise ValueError("sysctl swap output does not contain total: %r" % output)
total = values["total"]
used = values.get("used", 0.0)
free = values.get("free", max(total - used, 0.0))
return SwapStats(total, used, free)
def parse_proc_meminfo(text: str) -> SwapStats:
values = {}
for line in text.splitlines():
parts = line.split()
if len(parts) >= 2 and parts[0].endswith(":"):
key = parts[0][:-1]
try:
values[key] = float(parts[1])
except ValueError:
continue
if len(parts) > 2 and parts[2].upper() == "KB":
values[key] *= 1.0 / 1024.0
total = values.get("SwapTotal", 0.0)
free = values.get("SwapFree", 0.0)
used = max(total - free, 0.0)
return SwapStats(total, used, free)
def parse_free_output(output: str) -> SwapStats:
output = output.replace(",", ".")
for line in output.splitlines():
fields = line.split()
if fields and fields[0].lower().startswith("swap"):
total = float(fields[1]) / 1024.0 / 1024.0
used = float(fields[2]) / 1024.0 / 1024.0
free = float(fields[3]) / 1024.0 / 1024.0
return SwapStats(total, used, free)
lines = [line for line in output.splitlines() if line.strip()]
if not lines:
raise ValueError("empty free output")
last = lines[-1].split()
total = float(last[1]) / 1024.0 / 1024.0
used = float(last[2]) / 1024.0 / 1024.0
free = float(last[3]) / 1024.0 / 1024.0
return SwapStats(total, used, free)
def threshold_exceeded(config: dict[str, Any], stats: SwapStats) -> bool:
percent = config.get("swap_threshold_percent")
absolute = config.get("swap_threshold_mb")
has_rule = False
if percent is not None:
has_rule = True
if absolute is not None:
has_rule = True
if not has_rule:
return False
if percent is not None and absolute is not None:
return stats.percent >= float(percent) and stats.used_mb >= float(absolute)
if percent is not None:
return stats.percent >= float(percent)
return stats.used_mb >= float(absolute)
def parse_iso(value: Optional[str]) -> Optional[datetime]:
if not value:
return None
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
def now_utc() -> datetime:
return datetime.now(timezone.utc)
def format_iso(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="seconds")
def count_recent(times: List[str], cutoff: datetime) -> int:
total = 0
for item in times:
parsed = parse_iso(item)
if parsed is not None and parsed >= cutoff:
total += 1
return total
def is_localhost(host: str) -> bool:
return host in LOCALHOSTS
def ensure_parent(path: str) -> None:
Path(path).expanduser().resolve().parent.mkdir(parents=True, exist_ok=True)
def safe_json_loads(text: str) -> Any:
try:
return json.loads(text)
except (json.JSONDecodeError, TypeError):
return None
def service_command_preview(config: dict[str, Any]) -> str:
if platform.system().lower() == "darwin":
plist = config.get("launch_agent_plist")
return "launchctl unload %s 2>/dev/null, launchctl load -w %s" % (plist, plist)
return "systemctl restart %s" % config.get("linux_service")
+105
View File
@@ -0,0 +1,105 @@
import json
import sys
import threading
import time
from pathlib import Path
from unittest import mock
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import ollama_swap_agent as a
import ollama_swap_common as c
def make_config(tmp_path):
return c.merge_config(
{
"bind_host": "127.0.0.1",
"port": 0,
"token": "secret",
"state_file": str(tmp_path / "state.json"),
"launch_agent_plist": str(tmp_path / "com.ollama.serve.plist"),
"poll_interval_seconds": 60,
}
)
def test_check_once_restarts_when_threshold_exceeded(tmp_path):
config = make_config(tmp_path)
runtime = a.Runtime(config)
fake_stats = c.SwapStats(100, 40, 60)
with mock.patch.object(runtime.sampler, "sample", return_value=fake_stats), \
mock.patch.object(runtime, "_running_models", return_value=[]), \
mock.patch.object(a.RemoteOllama, "is_busy", return_value=False), \
mock.patch.object(runtime.controller, "restart", return_value={"restarted": True}):
result = runtime.check_once()
assert result["restarted"] is True
def test_check_once_defers_when_busy(tmp_path):
config = make_config(tmp_path)
runtime = a.Runtime(config)
fake_stats = c.SwapStats(100, 40, 60)
with mock.patch.object(runtime.sampler, "sample", return_value=fake_stats), \
mock.patch.object(a.RemoteOllama, "is_busy", return_value=True):
result = runtime.check_once()
assert result["restarted"] is False
assert result["deferred"] is True
def test_restart_rejects_busy_without_force(tmp_path):
config = make_config(tmp_path)
runtime = a.Runtime(config)
fake_stats = c.SwapStats(100, 40, 60)
with mock.patch.object(runtime.sampler, "sample", return_value=fake_stats), \
mock.patch.object(a.RemoteOllama, "is_busy", return_value=True):
try:
runtime.restart(force=False)
except a.BusyError:
pass
else:
raise AssertionError("expected BusyError")
def test_http_status_and_restart(tmp_path):
config = make_config(tmp_path)
runtime = a.Runtime(config)
a.JsonHandler.runtime = runtime
httpd = a.ReusableServer(("127.0.0.1", 0), a.JsonHandler)
port = httpd.server_address[1]
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
thread.start()
try:
# unauthorized
req = __import__("urllib.request", fromlist=["urlopen"]).urlopen(
"http://127.0.0.1:%d/status" % port, timeout=2
)
raise AssertionError("expected HTTPError")
except Exception as exc:
assert "401" in str(exc)
fake_stats = c.SwapStats(100, 40, 60)
with mock.patch.object(runtime.sampler, "sample", return_value=fake_stats), \
mock.patch.object(a.RemoteOllama, "is_busy", return_value=False), \
mock.patch.object(runtime.controller, "restart", return_value={"restarted": True}):
import urllib.request
req = urllib.request.Request(
"http://127.0.0.1:%d/status" % port,
headers={"Authorization": "Bearer secret"},
)
with urllib.request.urlopen(req, timeout=2) as response:
data = json.loads(response.read().decode())
assert data["recommendation"] in {"restart", "none", "defer"}
body = json.dumps({"reason": "test", "force": True}).encode()
req = urllib.request.Request(
"http://127.0.0.1:%d/restart" % port,
data=body,
headers={"Authorization": "Bearer secret", "Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=2) as response:
data = json.loads(response.read().decode())
assert data["ok"] is True
httpd.shutdown()
httpd.server_close()
+70
View File
@@ -0,0 +1,70 @@
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import ollama_swap_common as c
def test_parse_sysctl_swap():
output = "total = 2048.00M used = 512.00M free = 1536.00M (encrypted)"
stats = c.parse_sysctl_swap(output)
assert round(stats.total_mb, 2) == 2048.0
assert round(stats.used_mb, 2) == 512.0
assert round(stats.percent, 2) == 25.0
def test_parse_proc_meminfo():
text = "SwapTotal: 2097148 kB\nSwapFree: 1048572 kB\n"
stats = c.parse_proc_meminfo(text)
assert round(stats.total_mb, 2) == 2048.0
assert round(stats.used_mb, 2) == 1024.0
def test_parse_free_output():
output = " total used free\nSwap: 2147483648 536870912 1610612736\n"
stats = c.parse_free_output(output)
assert round(stats.total_mb, 2) == 2048.0
assert round(stats.used_mb, 2) == 512.0
def test_threshold_percent():
config = c.merge_config({"swap_threshold_percent": 25, "swap_threshold_mb": None})
stats = c.SwapStats(1000, 200, 800)
assert c.threshold_exceeded(config, stats) is False
stats2 = c.SwapStats(1000, 300, 700)
assert c.threshold_exceeded(config, stats2) is True
def test_threshold_combined():
config = c.merge_config({"swap_threshold_percent": 25, "swap_threshold_mb": 200})
assert c.threshold_exceeded(config, c.SwapStats(1000, 100, 900)) is False
assert c.threshold_exceeded(config, c.SwapStats(1000, 300, 700)) is True
def test_validate_config_requires_token_for_remote():
config = c.merge_config({"bind_host": "0.0.0.0", "token": None, "allow_insecure": False})
try:
c.validate_config(config)
except ValueError as exc:
assert "token" in str(exc)
else:
raise AssertionError("expected ValueError")
def test_count_recent():
now = c.now_utc()
import datetime as dt
times = [c.format_iso(now)]
recent = c.format_iso(now - dt.timedelta(minutes=10))
old = c.format_iso(now - dt.timedelta(hours=2))
assert c.count_recent([recent, old], now - dt.timedelta(hours=1)) == 1
def test_load_config_missing_file(tmp_path):
config = c.load_config(str(tmp_path / "config.json"))
assert config["port"] == 8731
assert config["swap_threshold_percent"] == 25