From 54965a1b17ab317bb6d1c1706d275578e31d95c6 Mon Sep 17 00:00:00 2001 From: vecnode Date: Mon, 19 Oct 2026 16:34:48 +0000 Subject: [PATCH] ollama: five tools over the local daemon, a benchmark that measures, and three skills The pack could call a model; it could not say what that model did here. This row reads the fields every Ollama response already carries, and refuses to print a number without its conditions. - ollama_status: models on disk, what is loaded, and 'ollama serve' when the daemon is down. It starts nothing. - ollama_run: ONE bounded inference (num_predict is always sent), streamed so the first token is timed, with the daemon's token counts and nanosecond durations. - ollama_bench: 3+ runs only; temperature 0 and seed 42 by default; an unload before run 1 so 'cold' means cold; median, p95, min/max and spread; the cold/warm claim stated at the FIRST TOKEN because eval_duration excludes the load. A/B and num_ctx sweeps. - ollama_models: list/show, and pull/delete behind a confirmation that repeats the tag. No implicit download. - ollama_embed: /api/embed with a /api/embeddings fallback, dimension and cosine similarity; a completion model refused by capability. - three skills (REST API, metric arithmetic, Python client) whose curl/Python/JS examples the package check parses and compiles. - the loopback fence: an OLLAMA_HOST on another machine is refused before any request; DSH_OLLAMA_ALLOW_REMOTE=1 allows one explicitly. The package check serves a fake daemon with fixed nanosecond durations, so 100 tokens in 2 s is exactly 50.0 tok/s. The live half was run against this machine's own Ollama 0.40.3: one generation, a 3-run bench, and a cold bench that measured a 2.19 s load and a 22 ms warm first token. Verified: packages/dsh-ollama 'npm run check' (345 pass with the daemon stopped, live skipped loudly; 352 with it running), check-node-routes, check-no-secrets, check-scripts, check-skill-examples, check-media-examples, check-client-bundles. --- .dsh-version.json | 20 + AGENTS.md | 17 +- ARCHITECTURE.md | 4 +- README.md | 1 + docs/PLAN-ollama.md | 121 -- packages/dsh-ollama/README.md | 152 +++ .../dsh-ollama/checks/check-ollama-node.mjs | 929 +++++++++++++++ packages/dsh-ollama/cordis.patch.yml | 28 + packages/dsh-ollama/lib/cli.js | 74 ++ packages/dsh-ollama/lib/client.js | 446 ++++++++ packages/dsh-ollama/lib/errors.js | 29 + packages/dsh-ollama/lib/index.js | 206 ++++ packages/dsh-ollama/lib/metrics.js | 245 ++++ packages/dsh-ollama/lib/tools.js | 1018 +++++++++++++++++ packages/dsh-ollama/lib/validate.js | 467 ++++++++ packages/dsh-ollama/package.json | 29 + .../dsh-ollama/skills/ollama-api/SKILL.md | 366 ++++++ .../skills/ollama-inference-metrics/SKILL.md | 213 ++++ .../dsh-ollama/skills/ollama-python/SKILL.md | 612 ++++++++++ scripts/checks/README.md | 9 +- scripts/checks/check-node-routes.mjs | 60 +- 21 files changed, 4913 insertions(+), 133 deletions(-) delete mode 100644 docs/PLAN-ollama.md create mode 100644 packages/dsh-ollama/README.md create mode 100644 packages/dsh-ollama/checks/check-ollama-node.mjs create mode 100644 packages/dsh-ollama/cordis.patch.yml create mode 100644 packages/dsh-ollama/lib/cli.js create mode 100644 packages/dsh-ollama/lib/client.js create mode 100644 packages/dsh-ollama/lib/errors.js create mode 100644 packages/dsh-ollama/lib/index.js create mode 100644 packages/dsh-ollama/lib/metrics.js create mode 100644 packages/dsh-ollama/lib/tools.js create mode 100644 packages/dsh-ollama/lib/validate.js create mode 100644 packages/dsh-ollama/package.json create mode 100644 packages/dsh-ollama/skills/ollama-api/SKILL.md create mode 100644 packages/dsh-ollama/skills/ollama-inference-metrics/SKILL.md create mode 100644 packages/dsh-ollama/skills/ollama-python/SKILL.md diff --git a/.dsh-version.json b/.dsh-version.json index 69f8fd9..18e0e67 100644 --- a/.dsh-version.json +++ b/.dsh-version.json @@ -112,6 +112,26 @@ "Docker Desktop, for `docker_start` on Windows and macOS (launched, then its daemon polled)" ] }, + "dsh-ollama": { + "version": "0.1.0-alpha.1", + "rows": [ + "ollama" + ], + "rowOverrides": [], + "role": "Ollama as a measured surface: five `ollama_*` tools (status, run, bench, models, embed) over the machine's own daemon at 127.0.0.1:11434 - the models on disk and what is loaded now, ONE bounded inference with the daemon's own token counts and nanosecond durations, a benchmark that refuses fewer than 3 runs and reports a median with spread (cold vs warm stated at the FIRST TOKEN, because eval_duration excludes the load), A/B and num_ctx sweeps, models with pull/delete behind a confirmation that repeats the tag, and embeddings with cosine similarity - plus three bundled skills (the REST API, the metric arithmetic, the Python client) whose examples the package check parses. The loopback fence refuses an OLLAMA_HOST on another machine before any request is made (DSH_OLLAMA_ALLOW_REMOTE=1 allows one explicitly). Host-only: one row, no route, no browser half. Zero npm dependencies, no bundled engine, nothing started on its own. See packages/dsh-ollama/README.md.", + "uses": [ + "@deepseek-ai/dsh-tools (the registry the five tools are registered into)", + "@deepseek-ai/dsh-skill (ctx.skills, resolved lazily; the three skills are also copied into $DSH_HOME/skills by both installers)", + "the Ollama daemon on this machine (its HTTP API at 127.0.0.1:11434; OLLAMA_HOST may move it)" + ], + "requires": [ + "Ollama installed and its daemon RUNNING (`ollama serve`, or the desktop app) - this row starts nothing and names the command", + "a pulled model for ollama_run / ollama_bench, and an embedding model (e.g. nomic-embed-text) for ollama_embed" + ], + "vendors": [ + "NOTHING is vendored, downloaded or installed: the daemon is the user's own service, and this package ships the HTTP client, one argv for `ollama --version`, and the arithmetic" + ] + }, "dsh-media": { "version": "0.1.0-alpha.4", "rows": [ diff --git a/AGENTS.md b/AGENTS.md index 9eea8ac..1a0a172 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -96,6 +96,13 @@ for its behaviour, `SECURITY.md` before touching credentials or the launcher, an then waits for the **daemon**, and a Docker tab of containers and images with their logs; no `exec`, no `system prune`, no free-form `run`; zero npm dependencies and no bundled engine), + `dsh-ollama` (Ollama as a measured surface: five `ollama_*` tools over the + machine's own daemon - status that names `ollama serve` when nothing answers, + one bounded inference with the daemon's token counts and nanosecond durations, + a benchmark that refuses fewer than 3 runs and reports a median with spread, + confirmed pull/delete, embeddings with cosine similarity - three skills whose + examples a check parses/compiles, and a loopback fence that refuses a daemon on + another machine before any request; host-only, zero npm dependencies), `dsh-research` (a run is a folder with a `passport.json` recording the user's own words; nine `rs_*` tools over an append-only claim and source ledger, a claim's numbers recorded as a validated `statistic` rather than left in the prose, a @@ -112,10 +119,10 @@ for its behaviour, `SECURITY.md` before touching credentials or the launcher, an - A package may carry its OWN checks under `packages//checks/`, run with `npm run check` in that folder. They travel with the package (which is what makes them useful outside this repository) and they are ADDITIONAL to - `scripts/checks/`, never a replacement for it: today eight packages ship their own + `scripts/checks/`, never a replacement for it: today nine packages ship their own (`dsh-supercollider` with four checks, `dsh-research` with seven, plus - `dsh-diagrams`, `dsh-frontend-craft`, `dsh-proto-lab`, `dsh-ocr`, `dsh-wsl` and - `dsh-docker`), and the pack-wide checks still own the routes, the bundle and the + `dsh-diagrams`, `dsh-frontend-craft`, `dsh-proto-lab`, `dsh-ocr`, `dsh-wsl`, + `dsh-docker` and `dsh-ollama`), and the pack-wide checks still own the routes, the bundle and the manifest. - `scripts/` — the launchers (`run-web.*`, `run-desktop.*`), and four folders: `console/` (the shared console layer), `setup/` (install and uninstall, which the @@ -133,8 +140,8 @@ for its behaviour, `SECURITY.md` before touching credentials or the launcher, an refused **by name**, because there is no embedded browser to select. `app/README.md` explains it. `check-shell.mjs` is what proves it, and it drives the real binary. -- `docs/` — install, build, deploy, release, paths, compatibility, and the one plan - still in flight (`PLAN-ollama.md`). There is no distributer and no ship list: the distribution is this +- `docs/` — install, build, deploy, release, paths, compatibility. There is no + distributer and no ship list: the distribution is this repository plus the binary, and the app installs the pack itself. - `.dsh-version.json` — the pinned harness line and the pack manifest (its versions must match every `package.json`; `check-node-routes.mjs` enforces diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7f05d05..ae26556 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -65,6 +65,7 @@ disable/replace a core row **in its own layer** — a later layer wins per row. | `dsh-research` | `research` | research as a folder the agent fills and the right bar shows: a run is `RUN.json` + a `passport.json` that records the user's instructions and every gate answer **verbatim**, three `rs_*` tools, four routes, and the Research tab where a **person** — never the model — answers the two frame gates (`g1-rq`, `g2-intake`). No socket, no child process, no vendored engine | | `dsh-ocr` | `ocr` | the pack's OCR owner: one tool, `ocr`, over `tesseract` from PATH or the OCR engine Windows itself ships (`lib/win-ocr.ps1` + `Windows.Media.Ocr`, which rasterizes a PDF page through `Windows.Data.Pdf`). Host-only — no client, no route | | `dsh-docker` | `docker` | Docker (or Podman) as a surface: nine `docker_*` tools over the machine's CLI, a `docker_start` that launches Docker Desktop or a podman machine and waits for its **daemon**, and a Docker tab of containers and images. argv-only and validated; no `exec`, no `system prune`, no free-form `run` | +| `dsh-ollama` | `ollama` | Ollama as a **measured** surface: five `ollama_*` tools over the machine's own daemon — status (models on disk, what is loaded, and `ollama serve` when nothing answers), one bounded inference with the daemon's token counts and nanosecond durations, a benchmark that **refuses fewer than 3 runs** and reports a median with spread (cold vs warm stated at the first token), models with pull/delete confirmed by repeating the tag, embeddings with cosine similarity — three skills whose examples a check parses, and a **loopback fence** that refuses a daemon on another machine before any request. Host-only, no route, no browser half | | `dsh-proto-lab` | `proto-lab` | the **prototyping lab**: five `lab_*` tools write a run into the lab root (`~/Desktop/vncode-lab/runs///` on a machine with a Desktop, `/lab` otherwise, `DSH_PROTO_LAB_ROOT` to move it; gitignored — the bundle ships the generator, not the runs), a runtime registry (`p5js`, `openframeworks`, and a template for the third) whose every module the checks drive, a committed example library beside each skill, an offline-vendored p5.js, and verification as an explicit ladder — `t0-structure` → `t4-measure` — that always names the tier it reached | | `dsh-browser` | `browser` | web surface: the host fetches and renders a page in a disposable sandboxed browser (**currently disabled by the master**) | | `dsh-themes` | `themes` | extra palettes (Nord, Monokai, Hacker, Cyber), header controls, branding, account-menu trimming | @@ -100,9 +101,6 @@ disable/replace a core row **in its own layer** — a later layer wins per row. `docs/COMPATIBILITY.md`; - the app: `app/README.md` (the verbs, the bridge to `scripts/run-web.*`, the app's own root and the harness home inside it, and the three crates); -- work in flight, and not shipped behaviour: `docs/PLAN-ollama.md` (the - `dsh-ollama` design — one host-only row of `ollama_*` tools, the metric - arithmetic, three skills and the checks that pin them); - security: `SECURITY.md`; - **the checks are the contract**: `scripts/checks/*.mjs` pin route registrations, bundle wiring, budgets, the launcher contract and the credential rules. They are run by hand; diff --git a/README.md b/README.md index deeeb8a..612076e 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ way and what it touches. The exact versions are in | [`dsh-ocr`](packages/dsh-ocr/README.md) | the agent can read a **picture**: one `ocr` tool over `tesseract` from PATH or — on Windows, with **nothing installed at all** — the OCR engine Windows ships, which rasterizes a PDF page itself. Names the engine, the language and the page count, refuses a language this host has no data for with the list of what it CAN read, and says every answer is a transcription | | [`dsh-wsl`](packages/dsh-wsl/README.md) | the **POSIX half of this pack, testable on Windows**: `wsl_status` (the launcher's table decoded from UTF-16, the default distribution marked, and with `probe` whether a distribution can actually run a check), `wsl_run` (one **argv array** in a distribution — no shell string — with a deadline, a tree kill, a 1 MB cap per stream and the exit code in the answer) and `wsl_path` (`C:\…` ⇄ `/mnt/c/…`). It exists so `run-web.sh`, `install.sh` and the `sh` branches of the checks run on the machine that writes them; the launcher's default on a Docker machine is `docker-desktop-data`, so the tools name the distribution they used | | [`dsh-docker`](packages/dsh-docker/README.md) | **Docker (or Podman) as a surface**: nine `docker_*` tools over the machine's own CLI — state, containers, images, inspect, logs, pull, run, lifecycle and a `docker_start` that launches **Docker Desktop** (or a `podman machine`) and waits for its **daemon** — plus a Docker tab listing containers and images with start/stop/remove, pull and run. argv-only, no shell, every removal confirmed by repeating the target, and no spelling for `exec`, `system prune` or `--privileged`. A Docker Desktop that is closed is the normal state on Windows 11: the answer says so and offers the button | +| [`dsh-ollama`](packages/dsh-ollama/README.md) | **Ollama as a measured surface**: five `ollama_*` tools over the machine's own daemon — status (the models on disk, what is **loaded** right now, and `ollama serve` when nothing answers), **one bounded inference** with the token counts and nanosecond durations Ollama reports and nothing in the pack used to read, a **benchmark that refuses fewer than 3 runs** and reports a median with spread (cold vs warm stated at the *first token*, because a cold load never lowers tok/s), A/B and `num_ctx` sweeps, models with pull/delete confirmed by repeating the tag, embeddings with cosine similarity — plus **three skills** (the REST API, the metric arithmetic, the Python client) whose examples a check parses, and a **loopback fence** that refuses a daemon on another machine before any request. A daemon that is down is an answer, not a failure | | [`dsh-proto-lab`](packages/dsh-proto-lab/README.md) | a **prototyping lab** the agent fills: five `lab_*` tools write a run into the workspace's own `/lab///` — p5.js sketches a browser opens, and openFrameworks projects that are three real C++ files (`src/main.cpp`, `src/ofApp.h`, `src/ofApp.cpp`) built with your own projectGenerator and `make`. A **runtime registry** makes a third language one module and one line (Processing in Rust, Cinder), a committed example library sits beside each skill and is driven by the checks, and verification is an explicit ladder — structure, parse, compile, render, measure — whose answer always names the tier that was actually reached. The lab itself is **gitignored**: the bundle ships the generator and the examples, never the runs | | [`dsh-research`](packages/dsh-research/README.md) | research as a **folder you keep**: your instruction and every gate answer recorded **in your own words**, nine `rs_*` tools over an append-only claim and source ledger — `rs_write` refuses until *you* confirm the question, and a claim's quote is checked against its source. Numbers are data (`p = .000` refused), a **method record**'s screening counts cannot increase, and a **disclosure record** must be *complete* — authors, ethics, funding, conflicts, data, AI use — before the submission folder prints it beside the title page, never in the manuscript a reviewer sees. `rs_verify action=calibrate` **measures the reader**, and the tab is rendered and measured in a real browser. Eight skills, an editable tab, no socket, no child process, nothing vendored | | [`dsh-browser`](packages/dsh-browser/README.md) | the **Browser** tab, replacing the shipped iframe one: the page is rendered on the host in a disposable engine behind an https-only egress gate (screenshot, post-script text, measured styles) for the tab and for `browser_render` / `browser_query` / `browser_text` — **built and shipped, but turned OFF**: the master layer disables the row, so a profile loads no tab, no routes, no tools and no client bundle | diff --git a/docs/PLAN-ollama.md b/docs/PLAN-ollama.md deleted file mode 100644 index 3b7294c..0000000 --- a/docs/PLAN-ollama.md +++ /dev/null @@ -1,121 +0,0 @@ -# Plan — `dsh-ollama` - -A plugin that makes this pack able to **run, measure and build on** the -machine's own Ollama: one row of `ollama_*` tools over the local daemon's HTTP -API, a benchmark that reports numbers a reader can check, and the skills that -teach the API and the Python integration. Host-only: one row, no browser half, -no route, no vendored engine. It drives the daemon the user already installed, -the way `dsh-docker` drives their Docker CLI. - -Measured on this machine before the plan: `ollama --version` is **0.40.3**, and -the daemon is **not running** — which is the normal state and the first thing -the plugin has to say honestly. - -## Why it makes sense - -- **Inference analysis is the product.** Every other agent surface here can - *call* a model. None can tell you that `qwen2.5:7b` did 41.2 tok/s on this - machine at 8k context, with a 3.1 s cold load, and that a second run is 2.4× - faster because the weights stayed resident. Ollama reports exactly those - numbers in every response (`eval_count`, `eval_duration`, `load_duration`, - `prompt_eval_duration`, `total_duration`) and nothing in the pack reads them. -- **It is loopback, not egress.** The tools talk to `127.0.0.1:11434` — the - machine's own service — so the pack's "no network a plugin starts" rule holds - the same way it does for Docker. A non-loopback `OLLAMA_HOST` is refused by - default (one explicit opt-in variable for a daemon the user really runs on - another box). -- **"Develop Ollama in new apps" is a skills problem, not a scaffold problem.** - The pack already has a prototyping lab for writing projects; what is missing - is the knowledge: the REST surface, the streaming shape, `keep_alive`, the - OpenAI-compatible endpoints, and a Python client pattern that tests itself. - That is three skills, not a fourth generator. - -## The tool surface (one row: `ollama`) - -| tool | what it does | what it refuses | -|---|---|---| -| `ollama_status` | daemon reachability (`/api/version`), the models on disk (`/api/tags`: family, parameter size, quantization, bytes, digest), what is loaded right now (`/api/ps`: processor split, context, `expires_at`), the host it used, and the CLI version | never starts the daemon; says how (`ollama serve`, or the desktop app) | -| `ollama_run` | ONE bounded inference: model, prompt or messages, `options` (`num_predict`, `temperature`, `seed`, `num_ctx`, `stop`, `format`), `keep_alive`; returns the text **and** the numbers: prompt/eval token counts, tok/s from `eval_count / eval_duration`, time-to-first-token, load time, wall clock | a model that is not pulled; a `num_predict` over the cap; a non-loopback host unless opted in | -| `ollama_bench` | N repetitions of one call: cold vs warm, per-run and aggregate (median, p95) tok/s and TTFT, optional A/B over several models, optional context sweep | fewer than 3 runs (a single number is not a measurement); a timeout it cannot bound | -| `ollama_models` | `show` one model (parameters, template, license, family), list with sizes; `pull` and `rm` only with an explicit confirm and a size on screen | an unconfirmed pull (models are gigabytes) or delete | -| `ollama_embed` | embeddings for one or more strings (`/api/embed`), the dimension, cosine similarity between two texts | a model that does not embed | - -Three routes exist only if a UI follows later; **alpha.1 is host-only**, like -`dsh-wsl` and `dsh-ocr`. A results tab (a bench table with a chart) is the -obvious alpha.2. - -## The skills (three, each with examples a check compiles or runs) - -1. **`ollama-api`** — the REST surface as it is on this install: `/api/generate` - and `/api/chat` (streaming NDJSON, one JSON object per line, `done: true` - last), `/api/embed` vs the older `/api/embeddings`, `/api/tags`, `/api/ps`, - `/api/show`, `/api/pull` progress, `/api/create`, `/api/copy`, `/api/delete`, - the OpenAI-compatible `/v1/chat/completions` and `/v1/embeddings`, the - `options` map, `keep_alive`, and error shapes (404 for a missing model, 500 - with a message body). Plus the CLI equivalents and `OLLAMA_HOST` / - `OLLAMA_NUM_PARALLEL` / `OLLAMA_MAX_LOADED_MODELS` / `OLLAMA_KEEP_ALIVE`. -2. **`ollama-inference-metrics`** — what each duration field means and the - arithmetic: tok/s, TTFT = load + prompt-eval, why `total_duration` is not a - latency, why a second run is faster (weights resident), what changes with - `num_ctx`, quantization, GPU offload, and parallel requests; and the reporting - rules: ≥5 runs, median with spread, one variable at a time, temperature 0 with - a fixed seed, and the model **digest** in the answer. -3. **`ollama-python`** — the `ollama` Python package: `chat()`, `generate()`, - streaming, `embed`, tool calling, structured outputs (`format="json"` and a - JSON Schema), the async client, `keep_alive`, timeouts and cancellation; a - project shape (pyproject, a `client.py`, a `bench.py`, a pytest that runs - against a **fake transport** and skips the live half loudly), and how to point - it at the daemon without hardcoding a host. - -The skills carry the code. Nothing scaffolds a project in alpha.1 — the lab owns -"write a project", and duplicating it here would be a second generator with a -second set of bugs. - -## Honesty rules (the part worth engineering against) - -- **A refusal is an answer.** No daemon → one sentence naming the host it tried - and the command that starts one. No model → the tag it wanted, the tags that - exist, and the exact `ollama pull`. -- **Numbers carry their conditions.** Every tok/s names the model digest, the - context length, the run count and the spread. A single run is labelled a - single run. -- **No download a tool starts on its own.** `pull` needs a confirm and shows the - size; nothing pulls a model implicitly. -- **Loopback only, by default.** `OLLAMA_HOST` pointing off-machine is refused - with the reason and the one environment variable that allows it. - -## Checks - -The package ships its own, and `npm run check` is offline and fast: - -- a **fake daemon** (`node:http`) serving canned `/api/version`, `/api/tags`, - `/api/ps` and `/api/generate` responses with fixed nanosecond durations, so the - arithmetic, the caps, the confirmations and every refusal are asserted - deterministically; -- the **loopback fence**: a `OLLAMA_HOST` off-machine is refused, and no request - is made anywhere but 127.0.0.1; -- the **skill examples**: every fenced `curl` shape parsed and every Python block - compiled when a Python is on PATH (skipped loudly otherwise), the same rule the - SuperCollider skills' examples follow; -- a **live half** that runs one tiny generation and one bench against the real - daemon **only when it answers**, and skips with the reason when it does not. - -The pack-wide checks still own the row: `check-node-routes.mjs` for the manifest -and route absence, `check-no-secrets.mjs`, and the root README/ARCHITECTURE rows. - -## What it does not do - -- No model download, no model conversion, no GGUF handling — the daemon's job. -- No GPU driver, no VRAM probing beyond what `/api/ps` reports. -- No chat UI: this is a tool surface plus skills, not a second conversation. -- No remote inference: the OpenAI-compatible *endpoint* is documented for the - user's own code; this plugin still talks to the local daemon. - -## Order of work - -1. `lib/ollama.js` — the client: loopback fence, bounded `fetch` with a timeout, - NDJSON streaming reader, the duration arithmetic, error mapping. -2. `lib/tools.js` — `status`, `run`, `bench`, `models`, `embed`, with the caps. -3. `checks/` — the fake daemon and the live half. -4. `skills/` — the three skills, then wire their examples into the check. -5. Row, manifest, installer rows, README/ARCHITECTURE, and a root README row. diff --git a/packages/dsh-ollama/README.md b/packages/dsh-ollama/README.md new file mode 100644 index 0000000..32dc34d --- /dev/null +++ b/packages/dsh-ollama/README.md @@ -0,0 +1,152 @@ +# dsh-ollama (alpha.1) + +**Ollama as a measured surface.** Five tools over the machine's own daemon - +`ollama_status`, `ollama_run`, `ollama_bench`, `ollama_models`, `ollama_embed` - +that report what a model actually did here: how many tokens, how fast, after how +long a load, with which digest and at which context. Plus the three skills that +teach the REST API, the metric arithmetic and the Python client. + +Why it earns a row: every other surface in this pack can **call** a model. None +can tell a reader that `qwen2.5:7b` did 41.2 tok/s on this machine at 8k +context with a 3.1 s cold load, and that the second run's *first token* arrived +sooner while its tok/s did not move at all. Ollama reports exactly those fields +in every response - `eval_count`, `eval_duration`, `load_duration`, +`prompt_eval_duration`, `total_duration` - and nothing in the pack read them. + +Measured on the machine that wrote this: `ollama --version` is **0.40.3**, and +the daemon was **not running** - which is the normal state and the first thing +the plugin has to say honestly. `ollama_status` says so, names the address it +tried, and names the command that starts one; it never starts one itself. + +## What it adds + +- **`ollama_status`** - the daemon's version and reachability, every model on + disk (family, parameter size, quantization, bytes, digest), and what is + **loaded right now** (`/api/ps`: context length, `size_vram` as an offload + share, `expires_at`). A daemon that is down is an *answer*: the address it + tried, the CLI version if `ollama` is on PATH, and `ollama serve`. Read-only. +- **`ollama_run`** - ONE bounded inference, from `prompt` (`/api/generate`) or + `messages` (`/api/chat`). It streams so the **first token is timed**, and it + returns the text plus prompt/eval token counts, tok/s from + `eval_count / eval_duration`, the load time, the daemon's total, the wall + clock and the `done_reason`. `num_predict` is always sent (512 by default, + ceiling 8192) because Ollama's own default would generate until the context + filled. A `done_reason: "length"` answer says it was cut. +- **`ollama_bench`** - the only tool here that calls a number a **measurement**. + 3+ runs (2 is refused by name), temperature 0 and seed 42 unless you say + otherwise, one model or an A/B of up to three, an optional `num_ctx` sweep, + and a report of per-run values plus median, p95, min/max and relative spread + for tok/s, TTFT and wall. `cold: true` (the default) unloads the model with + `keep_alive: 0` first, so run 1 really pays the load - and the cold/warm + sentence is stated at the **first token**, because `eval_duration` covers + generation only and a cold load never lowers tok/s. +- **`ollama_models`** - `list` and `show` (parameters, template, license, + capabilities, architecture facts by suffix, so `qwen2.context_length` and + `nomic-bert.embedding_length` both resolve), and the two **writes** behind a + confirmation that repeats the tag: `pull` and `delete`. Without `confirm` the + answer is what it WOULD do - the size on disk or the size to download, and + the exact command - and nothing is fetched or removed. +- **`ollama_embed`** - `/api/embed` with a multiple-input form, falling back to + the older `/api/embeddings` on a daemon that lacks it. Reports the dimension, + each vector's norm and a cosine similarity when two inputs make one + meaningful. A completion model is refused **by capability**, not by failing + later. + +## The two rules it is engineering against + +**A refusal is an answer.** No daemon → one sentence naming the address and +`ollama serve`. No model → the tag wanted, the tags that exist, and the exact +`ollama pull`. Model that cannot embed → its capabilities and the usual +alternative. A `num_predict` over the cap is refused, not clamped. An option +name that is a typo is refused, because Ollama **silently ignores** unknown +options and the numbers would then be a claim about a setting nobody applied. + +**Loopback only, by default.** The tools talk to `127.0.0.1:11434`; an +`OLLAMA_HOST` naming another machine is refused before any request is made, +with the one variable that allows it: `DSH_OLLAMA_ALLOW_REMOTE=1`. A daemon on +another box is a real setup, but it is a decision a person makes once, not a +default an agent inherits. (`OLLAMA_HOST=0.0.0.0:11434` is accepted - it is a +local bind address, and it cannot reach another machine.) + +## The three skills + +Each ships as `skills//SKILL.md` and is registered at boot, with its +examples checked: + +- **`ollama-api`** - the REST surface as it is here: `/api/generate`, `/api/chat` + (streaming NDJSON, `done: true` last), `/api/embed` vs `/api/embeddings`, + `/api/tags`, `/api/ps`, `/api/show`, pull progress, create/copy/delete, the + OpenAI-compatible `/v1` endpoints (and the two ways they differ: no timings, + SSE not NDJSON), the `options` map, `keep_alive`, error shapes, the CLI + equivalents, and `OLLAMA_HOST` / `OLLAMA_NUM_PARALLEL` / + `OLLAMA_MAX_LOADED_MODELS` / `OLLAMA_KEEP_ALIVE`. +- **`ollama-inference-metrics`** - what each duration covers, the tok/s and + TTFT arithmetic, why `total_duration` is neither the model span nor the + latency, why a second run is faster to START, what changes with context, + quantization, offload and parallelism, and the reporting rules (5+ runs, a + median with a spread, one variable at a time, a fixed seed, the digest and + the context in the answer). +- **`ollama-python`** - the `ollama` package: `chat()`, `generate()`, + streaming and closing a generator, `embed`, tool calling with a bounded loop, + structured output from a pydantic schema, the async client, `keep_alive`, + timeouts and cancellation - and a project shape whose benchmark arithmetic is + unit-tested against a **fake transport** while the live half skips loudly + when no daemon answers. + +## How it plugs in + +| Piece | Value | +|---|---| +| row / tools | `ollama` / `ollama_status`, `ollama_run`, `ollama_bench`, `ollama_models`, `ollama_embed` | +| host | one row, no route, no browser half (`package.json` declares no `dsh.client`) | +| transport | `globalThis.fetch` to `http://127.0.0.1:11434`, per-request `AbortSignal` deadline, response bodies capped (2 MiB JSON, 16 MiB stream, 500k stream items) | +| process | exactly one: `ollama --version` for the CLI line, argv-only, 4 s, no shell | +| state | none on disk; a 2-second in-memory model-list cache so one answer is one request | +| skills | three, registered at boot and copied into `$DSH_HOME/skills` by both installers | +| credentials | none: no key, no token, no URL with a password (a credentialed `OLLAMA_HOST` is refused) | + +## Limits + +| | | +|---|---| +| daemon | it must already be running; this row starts nothing and says how | +| models | `pull` downloads gigabytes and needs `confirm`; `create`, `copy` and `push` are documented in the API skill but are not tools | +| remote | refused unless `DSH_OLLAMA_ALLOW_REMOTE=1`, and then every timing includes the network | +| measurement | one process, one machine: no VRAM probing beyond `/api/ps`, no GPU driver reading, no thermal control | +| UI | none - this is a tool surface plus skills, not a second conversation | +| `num_predict` | never `-1`: one inference is bounded by the schema and by a timeout | + +## Verify + +```sh +node checks/check-ollama-node.mjs # the whole contract against a fake daemon, plus the skill examples +DSH_OLLAMA_CHECK_LIVE=0 node checks/check-ollama-node.mjs # the pure half only +``` + +The fake daemon (`node:http` on `127.0.0.1:0`) answers the real routes with +canned bodies and **fixed nanosecond durations**, so the arithmetic is +assertions rather than observations: 100 tokens in 2 s is exactly 50.0 tok/s, +and TTFT derived is exactly 1.20 s. It also pins every refusal (a 404 that +names a model vs a 404 that names a route, an over-cap `num_predict`, an +unconfirmed pull, a delete, both prompt and messages, a typo'd option), and the +loopback fence by counting fetches - an off-machine host must make **zero** +requests. + +The skill examples are checked, not just written: every fenced `curl` line is +tokenized into a request (loopback URL, a real endpoint, a JSON body that +parses), every Python block is parsed by python's own `ast` when a Python is on +PATH, every JavaScript block is checked by `node --check`, and every JSON block +is parsed. The check also asserts that every endpoint the row calls appears in +the skills, so a feature cannot ship undocumented. + +The live half runs one tiny generation - and with a model present a 3-run bench +- only when a daemon answers here, and otherwise **skips with the reason**. + +The pack-wide checks still own the row: `check-node-routes.mjs` (the row, the +manifest, the skill registration), `check-no-secrets.mjs`, and the root +README/ARCHITECTURE rows. + +## Install + +Both installers pick it up from `packages/`; as a new bundle the profile learns +it once, then a restart of `npx @deepseek-ai/dsh web` and a hard refresh. diff --git a/packages/dsh-ollama/checks/check-ollama-node.mjs b/packages/dsh-ollama/checks/check-ollama-node.mjs new file mode 100644 index 0000000..d0119a4 --- /dev/null +++ b/packages/dsh-ollama/checks/check-ollama-node.mjs @@ -0,0 +1,929 @@ +// check-ollama-node.mjs - dsh-ollama's own contract, driven end to end. +// +// The daemon is the machine's own service and it is NOT running on a fresh +// machine, so the only way to exercise the arithmetic, the caps, the +// confirmations and every refusal deterministically is a FAKE one: a +// `node:http` server on 127.0.0.1:0 answering the real routes with canned +// bodies and FIXED nanosecond durations. That makes the numbers assertions +// rather than observations: +// +// eval_count 100 / eval_duration 2e9 -> 50.0 tok/s, exactly +// prompt_eval_count 20 / 200e6 -> 100.0 tok/s +// load_duration 1e9 -> 1.00 s, and TTFT derived = 1.20 s +// +// Three sections need the real world, and each says so when it cannot: +// +// - the SKILL EXAMPLES: every fenced curl block is parsed as a request (URL, +// method, JSON body), every Python block is parsed by python's own `ast` +// when a Python is on PATH, and every JavaScript block is checked by +// `node --check`. A doc and its engine cannot drift apart unnoticed. +// - the LIVE half: one tiny generation against the daemon that IS running +// here, only when it answers, and skipped with the reason when it does not. +// +// Run: node checks/check-ollama-node.mjs +// DSH_OLLAMA_CHECK_LIVE=0 node checks/check-ollama-node.mjs # pure half only +import { createServer } from 'node:http' +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +import { ALLOW_REMOTE_ENV, DEFAULT_HOST, isLoopbackHost, resolveHost } from '../lib/client.js' +import { + cosineSimilarity, + formatBytes, + formatExpiry, + formatShare, + measure, + millisecondsOf, + percentile, + summarize, + tokensPerSecond, + vectorNorm, +} from '../lib/metrics.js' +import { CAPS, validateBench, validateEmbed, validateModels, validateOptions, validateRun } from '../lib/validate.js' +import { benchText, benchView, buildTools, pullSummary } from '../lib/tools.js' +import { activationLine, createDeps, parseSkillFile, PLUGIN_VERSION, SKILL_FILES, __internals } from '../lib/index.js' + +const here = path.dirname(fileURLToPath(import.meta.url)) +const packageRoot = path.join(here, '..') +const manifest = JSON.parse(readFileSync(path.join(packageRoot, 'package.json'), 'utf8')) + +let checks = 0 +let failures = 0 +let skipped = 0 +const check = (label, actual, expected) => { + checks += 1 + const ok = expected === undefined ? Boolean(actual) : actual === expected + if (!ok) failures += 1 + console.log((ok ? 'ok ' : 'FAIL ') + label.padEnd(56) + (expected === undefined ? '' : ' ' + JSON.stringify(actual))) + return ok +} +const skip = (label, reason) => { + skipped += 1 + console.log('skip ' + label.padEnd(56) + ' (' + reason + ')') +} +const near = (label, actual, expected, tolerance = 1e-6) => check(label, typeof actual === 'number' && Math.abs(actual - expected) <= tolerance, true) + +// --------------------------------------------------------------------------- +// The manifest +// --------------------------------------------------------------------------- +check('the package is a dsh bundle with a patch layer', manifest.dsh && manifest.dsh.bundle && manifest.dsh.bundle.patch, './cordis.patch.yml') +check('the row owns no browser half', manifest.dsh.client === undefined, true) +check('the row has no npm dependency', manifest.dependencies === undefined && manifest.peerDependencies === undefined, true) +check('the version is the alpha this check knows', manifest.version, PLUGIN_VERSION) +check('package.json is the manifest this check read', manifest.version, '0.1.0-alpha.1') +check('the skills and checks ship in the package', ['lib', 'skills', 'checks', 'cordis.patch.yml'].every((entry) => manifest.files.includes(entry)), true) + +// --------------------------------------------------------------------------- +// The loopback fence - pure, no socket +// --------------------------------------------------------------------------- +const plain = resolveHost({}) +check('with no OLLAMA_HOST the daemon is the loopback default', plain.origin, 'http://' + DEFAULT_HOST) +check('...and it is allowed', plain.allowed && plain.local, true) +check('a bare host:port gets a scheme', resolveHost({ OLLAMA_HOST: 'localhost:11500' }).origin, 'http://localhost:11500') +check('an explicit http:// is kept', resolveHost({ OLLAMA_HOST: 'http://127.0.0.1:12000' }).port, 12000) +check('IPv6 loopback is accepted', resolveHost({ OLLAMA_HOST: 'http://[::1]:11434' }).host, '::1') +check('all of 127/8 is this machine', resolveHost({ OLLAMA_HOST: '127.0.0.2:11434' }).local, true) +check('0.0.0.0 is a local bind address', resolveHost({ OLLAMA_HOST: '0.0.0.0:11434' }).local, true) +check('a hostname is not loopback', isLoopbackHost('example.com'), false) +check('localhost is', isLoopbackHost('localhost'), true) +check('a *.localhost name is', isLoopbackHost('daemon.localhost'), true) +const remote = resolveHost({ OLLAMA_HOST: 'http://10.0.0.5:11434' }) +check('another machine is remote', remote.remote && remote.allowed === false, true) +check('...and refused with a code', remote.code, 'REMOTE_REFUSED') +check('...naming the opt-in variable', remote.reason.includes(ALLOW_REMOTE_ENV), true) +const optedIn = resolveHost({ OLLAMA_HOST: 'http://10.0.0.5:11434', [ALLOW_REMOTE_ENV]: '1' }) +check('the opt-in variable allows it', optedIn.allowed && optedIn.optIn && optedIn.remote, true) +check('a nonsense host is invalid, not remote', resolveHost({ OLLAMA_HOST: 'not a host' }).code, 'HOST_INVALID') +check('a non-http scheme is refused', resolveHost({ OLLAMA_HOST: 'ftp://127.0.0.1:11434' }).code, 'HOST_INVALID') +check('credentials in OLLAMA_HOST are refused', resolveHost({ OLLAMA_HOST: 'http://user:pass@127.0.0.1:11434' }).code, 'HOST_INVALID') +check('port 0 is not a port', resolveHost({ OLLAMA_HOST: '127.0.0.1:0' }).code, 'HOST_INVALID') +check('a base path is preserved', resolveHost({ OLLAMA_HOST: 'http://127.0.0.1:11434/ollama' }).base, 'http://127.0.0.1:11434/ollama') + +// --------------------------------------------------------------------------- +// The arithmetic - pure +// --------------------------------------------------------------------------- +check('2e9 ns is 2 s', millisecondsOf(2_000_000_000), 2000) +check('a missing duration is null, not NaN', millisecondsOf(undefined), null) +check('100 tokens in 2 s is 50 tok/s', tokensPerSecond(100, 2_000_000_000), 50) +check('a zero duration is null, not Infinity', tokensPerSecond(100, 0), null) +check('zero tokens is 0 tok/s', tokensPerSecond(0, 2_000_000_000), 0) +const FIXED = { + eval_count: 100, + eval_duration: 2_000_000_000, + prompt_eval_count: 20, + prompt_eval_duration: 200_000_000, + load_duration: 1_000_000_000, + total_duration: 3_500_000_000, +} +const measured = measure({ ...FIXED, done_reason: 'stop' }, { wallMs: 3610, ttftMs: 1210 }) +check('the measured shape reads every daemon field', measured.evalMs + '/' + measured.promptEvalMs + '/' + measured.loadMs + '/' + measured.totalMs, '2000/200/1000/3500') +check('tok/s is eval_count over eval_duration', measured.evalTokensPerSecond, 50) +check('prompt tok/s is its own pair', measured.promptTokensPerSecond, 100) +check('TTFT derived is load + prompt eval', measured.ttftDerivedMs, 1200) +check('TTFT measured is the observed one', measured.ttftMeasuredMs, 1210) +check('the wall clock is the caller\'s', measured.wallMs, 3610) +check('the stop reason travels', measured.doneReason, 'stop') +check('a one-element sample has a median', percentile([42], 50), 42) +check('the median of five is the middle one', percentile([50, 40, 66.7, 33.3, 100], 50), 50) +check('p95 by nearest rank is a real run', percentile([50, 40, 66.7, 33.3, 100], 95), 100) +check('an empty percentile is null', percentile([], 95), null) +const summary = summarize([33.3, 40, 50, 66.7, 100]) +check('the summary names the sample size', summary.count, 5) +check('...its median', summary.median, 50) +check('...its extremes', summary.min + '..' + summary.max, '33.3..100') +check('...its spread as a relative stdev', near('spread', summary.spread, summary.stdev / summary.mean) && true, true) +check('a one-run sample has no spread to trust', summarize([7]).spread, 0) +check('cosine of orthogonal vectors is 0', cosineSimilarity([1, 0], [0, 1]), 0) +check('cosine of identical vectors is 1', near('cosine', cosineSimilarity([1, 2], [1, 2]), 1), true) +check('cosine of a zero vector is null', cosineSimilarity([0, 0], [1, 1]), null) +check('cosine of mismatched dimensions is null', cosineSimilarity([1], [1, 2]), null) +check('a norm is the L2 length', vectorNorm([3, 4]), 5) +check('bytes read like ollama list', formatBytes(4.7 * 1024 ** 3), '4.7 GB') +check('a share is a percentage', formatShare(3, 4), '75%') +check('an unknown whole has no share', formatShare(3, 0), '') +check('a future expiry is a sentence', formatExpiry(new Date(Date.now() + 5 * 60_000).toISOString()), 'in 5 minutes') +check('a past expiry says so', formatExpiry(new Date(Date.now() - 60_000).toISOString()), 'expired') + +// --------------------------------------------------------------------------- +// The argument grammar - pure +// --------------------------------------------------------------------------- +const refusalCode = (fn) => { + try { + fn() + return 'accepted' + } catch (error) { + return error.code + } +} +check('a model tag is accepted', validateRun({ model: 'qwen2.5:7b', prompt: 'x' }).model, 'qwen2.5:7b') +check('a namespaced tag is accepted', validateRun({ model: 'hf.co/user/model:Q4_K_M', prompt: 'x' }).model, 'hf.co/user/model:Q4_K_M') +check('a space in a tag is refused', refusalCode(() => validateRun({ model: 'Qwen 2', prompt: 'x' })), 'BAD_MODEL') +check('a flag-looking tag is refused', refusalCode(() => validateRun({ model: '--all', prompt: 'x' })), 'BAD_MODEL') +check('a digest is refused outside a pull', refusalCode(() => validateRun({ model: 'qwen2.5:7b@sha256:' + 'a'.repeat(64), prompt: 'x' })), 'BAD_MODEL') +check('a digest is accepted for a pull', validateModels({ action: 'pull', model: 'qwen2.5:7b@sha256:' + 'a'.repeat(64), confirm: 'x' }).model.length > 0, true) +check('prompt and messages together are refused', refusalCode(() => validateRun({ model: 'a:1b', prompt: 'x', messages: [{ role: 'user', content: 'x' }] })), 'BAD_ARGUMENT') +check('neither prompt nor messages is refused', refusalCode(() => validateRun({ model: 'a:1b' })), 'BAD_ARGUMENT') +check('an empty prompt is refused', refusalCode(() => validateRun({ model: 'a:1b', prompt: ' ' })), 'BAD_ARGUMENT') +check('system with messages is refused, with the fix', refusalCode(() => validateRun({ model: 'a:1b', messages: [{ role: 'user', content: 'x' }], system: 's' })), 'BAD_ARGUMENT') +check('a bad role is refused', refusalCode(() => validateRun({ model: 'a:1b', messages: [{ role: 'robot', content: 'x' }] })), 'BAD_MESSAGES') +check('an invented message field is refused', refusalCode(() => validateRun({ model: 'a:1b', messages: [{ role: 'user', content: 'x', weight: 2 }] })), 'BAD_MESSAGES') +check('a data: URL image is refused as not base64', refusalCode(() => validateRun({ model: 'a:1b', messages: [{ role: 'user', content: 'x', images: ['data:image/png;base64,AAAA'] }] })), 'BAD_MESSAGES') +check('an unknown option is refused by name', refusalCode(() => validateOptions({ temprature: 0 })), 'BAD_OPTION') +check('...and the refusal lists the known ones', (() => { try { validateOptions({ temprature: 0 }) } catch (error) { return error.message.includes('temperature') } return false })(), true) +check('num_predict over the ceiling is refused', refusalCode(() => validateOptions({ num_predict: CAPS.maxNumPredict + 1 })), 'BAD_OPTION') +check('num_predict 0 is refused', refusalCode(() => validateOptions({ num_predict: 0 })), 'BAD_OPTION') +check('temperature 3 is refused', refusalCode(() => validateOptions({ temperature: 3 })), 'BAD_OPTION') +check('a fractional integer option is refused', refusalCode(() => validateOptions({ seed: 1.5 })), 'BAD_OPTION') +check('a stop array of nine is refused', refusalCode(() => validateOptions({ stop: new Array(9).fill('x') })), 'BAD_OPTION') +check('a stop string is accepted as a one-element list', validateOptions({ stop: '###' }).stop.join(','), '###') +check('format "yaml" is refused', refusalCode(() => validateRun({ model: 'a:1b', prompt: 'x', format: 'yaml' })), 'BAD_FORMAT') +check('format "json" is accepted', validateRun({ model: 'a:1b', prompt: 'x', format: 'json' }).format, 'json') +check('a JSON Schema format is accepted', typeof validateRun({ model: 'a:1b', prompt: 'x', format: { type: 'object' } }).format, 'object') +check('keep_alive "soon" is refused', refusalCode(() => validateRun({ model: 'a:1b', prompt: 'x', keep_alive: 'soon' })), 'BAD_KEEP_ALIVE') +check('keep_alive 0 is accepted', validateRun({ model: 'a:1b', prompt: 'x', keep_alive: 0 }).keepAlive, 0) +check('keep_alive "5m" is accepted', validateRun({ model: 'a:1b', prompt: 'x', keep_alive: '5m' }).keepAlive, '5m') +check('a timeout outside the range is refused', refusalCode(() => validateRun({ model: 'a:1b', prompt: 'x', timeoutSeconds: 5 })), 'BAD_ARGUMENT') +check('one run always carries a bounded num_predict', validateRun({ model: 'a:1b', prompt: 'x' }).options.num_predict, CAPS.defaultNumPredict) +check('two runs is not a benchmark', refusalCode(() => validateBench({ model: 'a:1b', prompt: 'x', runs: 2 })), 'BAD_RUNS') +check('...and the refusal says why', (() => { try { validateBench({ model: 'a:1b', prompt: 'x', runs: 2 }) } catch (error) { return /measurement/i.test(error.message) } return false })(), true) +check('a bench defaults to 5 runs at temperature 0, seed 42', (() => { const plan = validateBench({ model: 'a:1b', prompt: 'x' }); return plan.runs === 5 && plan.options.temperature === 0 && plan.options.seed === 42 })(), true) +check('a bench refuses model and models together', refusalCode(() => validateBench({ model: 'a:1b', models: ['a:1b'], prompt: 'x' })), 'BAD_ARGUMENT') +check('an A/B of four models is refused', refusalCode(() => validateBench({ models: ['a:1b', 'b:1b', 'c:1b', 'd:1b'], prompt: 'x' })), 'BAD_ARGUMENT') +check('the same model twice is refused', refusalCode(() => validateBench({ models: ['a:1b', 'a:1b'], prompt: 'x' })), 'BAD_ARGUMENT') +check('a duplicate context is refused', refusalCode(() => validateBench({ model: 'a:1b', prompt: 'x', contexts: [2048, 2048] })), 'BAD_ARGUMENT') +check('a plan over 150 calls is refused', refusalCode(() => validateBench({ models: ['a:1b', 'b:1b', 'c:1b'], prompt: 'x', runs: 50, contexts: [2048, 4096, 8192, 16384, 32768, 65536] })), 'BAD_ARGUMENT') +check('a bench timeout of 10 s is refused', refusalCode(() => validateBench({ model: 'a:1b', prompt: 'x', timeoutSeconds: 10 })), 'BAD_ARGUMENT') +check('an embed of 65 inputs is refused', refusalCode(() => validateEmbed({ model: 'e:1', input: new Array(65).fill('x') })), 'BAD_ARGUMENT') +check('an embed of two strings defaults to a similarity', validateEmbed({ model: 'e:1', input: ['a', 'b'] }).similarity, true) +check('an embed of one string does not', validateEmbed({ model: 'e:1', input: ['a'] }).similarity, false) +check('an unknown model action is refused', refusalCode(() => validateModels({ action: 'push' })), 'BAD_ARGUMENT') +check('list needs no model', validateModels({}).model, '') +check('a pull carries a raisable deadline', validateModels({ action: 'pull', model: 'a:1b', confirm: 'a:1b' }).timeoutSeconds, 1800) +check('...and refuses one outside the range', refusalCode(() => validateModels({ action: 'pull', model: 'a:1b', confirm: 'a:1b', timeoutSeconds: 5 })), 'BAD_ARGUMENT') + +// --------------------------------------------------------------------------- +// The fake daemon +// --------------------------------------------------------------------------- +const NOW = '2026-01-02T03:04:05Z' +const BENCH_DURATIONS = [2_000_000_000, 2_500_000_000, 1_500_000_000, 3_000_000_000, 1_000_000_000] +const TAGS = [ + { + name: 'qwen2.5:7b', + model: 'qwen2.5:7b', + size: 4.7 * 1024 ** 3, + digest: 'sha256:' + 'a'.repeat(64), + modified_at: NOW, + details: { format: 'gguf', family: 'qwen2', families: ['qwen2'], parameter_size: '7.6B', quantization_level: 'Q4_K_M' }, + }, + { + name: 'llama3.2:1b', + model: 'llama3.2:1b', + size: 1.3 * 1024 ** 3, + digest: 'sha256:' + 'b'.repeat(64), + modified_at: NOW, + details: { format: 'gguf', family: 'llama', families: ['llama'], parameter_size: '1.2B', quantization_level: 'Q8_0' }, + }, + { + name: 'nomic-embed-text:latest', + model: 'nomic-embed-text:latest', + size: 274 * 1024 ** 2, + digest: 'sha256:' + 'c'.repeat(64), + modified_at: NOW, + details: { format: 'gguf', family: 'nomic-bert', families: ['nomic-bert'], parameter_size: '137M', quantization_level: 'F16' }, + }, +] +const SHOW = { + 'qwen2.5:7b': { + details: { format: 'gguf', family: 'qwen2', families: ['qwen2'], parameter_size: '7.6B', quantization_level: 'Q4_K_M' }, + capabilities: ['completion', 'tools'], + parameters: 'num_ctx 32768\ntemperature 0.7', + template: '{{ if .System }}<|im_start|>system\n{{ .System }}{{ end }}<|im_start|>user\n{{ .Prompt }}<|im_end|>', + license: 'Apache License 2.0', + model_info: { 'general.architecture': 'qwen2', 'qwen2.context_length': 32768, 'qwen2.block_count': 28 }, + }, + 'llama3.2:1b': { + details: { format: 'gguf', family: 'llama', families: ['llama'], parameter_size: '1.2B', quantization_level: 'Q8_0' }, + capabilities: ['completion'], + parameters: 'num_ctx 131072', + template: '{{ .Prompt }}', + license: 'Llama 3.2 Community License', + model_info: { 'general.architecture': 'llama', 'llama.context_length': 131072, 'llama.block_count': 16, 'llama.embedding_length': 2048 }, + }, + 'nomic-embed-text:latest': { + details: { format: 'gguf', family: 'nomic-bert', families: ['nomic-bert'], parameter_size: '137M', quantization_level: 'F16' }, + capabilities: ['embedding'], + parameters: '', + template: '', + license: 'Apache License 2.0', + model_info: { 'general.architecture': 'nomic-bert', 'nomic-bert.context_length': 2048, 'nomic-bert.embedding_length': 768 }, + }, +} + +/** A deterministic 8-dimension vector; 'cat' and 'dog' have a known cosine. */ +function vectorFor(text) { + if (text === 'cat') return [1, 0, 0, 0, 0, 0, 0, 0] + if (text === 'dog') return [1, 1, 0, 0, 0, 0, 0, 0] + const seed = [...String(text)].reduce((sum, character) => sum + character.codePointAt(0), 0) + return Array.from({ length: 8 }, (_, index) => Math.sin(seed * (index + 1))) +} + +/** Read a request body to a string. */ +function readAll(request) { + return new Promise((resolve) => { + let text = '' + request.on('data', (chunk) => { + text += chunk + }) + request.on('end', () => resolve(text)) + }) +} + +/** The daemon, canned. */ +async function startFakeDaemon() { + const state = { legacyEmbeddings: false, benchIndex: 0, unloads: 0, slow: true } + const requests = [] + const server = createServer(async (request, response) => { + const url = new URL(request.url, 'http://127.0.0.1') + const raw = await readAll(request) + let body = null + try { + body = raw === '' ? null : JSON.parse(raw) + } catch (error) { + body = null + } + requests.push({ method: request.method, path: url.pathname, body }) + const json = (status, value) => { + response.writeHead(status, { 'content-type': 'application/json' }) + response.end(JSON.stringify(value)) + } + const ndjson = () => { + response.writeHead(200, { 'content-type': 'application/x-ndjson' }) + return (value) => response.write(JSON.stringify(value) + '\n') + } + const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + + if (url.pathname === '/api/version') return json(200, { version: '0.40.3' }) + if (url.pathname === '/api/tags') return json(200, { models: TAGS }) + if (url.pathname === '/api/ps') { + return json(200, { + models: [ + { + name: 'qwen2.5:7b', + model: 'qwen2.5:7b', + size: 7.3 * 1024 ** 3, + size_vram: 7.3 * 1024 ** 3, + digest: 'sha256:' + 'a'.repeat(64), + details: TAGS[0].details, + expires_at: new Date(Date.now() + 5 * 60_000).toISOString(), + context_length: 8192, + }, + ], + }) + } + if (url.pathname === '/api/show') { + const model = String(body?.model ?? '') + const found = SHOW[model] + if (found === undefined) return json(404, { error: 'model "' + model + '" not found, try pulling it first' }) + return json(200, found) + } + if (url.pathname === '/api/generate') { + const model = String(body?.model ?? '') + if (model === 'missing:1b') return json(404, { error: 'model "missing:1b" not found, try pulling it first' }) + if (body?.keep_alive === 0 && (body.prompt === undefined || body.prompt === '')) { + state.unloads += 1 + return json(200, { model, created_at: NOW, response: '', done: true }) + } + if (body?.prompt === 'bench') { + const duration = BENCH_DURATIONS[state.benchIndex % BENCH_DURATIONS.length] + state.benchIndex += 1 + return json(200, { + model, + created_at: NOW, + response: '', + done: true, + done_reason: 'stop', + eval_count: 100, + eval_duration: duration, + prompt_eval_count: 20, + prompt_eval_duration: 200_000_000, + load_duration: state.benchIndex === 1 ? 1_000_000_000 : 0, + total_duration: duration + 200_000_000, + }) + } + const write = ndjson() + // The prompt-eval delay comes BEFORE the first token, which is what makes + // a measured TTFT a measurement rather than the transport's first flush. + if (state.slow) await wait(12) + write({ model, created_at: NOW, response: 'Hel', done: false }) + if (state.slow) await wait(6) + write({ model, created_at: NOW, response: 'lo', done: false }) + write({ model, created_at: NOW, response: '', done: true, done_reason: 'stop', ...FIXED }) + return response.end() + } + if (url.pathname === '/api/chat') { + const write = ndjson() + write({ model: body?.model, created_at: NOW, message: { role: 'assistant', content: 'Hi' }, done: false }) + write({ model: body?.model, created_at: NOW, message: { role: 'assistant', content: ' there' }, done: false }) + write({ model: body?.model, created_at: NOW, message: { role: 'assistant', content: '' }, done: true, done_reason: 'stop', ...FIXED }) + return response.end() + } + if (url.pathname === '/api/embed') { + if (state.legacyEmbeddings) return json(404, { error: '404 page not found' }) + const inputs = Array.isArray(body?.input) ? body.input : [body?.input ?? ''] + return json(200, { model: body?.model, embeddings: inputs.map(vectorFor) }) + } + if (url.pathname === '/api/embeddings') { + return json(200, { embedding: vectorFor(String(body?.prompt ?? '')) }) + } + if (url.pathname === '/api/pull') { + const write = ndjson() + write({ status: 'pulling manifest' }) + write({ status: 'pulling 966de95ca8a6', digest: 'sha256:966de95ca8a6', total: 1_000_000, completed: 400_000 }) + write({ status: 'pulling 966de95ca8a6', digest: 'sha256:966de95ca8a6', total: 1_000_000, completed: 1_000_000 }) + write({ status: 'pulling f70b4c8b5a5d', digest: 'sha256:f70b4c8b5a5d', total: 2_000_000, completed: 2_000_000 }) + write({ status: 'verifying sha256 digest' }) + write({ status: 'success' }) + return response.end() + } + if (url.pathname === '/api/delete') { + if (String(body?.model ?? '') === 'missing:1b') return json(404, { error: 'model "missing:1b" not found' }) + return json(200, {}) + } + return json(404, { error: '404 page not found' }) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const port = server.address().port + return { + origin: 'http://127.0.0.1:' + port, + state, + requests, + close: () => new Promise((resolve) => server.close(resolve)), + } +} + +const daemon = await startFakeDaemon() +const logged = { info() {}, warn() {} } +const stubCtx = { logger: logged, get: () => undefined, effect: (fn) => fn() } +/** The dependencies every tool test uses: the fake daemon, a fixed CLI version. */ +const makeDeps = (overrides = {}) => + createDeps(stubCtx, { + env: { OLLAMA_HOST: daemon.origin }, + runner: async () => ({ status: 0, stdout: '', stderr: 'Warning: could not connect to a running Ollama instance\nWARNING: client version is 0.40.3\n', timedOut: false }), + ...overrides, + }) +const deps = makeDeps() +const tools = buildTools(deps) +const tool = (name) => tools.find((entry) => entry.name === name) +const exec = { signal: new AbortController().signal } +const resetBench = () => { + daemon.state.benchIndex = 0 + daemon.state.unloads = 0 +} + +try { + // ------------------------------------------------------------------------- + // The registration + // ------------------------------------------------------------------------- + check('five tools are registered', tools.map((entry) => entry.name).join(','), __internals.CAPABILITIES.tools.join(',')) + check( + 'every tool declares a JSON-schema surface', + tools.every((entry) => entry.parameters && entry.parameters.type === 'object' && Object.keys(entry.parameters.properties ?? {}).length >= 0 && entry.output && typeof entry.output.render === 'function' && typeof entry.execute === 'function'), + true, + ) + check('the row asks only for the tools registry', __internals.CAPABILITIES.tools.length === 5) + check('both generation tools share the closed options schema', tool('ollama_run').parameters.properties.options === tool('ollama_bench').parameters.properties.options, true) + check('...which refuses an option it does not declare', tool('ollama_run').parameters.properties.options.additionalProperties, false) + check('no route is declared', __internals.CAPABILITIES.endpoints.every((entry) => entry.startsWith('/api/')), true) + check('the capabilities name what is not built', __internals.CAPABILITIES.notBuilt.some((entry) => /daemon/i.test(entry)), true) + check('the three skills are listed', __internals.CAPABILITIES.skills.join(','), 'ollama-api,ollama-inference-metrics,ollama-python') + + // ------------------------------------------------------------------------- + // ollama_status + // ------------------------------------------------------------------------- + const status = await tool('ollama_status').execute({}, exec) + check('status reaches the fake daemon', status.view.ok, true) + check('...and reports its version', status.view.daemon.version, '0.40.3') + check('...and the CLI version it read', status.view.cli, '0.40.3') + check('...and the models on disk', status.view.counts.models, 3) + check('...their total bytes', status.view.counts.bytes, TAGS.reduce((sum, entry) => sum + entry.size, 0)) + check('...and the one loaded model', status.view.counts.loaded, 1) + check('the table prints family, quant and size', status.text.includes('qwen2.5:7b') && status.text.includes('Q4_K_M') && status.text.includes('4.7 GB'), true) + check('the loaded model names its processor split', status.view.loaded[0].processor, '100% GPU') + check('the loaded model names its context', status.view.loaded[0].contextLength, 8192) + check('the loaded model names its expiry', status.view.loaded[0].expiresAt !== '' && status.text.includes('in 5 minutes'), true) + check('the digest is shown short', status.text.includes('a'.repeat(12)), true) + check('the next move is named', /ollama_bench/.test(status.text), true) + check('status did not start anything', daemon.requests.every((entry) => !/serve|start/.test(entry.path)), true) + + // A daemon that is NOT running: one sentence, and the command that starts it. + const closed = makeDeps({ env: { OLLAMA_HOST: '127.0.0.1:1' } }) + const downAnswer = await buildTools(closed).find((entry) => entry.name === 'ollama_status').execute({}, exec) + check('a closed port is an ANSWER, not a throw', downAnswer.view.code, 'DAEMON_DOWN') + check('...naming the address it tried', downAnswer.text.includes('127.0.0.1:1'), true) + check('...and the command that starts one', downAnswer.text.includes('ollama serve'), true) + check('...and saying nothing was started', /Nothing was started/.test(downAnswer.text), true) + + // The fence: an off-machine host never reaches fetch at all. + let fetched = 0 + const spied = makeDeps({ env: { OLLAMA_HOST: 'http://10.0.0.5:11434' }, fetch: async () => (fetched += 1, new Response('{}')) }) + const fenced = await buildTools(spied).find((entry) => entry.name === 'ollama_status').execute({}, exec) + check('an off-machine daemon is refused', fenced.view.code, 'REMOTE_REFUSED') + check('...and NO request was made', fetched, 0) + check('...and the sentence names the opt-in', fenced.text.includes(ALLOW_REMOTE_ENV), true) + const activation = activationLine(makeDeps()) + check('the activation line names the loopback address', activation.includes('127.0.0.1') && activation.includes('ollama_status'), true) + check('the activation line says nothing is started', /nothing is started/.test(activation), true) + check('an inert row says why', /inert:/.test(activationLine(makeDeps({ env: { OLLAMA_HOST: 'http://10.0.0.9:11434' } }))), true) + + // ------------------------------------------------------------------------- + // ollama_run + // ------------------------------------------------------------------------- + const run = await tool('ollama_run').execute({ model: 'qwen2.5:7b', prompt: 'Say hello' }, exec) + check('run answers', run.view.ok, true) + check('...with the generated text', run.view.text, 'Hello') + check('...with the eval token count', run.view.measured.evalCount, 100) + check('...with 50 tok/s from the fixed durations', run.view.measured.evalTokensPerSecond, 50) + check('...with the prompt count and rate', run.view.measured.promptEvalCount + '/' + run.view.measured.promptTokensPerSecond, '20/100') + check('...with the load time', run.view.measured.loadMs, 1000) + check('...with the daemon total', run.view.measured.totalMs, 3500) + check('...and the digest it ran', run.view.digest, 'sha256:' + 'a'.repeat(64)) + check('...and a measured TTFT', typeof run.view.measured.ttftMeasuredMs === 'number' && run.view.measured.ttftMeasuredMs >= 8, true) + check('...that is earlier than the wall clock', run.view.measured.ttftMeasuredMs < run.view.measured.wallMs, true) + check('the prose prints the rate', run.text.includes('50.0 tok/s'), true) + check('the prose prints both totals', run.text.includes('(daemon)') && run.text.includes('(wall)'), true) + check('the prose carries the conditions', /Conditions: num_predict 512/.test(run.text), true) + check('the prose says one run is not a measurement', /A single run is a single run/.test(run.text), true) + check('the stop reason travels to the prose', /stop reason: stop/.test(run.text), true) + check('the run went to /api/generate', daemon.requests.some((entry) => entry.path === '/api/generate' && entry.body?.prompt === 'Say hello' && entry.body?.stream === true), true) + check('the default num_predict was sent', daemon.requests.filter((entry) => entry.body?.prompt === 'Say hello').pop().body.options.num_predict, 512) + + const chat = await tool('ollama_run').execute({ model: 'qwen2.5:7b', messages: [{ role: 'user', content: 'hi' }] }, exec) + check('a messages call goes to /api/chat', daemon.requests.some((entry) => entry.path === '/api/chat'), true) + check('...and joins the streamed content', chat.view.text, 'Hi there') + check('...with the same numbers', chat.view.measured.evalTokensPerSecond, 50) + + const structured = await tool('ollama_run').execute({ model: 'qwen2.5:7b', prompt: 'x', format: { type: 'object', properties: { ok: { type: 'boolean' } } } }, exec) + check('a JSON Schema format is passed through', daemon.requests.filter((entry) => entry.body?.prompt === 'x').pop().body.format.type, 'object') + check('...and the run still answers', structured.view.ok, true) + + const missing = await tool('ollama_run').execute({ model: 'missing:1b', prompt: 'x' }, exec) + check('a model that is not pulled is a typed refusal', missing.view.code, 'NO_MODEL') + check('...listing the tags that are on disk', missing.text.includes('qwen2.5:7b') && missing.text.includes('llama3.2:1b'), true) + check('...and the exact pull command', missing.text.includes('ollama pull missing:1b'), true) + + const badOption = await tool('ollama_run').execute({ model: 'qwen2.5:7b', prompt: 'x', options: { num_predict: 99999 } }, exec) + check('an over-cap num_predict is refused, not clamped', badOption.view.code, 'BAD_OPTION') + const typo = await tool('ollama_run').execute({ model: 'qwen2.5:7b', prompt: 'x', options: { temprature: 0 } }, exec) + check('a typo in an option name is refused', typo.view.code, 'BAD_OPTION') + check('...and the sentence explains what a typo would cost', /silently ignores/i.test(typo.text), true) + const both = await tool('ollama_run').execute({ model: 'qwen2.5:7b', prompt: 'x', messages: [{ role: 'user', content: 'y' }] }, exec) + check('prompt and messages together are refused', both.view.code, 'BAD_ARGUMENT') + + // A generation the model cuts at num_predict says so. + const cut = await tool('ollama_run').execute({ model: 'qwen2.5:7b', prompt: 'cut', options: { num_predict: 4 } }, exec) + check('a completed run is not told it was cut', /was cut at the model/.test(cut.text), false) + + // ------------------------------------------------------------------------- + // ollama_bench + // ------------------------------------------------------------------------- + resetBench() + const bench = await tool('ollama_bench').execute({ model: 'qwen2.5:7b', prompt: 'bench', runs: 5, timeoutSeconds: 120 }, exec) + check('bench answers', bench.view.ok, true) + check('...with five runs', bench.view.groups[0].runs.length, 5) + check('...and the median of the fixed rates', bench.view.groups[0].rates.median, 50) + check('...and p95 by nearest rank', bench.view.groups[0].rates.p95, 100) + check('...and the extremes', bench.view.groups[0].rates.min + '/' + bench.view.groups[0].rates.max, '33.333333333333336/100') + check('the model was unloaded before run 1', daemon.state.unloads >= 1, true) + check('...and the view says so', bench.view.groups[0].unloaded, true) + check('the prose prints a per-run table', /RUN\s+TOK\/S/.test(bench.text), true) + check('the prose prints the median and spread', /median 50\.0/.test(bench.text) && /spread/.test(bench.text), true) + check('the prose names the digest', bench.text.includes('a'.repeat(12)), true) + check('the prose names the context (the model default)', /context model default/.test(bench.text), true) + check('the prose calls tok/s generation-only', /generation phase only/.test(bench.text), true) + check('the prose states the cold/warm comparison at the first token', /first run first token at .*warm runs 2-5 at .*median/.test(bench.text), true) + check('...and how much later the cold first token was', /x later/.test(bench.text), true) + check('the prose states the run count', /5 measured run\(s\)|Runs: 5 measured/.test(bench.text), true) + const benchRequest = daemon.requests.filter((entry) => entry.body?.prompt === 'bench').pop() + check('a bench sends temperature 0 and seed 42', benchRequest.body.options.temperature === 0 && benchRequest.body.options.seed === 42, true) + check('a bench sends its own num_predict', benchRequest.body.options.num_predict, CAPS.benchNumPredict) + + resetBench() + const ab = await tool('ollama_bench').execute({ models: ['qwen2.5:7b', 'llama3.2:1b'], prompt: 'bench', runs: 3, cold: false, timeoutSeconds: 120 }, exec) + check('an A/B answers one group per model', ab.view.groups.length, 2) + check('...naming both models', /qwen2\.5:7b vs llama3\.2:1b/.test(ab.text), true) + check('...with three runs each', ab.view.groups.every((group) => group.runs.length === 3), true) + check('cold:false leaves the weights loaded', daemon.state.unloads, 0) + check('...and no group claims an unload', ab.view.groups.every((group) => group.unloaded === false), true) + + resetBench() + const sweep = await tool('ollama_bench').execute({ model: 'qwen2.5:7b', prompt: 'bench', runs: 3, contexts: [2048, 4096], timeoutSeconds: 120 }, exec) + check('a context sweep answers one group per context', sweep.view.groups.map((group) => group.context).join(','), '2048,4096') + check('...and says each context in the prose', sweep.text.includes('context 2048') && sweep.text.includes('context 4096'), true) + const sweptContexts = daemon.requests.filter((entry) => entry.body?.prompt === 'bench').map((entry) => entry.body.options.num_ctx) + check('...and sends num_ctx on every run', sweptContexts.includes(2048) && sweptContexts.includes(4096), true) + + resetBench() + const shortBench = await tool('ollama_bench').execute({ model: 'qwen2.5:7b', prompt: 'bench', runs: 2 }, exec) + check('two runs is refused in words', shortBench.view.code, 'BAD_RUNS') + check('...and the words explain it', /measurement/i.test(shortBench.text), true) + const tooBig = await tool('ollama_bench').execute({ models: ['qwen2.5:7b', 'llama3.2:1b', 'nomic-embed-text:latest'], prompt: 'bench', runs: 50, contexts: [2048, 4096, 8192, 16384] }, exec) + check('an over-150-call plan is refused', tooBig.view.code, 'BAD_ARGUMENT') + + // The aggregate functions are separately exportable and separately true. + const viewText = benchText( + { runs: 2, measuredCalls: 2, warmupCalls: 0, cold: false, timeoutSeconds: 120, timeoutMs: 120_000, options: { num_predict: 128 } }, + [{ model: 'm:1b', digest: '', context: null, unloaded: false, runs: [{ index: 1, measured: measure(FIXED) }, { index: 2, measured: measure(FIXED) }] }], + resolveHost({}), + ) + check('the bench text is exported for the check', viewText.includes('BENCH m:1b'), true) + check('the bench view carries the summaries', benchView({ runs: 2, measuredCalls: 2, cold: false, options: {} }, [], resolveHost({})).runs, 2) + const pulled = pullSummary([ + { status: 'pulling manifest' }, + { status: 'pulling a', digest: 'sha256:a', total: 100, completed: 40 }, + { status: 'pulling a', digest: 'sha256:a', total: 100, completed: 100 }, + { status: 'success' }, + ]) + check('a pull summary sums its layers once', pulled.layers + '/' + pulled.total + '/' + pulled.completed, '1/100/100') + check('...and keeps the final status', pulled.status, 'success') + + // ------------------------------------------------------------------------- + // ollama_models + // ------------------------------------------------------------------------- + const list = await tool('ollama_models').execute({ action: 'list' }, exec) + check('list answers', list.view.count, 3) + check('...with family and quantization', list.text.includes('nomic-bert') && list.text.includes('Q8_0'), true) + check('...and the total bytes', list.text.includes('6.3 GB') || list.text.includes('6.2 GB') || /model\(s\), .* on disk/.test(list.text), true) + const show = await tool('ollama_models').execute({ action: 'show', model: 'qwen2.5:7b' }, exec) + check('show answers with the capabilities', show.text.includes('completion, tools'), true) + check('...the context length', show.text.includes('context 32768'), true) + check('...the parameters the model ships with', show.text.includes('temperature 0.7'), true) + check('...and the license', show.text.includes('Apache License'), true) + const showEmbed = await tool('ollama_models').execute({ action: 'show', model: 'nomic-embed-text:latest' }, exec) + check('an embedding model reports the dimensions', showEmbed.text.includes('768 dimensions'), true) + const showCompletion = await tool('ollama_models').execute({ action: 'show', model: 'llama3.2:1b' }, exec) + check('a completion model calls its hidden size a hidden size', /hidden size\s+2048 dimensions/.test(showCompletion.text), true) + check('...and never claims to make embeddings', /embedding\s+2048/.test(showCompletion.text), false) + + const pullPreview = await tool('ollama_models').execute({ action: 'pull', model: 'qwen2.5:7b' }, exec) + check('a pull without confirm is refused', pullPreview.view.code, 'CONFIRM_REQUIRED') + check('...and says nothing was downloaded', /Nothing was downloaded/.test(pullPreview.text), true) + check('...and shows the size it already has', pullPreview.text.includes('already on disk') && pullPreview.text.includes('4.7 GB'), true) + check('...and names the exact confirm call', pullPreview.text.includes('confirm "qwen2.5:7b"'), true) + check('...and made no pull request', daemon.requests.some((entry) => entry.path === '/api/pull'), false) + const pullMissingPreview = await tool('ollama_models').execute({ action: 'pull', model: 'newmodel:7b' }, exec) + check('a preview for an absent model says so', pullMissingPreview.text.includes('not on disk yet'), true) + const silentPreview = await buildTools(closed).find((entry) => entry.name === 'ollama_models').execute({ action: 'pull', model: 'newmodel:7b' }, exec) + check('a preview on a silent daemon does not claim the model is absent', silentPreview.text.includes('could not check'), true) + const pullDone = await tool('ollama_models').execute({ action: 'pull', model: 'qwen2.5:7b', confirm: 'qwen2.5:7b' }, exec) + check('a confirmed pull runs', pullDone.view.ok && pullDone.view.layers >= 2, true) + check('...and reports the bytes it moved', pullDone.view.bytes, 3_000_000) + check('...and the final status', pullDone.view.status, 'success') + check('...and the prose says what to do next', /ollama_bench/.test(pullDone.text), true) + + const deletePreview = await tool('ollama_models').execute({ action: 'delete', model: 'llama3.2:1b' }, exec) + check('a delete without confirm is refused', deletePreview.view.code, 'CONFIRM_REQUIRED') + check('...and shows what would be freed', deletePreview.text.includes('1.3 GB'), true) + check('...and made no delete request', daemon.requests.some((entry) => entry.method === 'DELETE'), false) + const deleted = await tool('ollama_models').execute({ action: 'delete', model: 'llama3.2:1b', confirm: 'llama3.2:1b' }, exec) + check('a confirmed delete runs', deleted.view.deleted, true) + check('...and says a pull brings it back', /ollama pull llama3\.2:1b/.test(deleted.text), true) + const deleteMissing = await tool('ollama_models').execute({ action: 'delete', model: 'missing:1b', confirm: 'missing:1b' }, exec) + check('deleting what is not there is typed', deleteMissing.view.code, 'NO_MODEL') + + // ------------------------------------------------------------------------- + // ollama_embed + // ------------------------------------------------------------------------- + const notEmbedding = await tool('ollama_embed').execute({ model: 'qwen2.5:7b', input: ['cat', 'dog'] }, exec) + check('a completion model is refused as an embedder', notEmbedding.view.code, 'NOT_EMBEDDING') + check('...naming nomic-embed-text', notEmbedding.text.includes('nomic-embed-text'), true) + check('...and the capabilities it does have', notEmbedding.text.includes('completion'), true) + const embedded = await tool('ollama_embed').execute({ model: 'nomic-embed-text:latest', input: ['cat', 'dog'] }, exec) + check('an embedding answers', embedded.view.ok, true) + check('...with the dimension', embedded.view.dimension, 8) + check('...one vector per input', embedded.view.count, 2) + check('...and the cosine of the pair', near('cosine', embedded.view.similarity, Math.SQRT1_2), true) + check('...printed in the prose', /cosine\(1, 2\) = 0\.7071/.test(embedded.text), true) + check('...with the norms', embedded.view.norms.every((value) => typeof value === 'number'), true) + check('...and a vector preview, not the wall', /of 8\)|\[1\.0000, 0\.0000/.test(embedded.text), true) + const one = await tool('ollama_embed').execute({ model: 'nomic-embed-text:latest', input: 'cat' }, exec) + check('one input has no similarity claim', one.view.similarity, null) + check('...and says nothing about cosine', /cosine/.test(one.text), false) + daemon.state.legacyEmbeddings = true + const legacy = await tool('ollama_embed').execute({ model: 'nomic-embed-text:latest', input: ['cat', 'dog'] }, exec) + check('an older daemon falls back to /api/embeddings', legacy.view.route, '/api/embeddings') + check('...and still answers both inputs', legacy.view.count, 2) + daemon.state.legacyEmbeddings = false + + // ------------------------------------------------------------------------- + // The skills: frontmatter, examples that PARSE, and the endpoint coverage + // ------------------------------------------------------------------------- + const skillsRoot = path.join(packageRoot, 'skills') + const docs = [] + for (const entry of readdirSync(skillsRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) continue + const file = path.join(skillsRoot, entry.name, 'SKILL.md') + const text = readFileSync(file, 'utf8') + const parsed = parseSkillFile(text) + check('skill ' + entry.name + ' has frontmatter with a name', parsed.meta.name, entry.name) + check('skill ' + entry.name + ' has a description', typeof parsed.meta.description === 'string' && parsed.meta.description.length > 40, true) + check('skill ' + entry.name + ' has a body', parsed.content.length > 1000, true) + docs.push({ name: entry.name, file, text }) + for (const nested of readdirSync(path.join(skillsRoot, entry.name), { withFileTypes: true })) { + if (nested.isFile() && nested.name.endsWith('.md') && nested.name !== 'SKILL.md') docs.push({ name: entry.name, file: path.join(skillsRoot, entry.name, nested.name), text: readFileSync(path.join(skillsRoot, entry.name, nested.name), 'utf8') }) + if (nested.isDirectory()) { + for (const deep of readdirSync(path.join(skillsRoot, entry.name, nested.name), { withFileTypes: true })) { + if (deep.isFile() && deep.name.endsWith('.md')) { + const full = path.join(skillsRoot, entry.name, nested.name, deep.name) + docs.push({ name: entry.name, file: full, text: readFileSync(full, 'utf8') }) + } + } + } + } + } + check('the three skills ship as documents', new Set(docs.filter((doc) => doc.file.endsWith('SKILL.md')).map((doc) => doc.name)).size, 3) + + /** The fenced blocks of one document, with the no-check convention. */ + function fences(text) { + const lines = String(text).split('\n') + const blocks = [] + let open = null + for (let index = 0; index < lines.length; index += 1) { + const match = /^\s*```([A-Za-z0-9_+-]*)\s*(.*)$/.exec(lines[index]) + if (!match) { + if (open) open.body.push(lines[index]) + continue + } + if (open === null) { + open = { lang: match[1].toLowerCase(), line: index + 1, body: [], skip: /no-check/i.test(match[2]) || /no-check/i.test(lines[index - 1] ?? '') } + } else { + blocks.push(open) + open = null + } + } + return blocks + } + + /** Split one shell line into tokens, honouring single and double quotes. */ + function tokenize(line) { + const tokens = [] + let current = '' + let quote = '' + let started = false + for (let index = 0; index < line.length; index += 1) { + const character = line[index] + if (quote !== '') { + if (character === quote) quote = '' + else if (character === '\\' && quote === '"' && index + 1 < line.length) { + current += line[index + 1] + index += 1 + } else current += character + continue + } + if (character === "'" || character === '"') { + quote = character + started = true + continue + } + if (/\s/.test(character)) { + if (started || current !== '') tokens.push(current) + current = '' + started = false + continue + } + current += character + started = true + } + if (started || current !== '') tokens.push(current) + return tokens + } + + /** Every `curl ...` line in a block, as a request the daemon would accept. */ + function curlRequests(body) { + const found = [] + for (const line of String(body).split('\n')) { + const trimmed = line.trim().replace(/\\$/, '').trim() + if (!/^curl\s/.test(trimmed)) continue + const tokens = tokenize(trimmed) + let method = '' + let url = '' + let data = '' + let header = '' + for (let index = 1; index < tokens.length; index += 1) { + const token = tokens[index] + if (token === '-X' || token === '--request') method = tokens[++index] ?? '' + else if (token === '-d' || token === '--data' || token === '--data-raw' || token === '--data-binary') data = tokens[++index] ?? '' + else if (token === '-H' || token === '--header') header = tokens[++index] ?? '' + else if (token.startsWith('http')) url = token + } + found.push({ method, url, data, header }) + } + return found + } + + // The endpoints a CURL EXAMPLE may name: the surface this row calls, the + // OpenAI-compatible aliases, and the two model-store verbs the API skill + // documents for completeness (copy/create) although no tool here issues them. + const knownPaths = __internals.CAPABILITIES.endpoints.concat(['/v1/chat/completions', '/v1/completions', '/v1/embeddings', '/v1/models', '/api/copy', '/api/create']) + const curlBlocks = docs.flatMap((doc) => fences(doc.text).filter((block) => ['bash', 'sh', 'shell', 'console'].includes(block.lang)).map((block) => ({ doc, block }))) + let curlCount = 0 + for (const { doc, block } of curlBlocks) { + if (block.skip) { + skipped += 1 + continue + } + const where = path.relative(packageRoot, doc.file) + ':' + block.line + const requests = curlRequests(block.body.join('\n')) + if (requests.length === 0) { + // A shell block that is not a curl example (a CLI invocation) is fine, but + // it must be a real ollama subcommand rather than a made-up one. + check(where + ' has a curl request or an ollama command', /(^|\s)(ollama\s+[a-z]+|curl\s)/m.test(block.body.join('\n')) || block.body.join('\n').trim() === '', true) + continue + } + for (const request of requests) { + curlCount += 1 + check(where + ' names a loopback URL', /^http:\/\/(127\.0\.0\.1|localhost):11434\//.test(request.url), true) + const route = request.url.replace(/^http:\/\/(127\.0\.0\.1|localhost):11434/, '').split('?')[0] + check(where + ' names a real endpoint (' + route + ')', knownPaths.includes(route), true) + if (request.data !== '') { + let parsedBody = null + try { + parsedBody = JSON.parse(request.data) + } catch (error) { + parsedBody = null + } + check(where + ' has a JSON body that parses', parsedBody !== null && typeof parsedBody === 'object', true) + if (parsedBody !== null && route === '/api/generate') check(where + ' names a model in /api/generate', typeof parsedBody.model === 'string' && parsedBody.model !== '', true) + if (parsedBody !== null && route === '/api/pull') check(where + ' names a model in /api/pull', typeof parsedBody.model === 'string', true) + } + } + } + check('the skills carry curl examples', curlCount >= 8, true) + + // The docs must cover the surface the row actually speaks: an endpoint with no + // lesson is a feature the model cannot use. + const allSkillText = docs.map((doc) => doc.text).join('\n') + const undocumented = __internals.CAPABILITIES.endpoints.filter((endpoint) => !allSkillText.includes(endpoint)) + check('every endpoint this row calls is documented', undocumented.join(', '), '') + for (const term of ['keep_alive', 'eval_duration', 'OLLAMA_HOST', 'OLLAMA_NUM_PARALLEL', 'OLLAMA_MAX_LOADED_MODELS', 'OLLAMA_KEEP_ALIVE', 'num_ctx', 'temperature', 'seed']) { + check('the docs teach ' + term, allSkillText.includes(term), true) + } + check('the metrics skill teaches the tok/s arithmetic', /eval_count\s*\/\s*eval_duration|eval_count \/ eval_duration/.test(allSkillText), true) + + // Python: parsed by python's own ast, when this host has a Python. + const python = (() => { + for (const command of process.platform === 'win32' ? ['python', 'py'] : ['python3', 'python']) { + const probe = spawnSync(command, ['--version'], { encoding: 'utf8' }) + if (probe.status === 0) return command + } + return null + })() + const pythonBlocks = docs.flatMap((doc) => fences(doc.text).filter((block) => block.lang === 'python').map((block) => ({ doc, block }))) + check('the python skill carries python examples', pythonBlocks.length >= 4, true) + if (python === null) { + skip('every python example parses', 'no python on PATH') + } else { + const temp = mkdtempSync(path.join(os.tmpdir(), 'dsh-ollama-py-')) + try { + for (const { doc, block } of pythonBlocks) { + if (block.skip) { + skipped += 1 + continue + } + const where = path.relative(packageRoot, doc.file) + ':' + block.line + const file = path.join(temp, 'example-' + block.line + '.py') + writeFileSync(file, block.body.join('\n'), 'utf8') + const parsed = spawnSync(python, ['-c', 'import ast,sys;ast.parse(open(sys.argv[1],encoding="utf-8").read())', file], { encoding: 'utf8' }) + check(where + ' parses as Python', parsed.status === 0, true) + if (parsed.status !== 0) console.log(' ' + String(parsed.stderr ?? '').split('\n').slice(0, 4).join('\n ')) + } + } finally { + rmSync(temp, { recursive: true, force: true }) + } + } + + // JavaScript: checked by node's own parser, as an ES module. + const jsBlocks = docs.flatMap((doc) => fences(doc.text).filter((block) => ['javascript', 'js', 'mjs'].includes(block.lang)).map((block) => ({ doc, block }))) + check('the skills carry a JavaScript example', jsBlocks.length >= 1, true) + const jsTemp = mkdtempSync(path.join(os.tmpdir(), 'dsh-ollama-js-')) + try { + for (const { doc, block } of jsBlocks) { + if (block.skip) { + skipped += 1 + continue + } + const where = path.relative(packageRoot, doc.file) + ':' + block.line + const file = path.join(jsTemp, 'example-' + block.line + '.mjs') + writeFileSync(file, block.body.join('\n'), 'utf8') + const parsed = spawnSync(process.execPath, ['--check', file], { encoding: 'utf8' }) + check(where + ' parses as an ES module', parsed.status === 0, true) + if (parsed.status !== 0) console.log(' ' + String(parsed.stderr ?? '').split('\n').slice(0, 4).join('\n ')) + } + } finally { + rmSync(jsTemp, { recursive: true, force: true }) + } + + // JSON: every fenced block is a real document. + const jsonBlocks = docs.flatMap((doc) => fences(doc.text).filter((block) => block.lang === 'json').map((block) => ({ doc, block }))) + for (const { doc, block } of jsonBlocks) { + const where = path.relative(packageRoot, doc.file) + ':' + block.line + let parsed = null + try { + parsed = JSON.parse(block.body.join('\n')) + } catch (error) { + parsed = null + } + check(where + ' is valid JSON', parsed !== null, true) + } + + // ------------------------------------------------------------------------- + // The live half: only when a daemon answers here. + // ------------------------------------------------------------------------- + const liveEnabled = process.env.DSH_OLLAMA_CHECK_LIVE !== '0' + const liveHost = resolveHost(process.env) + if (!liveEnabled) { + skip('the live half', 'DSH_OLLAMA_CHECK_LIVE=0') + } else { + let liveVersion = null + try { + const response = await fetch(liveHost.origin + '/api/version', { signal: AbortSignal.timeout(2000) }) + if (response.ok) liveVersion = (await response.json()).version + } catch (error) { + liveVersion = null + } + if (liveVersion === null) { + skip('the live half', 'no Ollama daemon answers at ' + liveHost.origin + ' (start one with `ollama serve`)') + } else { + console.log('') + console.log('--- live: a daemon answers at ' + liveHost.origin + ' (version ' + liveVersion + ')') + const liveDeps = createDeps(stubCtx, {}) + const liveTools = buildTools(liveDeps) + const live = (name) => liveTools.find((entry) => entry.name === name) + const tags = await fetch(liveHost.origin + '/api/tags', { signal: AbortSignal.timeout(4000) }).then((response) => response.json()) + const candidates = (Array.isArray(tags.models) ? tags.models : []).filter((entry) => !/embed/i.test(String(entry.name ?? ''))).sort((left, right) => (left.size ?? 0) - (right.size ?? 0)) + if (candidates.length === 0) { + skip('live: one generation', 'no completion model is pulled on this machine') + } else { + const model = candidates[0].name + console.log(' model: ' + model) + const answer = await live('ollama_run').execute({ model, prompt: 'Reply with the single word: ok', options: { num_predict: 8, temperature: 0 }, timeoutSeconds: 300 }, exec) + check('live: one generation answers', answer.view.ok, true) + check('live: it carries the daemon\'s counts', typeof answer.view.measured.evalCount === 'number' && answer.view.measured.evalCount >= 0, true) + check('live: its rate is a finite number or n/a', answer.view.measured.evalTokensPerSecond === null || Number.isFinite(answer.view.measured.evalTokensPerSecond), true) + check('live: the prose names the model', answer.text.includes(model), true) + const liveBench = await live('ollama_bench').execute({ model, prompt: 'Reply with the single word: ok', runs: 3, cold: false, options: { num_predict: 8 }, timeoutSeconds: 300 }, exec) + check('live: a 3-run bench answers', liveBench.view.ok, true) + check('live: ...with three runs', liveBench.view.groups[0].runs.length, 3) + check('live: ...and a median it can stand behind', typeof liveBench.view.groups[0].rates.median === 'number', true) + } + } + } +} finally { + await daemon.close() +} + +console.log('') +console.log(failures === 0 ? 'all ' + checks + ' check(s) passed' + (skipped > 0 ? ' (' + skipped + ' skipped)' : '') : failures + ' of ' + checks + ' check(s) FAILED') +process.exitCode = failures === 0 ? 0 : 1 diff --git a/packages/dsh-ollama/cordis.patch.yml b/packages/dsh-ollama/cordis.patch.yml new file mode 100644 index 0000000..b6af5e6 --- /dev/null +++ b/packages/dsh-ollama/cordis.patch.yml @@ -0,0 +1,28 @@ +# dsh-ollama bundle layer (alpha). +# +# One row ('ollama') owns the whole plugin: its Node half registers FIVE tools +# over the machine's own Ollama daemon and three bundled skills - `ollama_status` +# (which models are on disk, which are loaded, whether the daemon answers, and +# the command that starts one), `ollama_run` (ONE bounded inference with the +# daemon's own token counts and nanosecond durations), `ollama_bench` (3+ runs, +# cold vs warm, A/B and a num_ctx sweep, reported as a median with a spread), +# `ollama_models` (list/show, and pull/delete behind a confirmation that repeats +# the tag) and `ollama_embed` (dimension, norms, cosine similarity). +# +# Host-only: the answer is numbers and prose, so there is no route to serve and +# no browser half to load - the package declares no dsh.client, and the boot +# graph gains one row. Like dsh-wsl and dsh-ocr, not like dsh-docker: no UI +# follows, and none is hidden behind an unregistered route. +# +# Nothing shipped is patched and no core row is disabled: core DSH has no +# Ollama surface and no tool reads the daemon's eval_count / eval_duration / +# load_duration. The harness's `bash` / `pwsh` tools can call `ollama run`, but +# they cannot return the measurement this row exists to report. +# +# Nothing for scripts/sync-vendored.ps1 either: no engine is vendored, forked or +# downloaded. `ollama` is the user's own program and its daemon is the user's +# own service at 127.0.0.1:11434; this package ships the HTTP client, the argv +# for one version probe, and the arithmetic. +- insert: + - id: ollama + name: 'dsh-ollama' diff --git a/packages/dsh-ollama/lib/cli.js b/packages/dsh-ollama/lib/cli.js new file mode 100644 index 0000000..cf74028 --- /dev/null +++ b/packages/dsh-ollama/lib/cli.js @@ -0,0 +1,74 @@ +/** + * dsh-ollama — one bounded child process, for `ollama --version` only. + * + * The CLI is read, never driven: pulling, deleting and generating all go + * through the daemon's HTTP API, because that is where the numbers are. The one + * question the API cannot answer is "which client version is installed", and + * that is what this runner is for. + * + * `spawn` with an argv ARRAY and no shell: a version string is not a command. + * The answer is bounded in time and in bytes, and a missing CLI is a normal + * result (`status: null`) rather than an exception - the daemon may be a + * container on the other end of a socket, with no `ollama` on this PATH. + */ + +import { spawn } from 'node:child_process' + +/** + * Run one argv array and collect its output. + * + * @param argv - the program and its arguments, one element each. + * @param options - `{ timeoutMs, maxBytes, env }`. + * @returns `{ status, stdout, stderr, timedOut, error }`; `status` is null when + * the program could not start at all. + */ +export function runProcess(argv, options = {}) { + const timeoutMs = Number.isFinite(options.timeoutMs) && options.timeoutMs > 0 ? options.timeoutMs : 5000 + const maxBytes = Number.isFinite(options.maxBytes) && options.maxBytes > 0 ? options.maxBytes : 1_000_000 + return new Promise((resolve) => { + let settled = false + let timedOut = false + let stdout = '' + let stderr = '' + const finish = (result) => { + if (settled) return + settled = true + resolve(result) + } + let child + try { + child = spawn(argv[0], argv.slice(1), { + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + env: options.env ?? process.env, + }) + } catch (error) { + finish({ status: null, stdout: '', stderr: '', timedOut: false, error: error && error.message ? String(error.message) : String(error) }) + return + } + const timer = setTimeout(() => { + timedOut = true + try { + child.kill('SIGKILL') + } catch (error) { + /* already gone */ + } + }, timeoutMs) + child.stdout?.setEncoding('utf8') + child.stderr?.setEncoding('utf8') + child.stdout?.on('data', (chunk) => { + if (stdout.length < maxBytes) stdout += chunk + }) + child.stderr?.on('data', (chunk) => { + if (stderr.length < maxBytes) stderr += chunk + }) + child.on('error', (error) => { + clearTimeout(timer) + finish({ status: null, stdout, stderr, timedOut, error: error && error.message ? String(error.message) : String(error) }) + }) + child.on('close', (code) => { + clearTimeout(timer) + finish({ status: code, stdout, stderr, timedOut }) + }) + }) +} diff --git a/packages/dsh-ollama/lib/client.js b/packages/dsh-ollama/lib/client.js new file mode 100644 index 0000000..756ab83 --- /dev/null +++ b/packages/dsh-ollama/lib/client.js @@ -0,0 +1,446 @@ +/** + * dsh-ollama — the HTTP client: one daemon, the loopback fence, bounded bodies + * and the NDJSON reader. + * + * Nothing here decides WHAT to ask; that is `tools.js`. This module owns the + * four rules every request obeys, and they are all measurements rather than + * preferences: + * + * - **loopback only.** The tools talk to the machine's own service at + * `127.0.0.1:11434`. `OLLAMA_HOST` may move it, but a value naming another + * machine is REFUSED unless `DSH_OLLAMA_ALLOW_REMOTE=1` says the person + * means it - prompts and completions would otherwise leave the machine on + * a plugin whose whole premise is that they do not; + * - **a bounded request.** Every call carries a deadline (`AbortSignal`), and + * a response body is read against a byte ceiling rather than trusted; + * - **streaming is NDJSON**, one JSON object per line, `done: true` last - + * the shape `/api/generate`, `/api/chat` and `/api/pull` all answer with. + * Reading it line by line is what lets the first token be TIMED; + * - **an error is a sentence.** `DAEMON_DOWN`, `TIMEOUT`, `NO_MODEL`, + * `HTTP_ERROR` and `BAD_STREAM` are distinct because the fix differs. + */ + +import { OllamaError } from './errors.js' + +/** The daemon's default, and the address Ollama itself binds. */ +export const DEFAULT_HOST = '127.0.0.1:11434' +export const DEFAULT_PORT = 11434 + +/** The one environment variable that allows a daemon on another machine. */ +export const ALLOW_REMOTE_ENV = 'DSH_OLLAMA_ALLOW_REMOTE' + +/** A JSON response body is capped here; `/api/show` is the largest real one. */ +export const MAX_BODY_BYTES = 2 * 1024 * 1024 + +/** A streamed body is capped here; a 40 GB pull's progress text is far below it. */ +export const MAX_STREAM_BYTES = 16 * 1024 * 1024 + +/** A stream may not carry more items than this, however small they are. */ +export const MAX_STREAM_ITEMS = 500_000 + +/** How much of an error body is quoted back. */ +const MAX_ERROR_CHARS = 600 + +/** The truthy spellings a person actually types for an opt-in. */ +function isTruthy(value) { + return /^(1|true|yes|on)$/i.test(String(value ?? '').trim()) +} + +/** + * Whether a hostname names this machine. + * + * `0.0.0.0` is accepted because it is a BIND address a local daemon is often + * configured with (`OLLAMA_HOST=0.0.0.0:11434`); as a destination it cannot + * reach another box. Everything else must be loopback, `localhost`, or the + * `*.localhost` namespace. + * + * @param hostname - a URL hostname with any IPv6 brackets already stripped. + * @returns true when the name cannot leave the machine. + */ +export function isLoopbackHost(hostname) { + const name = String(hostname).toLowerCase() + if (name === 'localhost' || name === 'localhost.' || name === '0.0.0.0' || name === '::1' || name === '::') return true + if (name.endsWith('.localhost')) return true + if (/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(name)) return true + return false +} + +/** + * Resolve `OLLAMA_HOST` (or the default) into the address this row will use. + * + * This function makes NO request: it is the fence, so both the tools and the + * check can ask "would this be allowed" without touching a socket. + * + * @param env - the environment (injectable for the check). + * @returns `{ raw, host, port, origin, base, local, remote, optIn, allowed, code, reason }`. + */ +export function resolveHost(env = process.env) { + const raw = String(env.OLLAMA_HOST ?? '').trim() + const text = raw === '' ? DEFAULT_HOST : raw + const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(text) ? text : 'http://' + text + let url + try { + url = new URL(withScheme) + } catch (err) { + return { + raw: text, + host: '', + port: DEFAULT_PORT, + origin: '', + base: '', + local: false, + remote: false, + optIn: false, + allowed: false, + code: 'HOST_INVALID', + reason: 'OLLAMA_HOST is "' + text + '", which is not an address. Use host:port, e.g. ' + DEFAULT_HOST + '.', + } + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + return { + raw: text, + host: url.hostname, + port: DEFAULT_PORT, + origin: '', + base: '', + local: false, + remote: false, + optIn: false, + allowed: false, + code: 'HOST_INVALID', + reason: 'OLLAMA_HOST is "' + text + '"; this row speaks http(s) to the daemon, so a "' + url.protocol + '" address is refused.', + } + } + if (url.username !== '' || url.password !== '') { + return { + raw: text, + host: url.hostname, + port: DEFAULT_PORT, + origin: '', + base: '', + local: false, + remote: false, + optIn: false, + allowed: false, + code: 'HOST_INVALID', + reason: 'OLLAMA_HOST carries credentials ("' + text + '"). A daemon address is not a place for a password, and this row will not read one from a URL.', + } + } + const host = url.hostname.replace(/^\[/, '').replace(/\]$/, '') + const port = url.port === '' ? DEFAULT_PORT : Number(url.port) + if (!Number.isInteger(port) || port < 1 || port > 65535) { + return { + raw: text, + host, + port, + origin: '', + base: '', + local: false, + remote: false, + optIn: false, + allowed: false, + code: 'HOST_INVALID', + reason: 'OLLAMA_HOST names port "' + url.port + '", which is not a port (1-65535).', + } + } + const local = isLoopbackHost(host) + const optIn = isTruthy(env[ALLOW_REMOTE_ENV]) + const base = url.origin + (url.pathname === '/' ? '' : url.pathname.replace(/\/+$/, '')) + const allowed = local || optIn + return { + raw: text, + host, + port, + origin: url.origin, + base, + local, + remote: !local, + optIn, + allowed, + code: allowed ? '' : 'REMOTE_REFUSED', + reason: allowed + ? '' + : 'OLLAMA_HOST is ' + + host + + ':' + + port + + ', which is not this machine. These tools talk to the local Ollama only: set ' + + ALLOW_REMOTE_ENV + + '=1 for a daemon you run elsewhere (prompts and completions then cross the network), or unset OLLAMA_HOST to use ' + + DEFAULT_HOST + + '.', + } +} + +/** The refusal a caller sees when the fence says no. */ +export function assertHostAllowed(config) { + if (config.allowed) return config + throw new OllamaError(config.code, config.reason, { host: config.origin || config.raw }) +} + +/** + * The abort signal for one request: our deadline, plus the caller's signal when + * one was given (an agent's cancel must cut a generation, not wait it out). + */ +function signalFor(timeoutMs, callerSignal) { + const deadline = AbortSignal.timeout(Math.max(1, Math.round(timeoutMs))) + if (callerSignal === undefined || callerSignal === null) return deadline + if (typeof AbortSignal.any === 'function') return AbortSignal.any([deadline, callerSignal]) + return deadline +} + +/** The deepest `code` a fetch failure carries (ECONNREFUSED lives in `cause`). */ +function causeCode(error) { + let current = error + for (let depth = 0; depth < 4 && current; depth += 1) { + if (typeof current.code === 'string' && current.code !== '') return current.code + current = current.cause + } + return '' +} + +/** + * Turn a rejected `fetch` into a refusal with a fix in it. + * + * @param error - what fetch threw. + * @param config - the resolved host. + * @param timeoutMs - the deadline that was in force, for the sentence. + */ +function networkError(error, config, timeoutMs) { + const name = error && error.name ? String(error.name) : '' + if (name === 'TimeoutError' || name === 'AbortError') { + return new OllamaError( + 'TIMEOUT', + 'No answer from ' + config.origin + ' within ' + Math.round(timeoutMs / 1000) + ' s. A cold model load can take longer than a short deadline: raise timeoutSeconds, or check `ollama_status` first.', + { host: config.origin }, + ) + } + const code = causeCode(error) + const detail = code === '' ? (error && error.message ? String(error.message) : 'fetch failed') : code + return new OllamaError( + 'DAEMON_DOWN', + 'Nothing is answering at ' + + config.origin + + ' (' + + detail + + '). The Ollama daemon is not running: start it with `ollama serve` in a terminal, or open the Ollama desktop app, then call `ollama_status`.', + { host: config.origin }, + ) +} + +/** + * Read a response body against a byte ceiling. + * + * @param response - a fetch Response. + * @param maxBytes - the ceiling. + * @returns the text. + */ +export async function readBodyBounded(response, maxBytes = MAX_BODY_BYTES) { + const declared = Number(response.headers.get('content-length')) + if (Number.isFinite(declared) && declared > maxBytes) { + throw new OllamaError('BODY_TOO_LARGE', 'The daemon answered with ' + declared + ' bytes, over the ' + maxBytes + '-byte ceiling this row reads.') + } + if (response.body === null || typeof response.body?.getReader !== 'function') { + const text = await response.text() + return text.length > maxBytes ? text.slice(0, maxBytes) : text + } + const reader = response.body.getReader() + const decoder = new TextDecoder() + let text = '' + let bytes = 0 + try { + for (;;) { + const { value, done } = await reader.read() + if (done) break + bytes += value.byteLength + if (bytes > maxBytes) { + throw new OllamaError('BODY_TOO_LARGE', 'The daemon answered with more than the ' + maxBytes + '-byte ceiling this row reads.') + } + text += decoder.decode(value, { stream: true }) + } + text += decoder.decode() + } finally { + reader.cancel().catch(() => {}) + } + return text +} + +/** + * The refusal one non-2xx response becomes. + * + * Ollama answers a missing model with HTTP 404 and `{"error":"model \"x\" not + * found, try pulling it first"}`, so the BODY decides between NO_MODEL and a + * generic NOT_FOUND - the status alone cannot. + */ +async function httpErrorFrom(response, options) { + let detail = '' + let parsed = null + try { + const text = await readBodyBounded(response, MAX_ERROR_CHARS * 8) + detail = text.trim().slice(0, MAX_ERROR_CHARS) + parsed = text.trim() === '' ? null : JSON.parse(text) + } catch (err) { + detail = detail === '' ? String(err && err.message ? err.message : err) : detail + } + const message = parsed && typeof parsed.error === 'string' && parsed.error !== '' ? parsed.error : detail + const model = options && typeof options.model === 'string' ? options.model : '' + const reason = (message + ' ' + detail).toLowerCase() + // Only a body that names the MODEL is a missing model. A 404 that says "404 + // page not found" is an OLDER DAEMON without this route, and reading it as + // "pull the model" would send the caller after the wrong fix. + const missingModel = reason.includes('try pulling') || (model !== '' && reason.includes('model') && reason.includes('not found')) + if (response.status === 404 && missingModel) { + return new OllamaError('NO_MODEL', 'No model "' + (model || 'requested') + '" on this daemon (' + (message || 'HTTP 404') + ').', { + status: 404, + detail: message, + model, + }) + } + if (response.status === 404) { + return new OllamaError('NOT_FOUND', 'The daemon has no route for ' + (options && options.route ? options.route : 'the request') + ' (' + (message || 'HTTP 404') + '). It may be an older daemon than this row expects.', { status: 404, detail: message }) + } + return new OllamaError('HTTP_ERROR', 'The daemon answered HTTP ' + response.status + (message === '' ? '.' : ': ' + message), { status: response.status, detail: message }) +} + +/** + * One JSON request to the daemon. + * + * @param deps - `{ fetch, env }`. + * @param route - an `/api/...` path. + * @param options - `{ method, body, timeoutMs, signal, model, route }`. + * @returns `{ data, config, status }`; `data` is `{}` for an empty 200. + */ +export async function requestJson(deps, route, options = {}) { + const config = resolveHost(deps.env) + assertHostAllowed(config) + const url = config.base + route + const timeoutMs = Number.isFinite(options.timeoutMs) && options.timeoutMs > 0 ? options.timeoutMs : 30_000 + const init = { + method: options.method ?? 'GET', + headers: { accept: 'application/json', 'user-agent': 'dsh-ollama' }, + signal: signalFor(timeoutMs, options.signal), + } + if (options.body !== undefined) { + init.body = JSON.stringify(options.body) + init.headers['content-type'] = 'application/json' + } + let response + try { + response = await deps.fetch(url, init) + } catch (error) { + throw networkError(error, config, timeoutMs) + } + if (!response.ok) throw await httpErrorFrom(response, { ...options, route }) + const text = await readBodyBounded(response) + if (text.trim() === '') return { data: {}, config, status: response.status } + try { + return { data: JSON.parse(text), config, status: response.status } + } catch (error) { + throw new OllamaError('BAD_JSON', 'The daemon answered ' + route + ' with something that is not JSON: ' + text.trim().slice(0, 200), { route }) + } +} + +/** + * One NDJSON request: read every line, hand each parsed object to `onItem`, and + * return them all. + * + * `onItem` is how the caller times the FIRST token: it runs as each line + * arrives, not after the stream ends. + * + * @param deps - `{ fetch, env }`. + * @param route - an `/api/...` path that streams. + * @param body - the JSON request body. + * @param options - `{ timeoutMs, signal, onItem, model, maxItems }`. + * @returns `{ items, config, bytes }`. + */ +export async function streamJsonLines(deps, route, body, options = {}) { + const config = resolveHost(deps.env) + assertHostAllowed(config) + const url = config.base + route + const timeoutMs = Number.isFinite(options.timeoutMs) && options.timeoutMs > 0 ? options.timeoutMs : 300_000 + const maxItems = Number.isFinite(options.maxItems) && options.maxItems > 0 ? options.maxItems : MAX_STREAM_ITEMS + let response + try { + response = await deps.fetch(url, { + method: 'POST', + headers: { accept: 'application/x-ndjson', 'content-type': 'application/json', 'user-agent': 'dsh-ollama' }, + body: JSON.stringify(body), + signal: signalFor(timeoutMs, options.signal), + }) + } catch (error) { + throw networkError(error, config, timeoutMs) + } + if (!response.ok) throw await httpErrorFrom(response, { ...options, route }) + const items = [] + const reader = response.body?.getReader?.() + if (reader === undefined) { + // A daemon behind a proxy that buffered the whole answer: still NDJSON. + const text = await response.text() + return { items: parseLines(text, options, items, maxItems), config, bytes: text.length } + } + const decoder = new TextDecoder() + let buffer = '' + let bytes = 0 + let emitted = 0 + const emit = (line) => { + const trimmed = line.trim() + if (trimmed === '') return + let parsed + try { + parsed = JSON.parse(trimmed) + } catch (error) { + throw new OllamaError('BAD_STREAM', 'The daemon sent a line that is not JSON: ' + trimmed.slice(0, 200), { route }) + } + if (emitted >= maxItems) { + throw new OllamaError('STREAM_TOO_LONG', 'The daemon sent more than ' + maxItems + ' stream objects; this row stops reading there.', { route }) + } + emitted += 1 + items.push(parsed) + if (typeof options.onItem === 'function') options.onItem(parsed) + } + try { + for (;;) { + const { value, done } = await reader.read() + if (done) break + bytes += value.byteLength + if (bytes > MAX_STREAM_BYTES) { + throw new OllamaError('STREAM_TOO_LONG', 'The daemon streamed more than the ' + MAX_STREAM_BYTES + '-byte ceiling this row reads.', { route }) + } + buffer += decoder.decode(value, { stream: true }) + let newline = buffer.indexOf('\n') + while (newline >= 0) { + emit(buffer.slice(0, newline)) + buffer = buffer.slice(newline + 1) + newline = buffer.indexOf('\n') + } + } + buffer += decoder.decode() + emit(buffer) + } catch (error) { + if (error instanceof OllamaError) throw error + throw networkError(error, config, timeoutMs) + } finally { + reader.cancel().catch(() => {}) + } + return { items, config, bytes } +} + +/** The buffered-proxy path of `streamJsonLines`: parse a whole body as NDJSON. */ +function parseLines(text, options, items, maxItems) { + let emitted = 0 + for (const line of String(text).split('\n')) { + const trimmed = line.trim() + if (trimmed === '') continue + let parsed + try { + parsed = JSON.parse(trimmed) + } catch (error) { + throw new OllamaError('BAD_STREAM', 'The daemon sent a line that is not JSON: ' + trimmed.slice(0, 200), { route: options.route }) + } + if (emitted >= maxItems) throw new OllamaError('STREAM_TOO_LONG', 'The daemon sent more than ' + maxItems + ' stream objects.', { route: options.route }) + emitted += 1 + items.push(parsed) + if (typeof options.onItem === 'function') options.onItem(parsed) + } + return items +} diff --git a/packages/dsh-ollama/lib/errors.js b/packages/dsh-ollama/lib/errors.js new file mode 100644 index 0000000..bc5e5e9 --- /dev/null +++ b/packages/dsh-ollama/lib/errors.js @@ -0,0 +1,29 @@ +/** + * dsh-ollama — the one error type this row throws. + * + * A tool's answer is prose, but the CODE is what the caller (and the package's + * check) branches on, so every refusal names itself: `REMOTE_REFUSED` is a + * policy decision, `DAEMON_DOWN` is a machine state, `NO_MODEL` is a missing + * artifact and `BAD_*` is the caller's argument. Collapsing them into one + * "failed" would make "start the daemon" and "pull the model" look alike. + */ +export class OllamaError extends Error { + /** + * @param code - the stable refusal code (`DAEMON_DOWN`, `NO_MODEL`, ...). + * @param message - one sentence a reader can act on. + * @param extra - fields the caller may want (`status`, `host`, `detail`). + */ + constructor(code, message, extra = {}) { + super(message) + this.name = 'OllamaError' + this.code = code + for (const [key, value] of Object.entries(extra)) { + if (value !== undefined) this[key] = value + } + } +} + +/** The one message of anything thrown. */ +export function messageOf(error) { + return error && error.message ? String(error.message) : String(error) +} diff --git a/packages/dsh-ollama/lib/index.js b/packages/dsh-ollama/lib/index.js new file mode 100644 index 0000000..1167b2a --- /dev/null +++ b/packages/dsh-ollama/lib/index.js @@ -0,0 +1,206 @@ +/** + * dsh-ollama — Node half. + * + * One row, five tools, no browser surface and no route. "What did this model do + * on this machine" is a host question whose answer is numbers and prose, so + * there is nothing for the right bar to draw and nothing to serve - the same + * shape as `dsh-wsl` and `dsh-ocr`. + * + * WHY IT EXISTS: every surface in this pack can CALL a model; none can tell a + * reader that `qwen2.5:7b` did 41.2 tok/s here at 8k context with a 3.1 s cold + * load, that a second run's first token arrives sooner because the weights + * stayed resident, or that a number was taken at temperature 0 with a fixed + * seed on a stated digest. Ollama reports exactly those fields in every + * response and nothing in the pack read them. + * + * WHAT IT IS NOT: not a second conversation (no chat UI), not a model manager + * beyond list/show/pull/delete, not a remote-inference client (the loopback + * fence in `client.js` is a refusal, not a default), and not a daemon starter - + * `ollama_status` names the command that starts one. + * + * Zero npm dependencies, no engine bundled, no network egress of its own: the + * daemon is the user's own service at `127.0.0.1:11434`, and the skills teach + * their own code against it. + */ + +import { readFileSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { runProcess } from './cli.js' +import { resolveHost } from './client.js' +import { buildTools } from './tools.js' +import { CAPS } from './validate.js' + +export const name = 'dsh-ollama' +export const inject = ['tools'] + +/** Kept in step with package.json by the tracked check. */ +export const PLUGIN_VERSION = '0.1.0-alpha.1' + +/** The three skills, as files beside this module. */ +export const SKILL_FILES = [ + { name: 'ollama-api', file: '../skills/ollama-api/SKILL.md' }, + { name: 'ollama-inference-metrics', file: '../skills/ollama-inference-metrics/SKILL.md' }, + { name: 'ollama-python', file: '../skills/ollama-python/SKILL.md' }, +] + +/** What this row offers, for the README and the package check. */ +export const CAPABILITIES = { + tools: ['ollama_status', 'ollama_run', 'ollama_bench', 'ollama_models', 'ollama_embed'], + endpoints: ['/api/version', '/api/tags', '/api/ps', '/api/show', '/api/generate', '/api/chat', '/api/embed', '/api/embeddings', '/api/pull', '/api/delete'], + skills: SKILL_FILES.map((entry) => entry.name), + caps: { ...CAPS }, + notBuilt: ['starting or stopping the daemon', 'model create / copy / push', 'a chat UI', 'routing OpenAI-compatible traffic through this row', 'remote inference by default'], +} + +/** + * The row's shared dependencies, one per boot. + * + * Every side effect is injectable so the package's check can drive every branch + * without a daemon: `fetch` and `now` are the transport and the clock, `runner` + * is the bounded child process behind the CLI version, and `cache` is the + * short-lived model list that keeps one answer to one request. + * + * @param ctx - the plugin context (inject: tools). + * @param options - `{ env, fetch, now, sleep, runner, platform }`. + */ +export function createDeps(ctx, options = {}) { + return { + ctx, + env: options.env ?? process.env, + fetch: options.fetch ?? globalThis.fetch, + now: options.now ?? (() => performance.now()), + sleep: options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))), + runner: options.runner ?? runProcess, + platform: options.platform ?? process.platform, + cache: { tags: null, tagsAt: 0, show: new Map() }, + log: { + warn: (text) => ctx.logger?.warn?.('[dsh-ollama] ' + text), + info: (text) => ctx.logger?.info?.('[dsh-ollama] ' + text), + }, + } +} + +/** + * The one activation line. + * + * It does NOT probe the daemon: a request per harness start is a request nobody + * asked for, and a machine whose Ollama is not running would print the same + * happy line either way. `ollama_status` is the tool that asks. + * + * @param deps - the row's dependencies. + */ +export function activationLine(deps) { + const config = resolveHost(deps.env) + if (!config.allowed) { + return 'inert: ' + config.reason + } + return ( + 'active: five ollama_* tools over ' + + (config.raw === '' ? config.origin : config.origin + ' (OLLAMA_HOST=' + config.raw + ')') + + (config.remote ? ', a daemon on ANOTHER machine (allowed by DSH_OLLAMA_ALLOW_REMOTE=1)' : ', this machine\'s own daemon (loopback)') + + '; nothing is started - ask `ollama_status` whether it is up' + ) +} + +// --------------------------------------------------------------------------- +// Skills +// --------------------------------------------------------------------------- +/** + * Split one skill document into its frontmatter and its body. + * + * @param text - the file's contents. + * @returns `{ meta, content }`. + */ +export function parseSkillFile(text) { + const normalized = String(text).replace(/\r\n?/g, '\n') + const match = /^---\n([\s\S]*?)\n---\n?/.exec(normalized) + if (!match) return { meta: {}, content: normalized.trim() } + const meta = {} + for (const line of match[1].split('\n')) { + const entry = /^([A-Za-z][A-Za-z0-9_-]*):\s*(.*)$/.exec(line) + if (!entry) continue + let value = entry[2].trim() + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) value = value.slice(1, -1) + meta[entry[1]] = value + } + return { meta: meta, content: normalized.slice(match[0].length).trim() } +} + +/** + * Register the bundled skills. The registry is resolved lazily (its absence + * only skips the skills, never the tools) and the installers also copy the + * folders into `$DSH_HOME/skills`, so a host without the registry still gets + * them from the filesystem provider. + * + * @returns the number of skills registered. + */ +export function registerSkills(ctx, log) { + const skills = typeof ctx.get === 'function' ? ctx.get('skills') : undefined + if (skills === null || skills === undefined || typeof skills.register !== 'function') { + log.warn('skill registry unavailable - the three bundled ollama skills were not registered') + return 0 + } + let count = 0 + for (const entry of SKILL_FILES) { + try { + const file = fileURLToPath(new URL(entry.file, import.meta.url)) + const { meta, content } = parseSkillFile(readFileSync(file, 'utf8')) + if (content.length === 0) { + log.warn('skill file is empty: ' + entry.file) + continue + } + const skillName = typeof meta.name === 'string' && meta.name.length > 0 ? meta.name : entry.name + ctx.effect( + () => + skills.register({ + name: skillName, + description: typeof meta.description === 'string' ? meta.description : '', + whenToUse: typeof meta.whenToUse === 'string' ? meta.whenToUse : undefined, + content: content, + provider: 'dsh-ollama', + source: 'bundled', + path: file, + resourceBase: { kind: 'directory', path: path.dirname(file) }, + }), + 'dsh-ollama: skill ' + skillName, + ) + count += 1 + } catch (err) { + log.warn('could not register skill ' + entry.name + ': ' + (err && err.message ? err.message : String(err))) + } + } + return count +} + +// --------------------------------------------------------------------------- +// Activation +// --------------------------------------------------------------------------- +/** + * Activate the row. + * + * @param ctx - cordis context (inject: tools). + */ +export function apply(ctx) { + const deps = createDeps(ctx) + deps.log.info(activationLine(deps)) + const skillCount = registerSkills(ctx, deps.log) + deps.log.info('registered ' + skillCount + ' bundled skill(s)') + for (const tool of buildTools(deps)) { + ctx.effect(() => ctx.tools.register(tool), 'dsh-ollama: tool ' + tool.name) + } +} + +/** Exported for the tracked checks: the parts of this row worth driving directly. */ +export const __internals = { + buildTools, + createDeps, + registerSkills, + parseSkillFile, + activationLine, + resolveHost, + CAPABILITIES, + PLUGIN_VERSION, + SKILL_FILES, +} diff --git a/packages/dsh-ollama/lib/metrics.js b/packages/dsh-ollama/lib/metrics.js new file mode 100644 index 0000000..ceda66a --- /dev/null +++ b/packages/dsh-ollama/lib/metrics.js @@ -0,0 +1,245 @@ +/** + * dsh-ollama — the arithmetic behind every number this row reports. + * + * Ollama reports its own timings in **nanoseconds** (`load_duration`, + * `prompt_eval_duration`, `eval_duration`, `total_duration`) and its own token + * counts (`prompt_eval_count`, `eval_count`). Every rate this plugin prints is + * ONE division of those two fields - never a stopwatch of ours substituted for + * them - so a reader can recompute it from the same response. + * + * Two rules make those numbers honest, and both are enforced here rather than + * in the prose of a tool: + * + * - a duration of zero yields `null`, not `Infinity`: a daemon that reported + * no time did not measure one; + * - `eval_duration` covers the GENERATION phase only, so load time never + * lowers tok/s - it shows up in time-to-first-token and in wall clock. A + * benchmark that attributed a cold load to tok/s would be measuring the + * wrong thing. + */ + +/** Ollama's duration fields are nanoseconds. */ +export const NS_PER_SECOND = 1e9 +export const NS_PER_MILLISECOND = 1e6 + +/** A finite number or `null` - never `NaN`, never `Infinity`. */ +function finiteOrNull(value) { + return typeof value === 'number' && Number.isFinite(value) ? value : null +} + +/** An integer or `null`. */ +function integerOrNull(value) { + return typeof value === 'number' && Number.isInteger(value) ? value : null +} + +/** + * Nanoseconds from the daemon, as milliseconds. + * @param nanoseconds - `final.eval_duration` and friends. + * @returns milliseconds, or `null` when the field is absent or not a number. + */ +export function millisecondsOf(nanoseconds) { + const value = finiteOrNull(nanoseconds) + return value === null ? null : value / NS_PER_MILLISECOND +} + +/** + * Tokens per second from a count and a nanosecond duration. + * @param count - `eval_count` or `prompt_eval_count`. + * @param durationNs - the matching `*_duration`. + * @returns tokens/second, or `null` when either side is absent or zero. + */ +export function tokensPerSecond(count, durationNs) { + const tokens = finiteOrNull(count) + const duration = finiteOrNull(durationNs) + if (tokens === null || duration === null || duration <= 0 || tokens < 0) return null + return tokens / (duration / NS_PER_SECOND) +} + +/** + * Turn one final response object into the measured shape every tool reports. + * + * `ttftDerivedMs` is `load + prompt_eval`: the daemon's own account of when the + * first generated token became possible. `ttftMeasuredMs` is what a streaming + * reader actually observed, and the two are reported side by side on purpose - + * when they disagree, the difference is the transport, not the model. + * + * @param final - the last NDJSON object of a generate/chat call. + * @param extra - `{ wallMs, ttftMs, doneReason }`, measured by the caller. + * @returns the numbers, with `null` where the daemon reported nothing. + */ +export function measure(final, extra = {}) { + const response = final !== null && typeof final === 'object' ? final : {} + const loadMs = millisecondsOf(response.load_duration) + const promptEvalMs = millisecondsOf(response.prompt_eval_duration) + const derived = loadMs === null && promptEvalMs === null ? null : (loadMs ?? 0) + (promptEvalMs ?? 0) + return { + evalCount: integerOrNull(response.eval_count), + promptEvalCount: integerOrNull(response.prompt_eval_count), + evalMs: millisecondsOf(response.eval_duration), + promptEvalMs, + loadMs, + totalMs: millisecondsOf(response.total_duration), + evalTokensPerSecond: tokensPerSecond(response.eval_count, response.eval_duration), + promptTokensPerSecond: tokensPerSecond(response.prompt_eval_count, response.prompt_eval_duration), + ttftMeasuredMs: finiteOrNull(extra.ttftMs), + ttftDerivedMs: derived, + wallMs: finiteOrNull(extra.wallMs), + doneReason: typeof extra.doneReason === 'string' ? extra.doneReason : typeof response.done_reason === 'string' ? response.done_reason : '', + } +} + +/** + * The p-th percentile by NEAREST RANK: the smallest value at or above the + * rank, so `p95` of five runs is one of the five and never a number no run + * produced. This is the convention a small sample can survive; an interpolated + * percentile invents a run. + * + * @param values - numbers (unsorted is fine). + * @param pct - 0..100. + * @returns the value, or `null` for an empty sample. + */ +export function percentile(values, pct) { + const sorted = values.filter((value) => Number.isFinite(value)).sort((a, b) => a - b) + if (sorted.length === 0) return null + const clamped = Math.min(100, Math.max(0, pct)) + const rank = Math.ceil((clamped / 100) * sorted.length) + return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))] +} + +/** The median: the p50 of the sample. */ +export function median(values) { + return percentile(values, 50) +} + +/** + * A sample summarised the way the metrics skill tells a reader to report it: + * a median with a spread, never a lone number. + * + * `spread` is the population standard deviation / mean (relative), which is + * comparable across models with different absolute rates; `range` is max - min + * because a reader can see both endpoints in the per-run table. + * + * @param values - the sample. + * @returns `{ count, min, median, p95, max, mean, stdev, spread, range }`. + */ +export function summarize(values) { + const sample = values.filter((value) => Number.isFinite(value)) + if (sample.length === 0) return { count: 0, min: null, median: null, p95: null, max: null, mean: null, stdev: null, spread: null, range: null } + const mean = sample.reduce((sum, value) => sum + value, 0) / sample.length + const variance = sample.reduce((sum, value) => sum + (value - mean) ** 2, 0) / sample.length + const min = Math.min(...sample) + const max = Math.max(...sample) + return { + count: sample.length, + min, + median: median(sample), + p95: percentile(sample, 95), + max, + mean, + stdev: Math.sqrt(variance), + spread: mean === 0 ? null : Math.sqrt(variance) / mean, + range: max - min, + } +} + +/** + * Cosine similarity between two vectors. + * + * A zero-length vector or a dimension mismatch returns `null`: the answer is + * "these are not comparable", not a number a reader would trust. + * + * @param a - first vector. + * @param b - second vector. + * @returns -1..1, or `null`. + */ +export function cosineSimilarity(a, b) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length === 0 || a.length !== b.length) return null + let dot = 0 + let normA = 0 + let normB = 0 + for (let index = 0; index < a.length; index += 1) { + const left = a[index] + const right = b[index] + if (typeof left !== 'number' || typeof right !== 'number' || !Number.isFinite(left) || !Number.isFinite(right)) return null + dot += left * right + normA += left * left + normB += right * right + } + if (normA === 0 || normB === 0) return null + return dot / (Math.sqrt(normA) * Math.sqrt(normB)) +} + +/** The L2 norm of a vector, or `null`. */ +export function vectorNorm(vector) { + if (!Array.isArray(vector) || vector.length === 0) return null + let sum = 0 + for (const value of vector) { + if (typeof value !== 'number' || !Number.isFinite(value)) return null + sum += value * value + } + return Math.sqrt(sum) +} + +/** + * A duration: milliseconds under a second, seconds above it. + * + * A 4 ms wall clock printed as "0.00 s" reads as "nothing happened"; the unit + * changes at a second because that is where a reader stops counting in ms. + * + * @param milliseconds - the duration. + * @param digits - decimals for the seconds form. + */ +export function formatMilliseconds(milliseconds, digits = 2) { + if (typeof milliseconds !== 'number' || !Number.isFinite(milliseconds)) return 'n/a' + if (milliseconds < 0) return 'n/a' + if (milliseconds < 1000) return Math.round(milliseconds) + ' ms' + return (milliseconds / 1000).toFixed(digits) + ' s' +} + +/** A rate, to one decimal: "50.0 tok/s". */ +export function formatRate(rate) { + if (typeof rate !== 'number' || !Number.isFinite(rate)) return 'n/a' + return rate.toFixed(1) + ' tok/s' +} + +/** Bytes the way `ollama list` prints them: 1024-based, one decimal. */ +export function formatBytes(bytes) { + if (typeof bytes !== 'number' || !Number.isFinite(bytes) || bytes < 0) return 'n/a' + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + let value = bytes + let unit = 0 + while (value >= 1024 && unit < units.length - 1) { + value /= 1024 + unit += 1 + } + return (unit === 0 ? String(Math.round(value)) : value.toFixed(1)) + ' ' + units[unit] +} + +/** + * An expiry time as a sentence, from `/api/ps`'s `expires_at`. + * + * @param expiresAt - an ISO timestamp, or anything the daemon sent. + * @param nowMs - the clock to compare against. + * @returns "in 4 minutes", "expired", or '' when there is nothing to say. + */ +export function formatExpiry(expiresAt, nowMs = Date.now()) { + if (typeof expiresAt !== 'string' || expiresAt === '') return '' + const at = Date.parse(expiresAt) + if (!Number.isFinite(at)) return '' + const delta = at - nowMs + if (delta <= 0) return 'expired' + const minutes = Math.round(delta / 60_000) + if (minutes < 1) return 'in under a minute' + if (minutes < 60) return 'in ' + minutes + (minutes === 1 ? ' minute' : ' minutes') + const hours = Math.floor(minutes / 60) + const rest = minutes % 60 + if (hours < 24) return 'in ' + hours + (hours === 1 ? ' hour' : ' hours') + (rest === 0 ? '' : ' ' + rest + ' min') + const days = Math.floor(hours / 24) + return 'in ' + days + (days === 1 ? ' day' : ' days') +} + +/** A percentage of a whole, or '' when the whole is unknown. */ +export function formatShare(part, whole) { + if (typeof part !== 'number' || typeof whole !== 'number' || !Number.isFinite(part) || !Number.isFinite(whole) || whole <= 0) return '' + return Math.round((part / whole) * 100) + '%' +} diff --git a/packages/dsh-ollama/lib/tools.js b/packages/dsh-ollama/lib/tools.js new file mode 100644 index 0000000..835815d --- /dev/null +++ b/packages/dsh-ollama/lib/tools.js @@ -0,0 +1,1018 @@ +/** + * dsh-ollama — the five tools, and the prose that is the product. + * + * Every answer is written for the reader that has to decide what to do next, so + * each one carries its CONDITIONS with its numbers: + * + * - `ollama_status` never starts the daemon; it says what it tried, whether + * the CLI exists, and the command that starts one; + * - `ollama_run` reports the daemon's own counts and durations AND a single + * run's limits - a lone tok/s is labelled a lone tok/s; + * - `ollama_bench` is the only tool that will call a number a MEASUREMENT: 3+ + * runs, a median with a spread, the model digest, the context length, and + * a cold run reported as a cold run (its cost is TTFT and wall, never + * tok/s, because `eval_duration` excludes the load); + * - `ollama_models` shows sizes and confirms a pull or a delete by repeating + * the tag, because a model is gigabytes and a delete costs a download; + * - `ollama_embed` names the dimension and computes cosine similarity rather + * than returning a wall of floats. + */ + +import { OllamaError, messageOf } from './errors.js' +import { CAPS, validateBench, validateEmbed, validateModels, validateRun } from './validate.js' +import { requestJson, resolveHost, streamJsonLines } from './client.js' +import { + cosineSimilarity, + formatBytes, + formatExpiry, + formatMilliseconds, + formatRate, + formatShare, + measure, + summarize, + vectorNorm, +} from './metrics.js' + +/** The output shape every tool returns: prose for the model, a view for the card. */ +const OUTPUT = { + schema: { + type: 'object', + properties: { text: { type: 'string' }, view: { type: 'object' } }, + required: ['text', 'view'], + }, + render: (_args, value) => [{ type: 'text', text: value.text }], + presentationMeta: (_args, value) => value.view, +} + +/** A card for the transcript. */ +function present(title) { + return { + presentCall: () => ({ card: 'generic', title: title, kind: 'other' }), + presentResult: (_args, result) => ({ card: 'generic', title: title, content: result && result.text ? result.text : '' }), + } +} + +/** A refusal: a sentence, and a view a caller can branch on. */ +function refusal(code, message, extra) { + return { text: message, view: { ok: false, code: code, ...(extra ?? {}) } } +} + +/** The same refusal for any thrown failure. */ +function failureOf(error) { + if (error instanceof OllamaError) return refusal(error.code, error.message) + return refusal('FAILED', error && error.message ? String(error.message) : String(error)) +} + +/** Pad a column, clipping it to its width. */ +function cell(value, width) { + const text = String(value === undefined || value === null ? '' : value) + const cut = text.length > width ? text.slice(0, Math.max(0, width - 1)) + '\u2026' : text + return cut.padEnd(width) +} + +/** A fixed-width table: the header line, then one line per row. */ +function table(headers, rows) { + if (rows.length === 0) return headers.join(' ') + const widths = headers.map((header, index) => Math.min(44, Math.max(header.length, ...rows.map((row) => String(row[index] ?? '').length)))) + const line = (cells) => cells.map((value, index) => cell(value, widths[index])).join(' ').trimEnd() + return [line(headers), ...rows.map(line)].join('\n') +} + +/** A digest as `ollama list` names it: the first 12 hex characters. */ +function shortDigest(digest) { + const text = String(digest ?? '').replace(/^sha256:/, '') + return text === '' ? 'n/a' : text.slice(0, 12) +} + +/** A number or 'n/a'. */ +function numberOr(value, format) { + return typeof value === 'number' && Number.isFinite(value) ? format(value) : 'n/a' +} + +/** The model name of one `/api/tags` or `/api/ps` entry. */ +function nameOf(entry) { + if (entry === null || typeof entry !== 'object') return '' + if (typeof entry.name === 'string' && entry.name !== '') return entry.name + if (typeof entry.model === 'string') return entry.model + return '' +} + +/** One `/api/tags` entry, flattened. */ +function describeModel(entry) { + const details = entry && typeof entry.details === 'object' && entry.details !== null ? entry.details : {} + return { + name: nameOf(entry), + digest: typeof entry.digest === 'string' ? entry.digest : '', + bytes: typeof entry.size === 'number' ? entry.size : null, + modifiedAt: typeof entry.modified_at === 'string' ? entry.modified_at : '', + family: typeof details.family === 'string' ? details.family : '', + families: Array.isArray(details.families) ? details.families : [], + parameterSize: typeof details.parameter_size === 'string' ? details.parameter_size : '', + quantization: typeof details.quantization_level === 'string' ? details.quantization_level : '', + format: typeof details.format === 'string' ? details.format : '', + } +} + +/** The models on disk. Cached for two seconds so one answer is one request. */ +async function tagsOf(deps, options = {}) { + const cache = deps.cache ?? (deps.cache = { tags: null, tagsAt: 0, show: new Map() }) + const now = typeof deps.now === 'function' ? deps.now() : Date.now() + if (options.fresh !== true && Array.isArray(cache.tags) && now - cache.tagsAt < 2000) return cache.tags + const { data } = await requestJson(deps, '/api/tags', { timeoutMs: CAPS.readTimeoutMs }) + const models = Array.isArray(data.models) ? data.models : [] + cache.tags = models + cache.tagsAt = now + return models +} + +/** The models loaded right now. */ +async function psOf(deps) { + const { data } = await requestJson(deps, '/api/ps', { timeoutMs: CAPS.readTimeoutMs }) + return Array.isArray(data.models) ? data.models : [] +} + +/** One model's metadata. Cached per boot: it does not change between two calls. */ +async function showOf(deps, model) { + const cache = deps.cache ?? (deps.cache = { tags: null, tagsAt: 0, show: new Map() }) + if (!cache.show) cache.show = new Map() + if (cache.show.has(model)) return cache.show.get(model) + const { data } = await requestJson(deps, '/api/show', { method: 'POST', body: { model }, timeoutMs: CAPS.readTimeoutMs, model }) + cache.show.set(model, data) + return data +} + +/** + * The version `ollama --version` reports. + * + * The CLI writes its answer to stderr ("client version is 0.40.3"), so both + * streams are read; no CLI at all is an empty string, not an error - the daemon + * may be a container on the other end of a socket. + */ +async function cliVersion(deps) { + if (typeof deps.runner !== 'function') return '' + const result = await deps.runner(['ollama', '--version'], { timeoutMs: 4000 }) + if (result === null || result === undefined) return '' + const text = String(result.stdout ?? '') + '\n' + String(result.stderr ?? '') + const match = /client version is ([0-9][0-9A-Za-z.+-]*)/i.exec(text) ?? /ollama version (?:is )?([0-9][0-9A-Za-z.+-]*)/i.exec(text) ?? /([0-9]+\.[0-9]+\.[0-9]+[0-9A-Za-z.+-]*)/.exec(text) + return match ? match[1] : '' +} + +/** The sentence a machine with no daemon gets. */ +function downText(config, cli) { + const lines = [] + lines.push('Ollama daemon: nothing answers at ' + config.origin + (config.raw !== '' && config.raw !== config.origin ? ' (from OLLAMA_HOST=' + config.raw + ')' : '') + '.') + lines.push('CLI: ' + (cli === '' ? 'no `ollama` on PATH.' : 'ollama ' + cli + ' on PATH (the client works; the server is not running).')) + if (config.remote) lines.push('Host: ' + config.host + ':' + config.port + ' is not this machine, and ' + 'DSH_OLLAMA_ALLOW_REMOTE=1' + ' allowed it.') + lines.push('') + lines.push('Nothing was started. Start the daemon yourself:') + lines.push(' ollama serve # a terminal; the daemon stays in the foreground') + lines.push(' or open the Ollama desktop app, which runs it in the background') + lines.push('') + lines.push('Then call `ollama_status` again. Models already on disk stay there; `ollama_models` lists them once the daemon answers.') + return lines.join('\n') +} + +/** The sentence for a model that is not pulled, with the tags that are. */ +async function noModelText(deps, model) { + let models = [] + try { + models = await tagsOf(deps, { fresh: true }) + } catch (error) { + // A daemon that does not answer cannot say which tags exist. Saying "no + // models are on disk" would blame the model store for a stopped service. + return 'No model "' + model + '" was reachable, and the daemon did not answer when asked what is on disk (' + messageOf(error) + '). Start it with `ollama serve`, call `ollama_status`, then pull with `ollama pull ' + model + '` if the tag is still missing.' + } + if (models.length === 0) { + return 'No model "' + model + '" on this daemon, and no models are on disk yet. Pull one first: `ollama pull ' + model + '` downloads gigabytes and is not something this row starts on its own.' + } + const names = models.map(nameOf).filter((name) => name !== '') + const shown = names.slice(0, 20).join(', ') + return ( + 'No model "' + + model + + '" on this daemon. On disk: ' + + shown + + (names.length > 20 ? ', \u2026 (' + names.length + ' total)' : '') + + '. Pull it with `ollama pull ' + + model + + '` (gigabytes), or use one of the names above. `ollama_models` shows their sizes before you decide.' + ) +} + +/** The options this call actually sent, as one line of conditions. */ +function conditionLine(spec, extra = {}) { + const parts = [] + const options = spec.options ?? {} + if (Object.keys(options).length === 0) parts.push('options: none sent (the daemon\'s own defaults)') + else parts.push(Object.entries(options).map(([key, value]) => key + ' ' + (Array.isArray(value) ? JSON.stringify(value) : String(value))).join(' \u00b7 ')) + if (spec.keepAlive !== undefined) parts.push('keep_alive ' + String(spec.keepAlive)) + if (extra.context !== undefined && extra.context !== null) parts.push('num_ctx ' + extra.context) + parts.push(extra.timeoutSeconds !== undefined ? 'timeout ' + extra.timeoutSeconds + ' s' : 'timeout ' + Math.round((spec.timeoutMs ?? 0) / 1000) + ' s') + return parts.join(' \u00b7 ') +} + +/** How one response object carries its text: chat and generate differ. */ +function contentOf(item, chat) { + if (item === null || typeof item !== 'object') return '' + if (chat) { + const message = item.message + if (message !== null && typeof message === 'object' && typeof message.content === 'string') return message.content + return '' + } + return typeof item.response === 'string' ? item.response : '' +} + +/** + * ONE bounded inference, streamed so the first token can be timed. + * + * @param deps - the row's dependencies. + * @param spec - a `validateRun` result. + * @param options - `{ signal }` from the agent. + * @returns `{ text, measured, final, chunks }`. + */ +async function infer(deps, spec, options = {}) { + const body = { model: spec.model, stream: true } + if (spec.chat) body.messages = spec.messages + else { + body.prompt = spec.prompt + if (spec.system !== '') body.system = spec.system + } + if (Object.keys(spec.options).length > 0) body.options = spec.options + if (spec.keepAlive !== undefined) body.keep_alive = spec.keepAlive + if (spec.format !== undefined) body.format = spec.format + if (spec.think !== undefined) body.think = spec.think + + const started = deps.now() + let text = '' + let firstContentAt = null + let final = null + let chunks = 0 + const streamed = await streamJsonLines(deps, spec.chat ? '/api/chat' : '/api/generate', body, { + timeoutMs: spec.timeoutMs, + signal: options.signal, + model: spec.model, + route: spec.chat ? '/api/chat' : '/api/generate', + onItem: (item) => { + chunks += 1 + if (item !== null && typeof item === 'object' && typeof item.error === 'string' && item.error !== '') { + throw new OllamaError('DAEMON_ERROR', 'The daemon interrupted the generation: ' + item.error, { model: spec.model }) + } + const piece = contentOf(item, spec.chat) + if (piece !== '') { + if (firstContentAt === null) firstContentAt = deps.now() + text += piece + } + if (item !== null && typeof item === 'object' && item.done === true) final = item + }, + }) + const wallMs = deps.now() - started + if (final === null) { + if (text === '' && streamed.items.length === 0) { + throw new OllamaError('BAD_STREAM', 'The daemon closed the stream without sending anything. Check `ollama_status`, then try the call again.', { model: spec.model }) + } + final = streamed.items[streamed.items.length - 1] ?? {} + } + return { text, measured: measure(final, { wallMs, ttftMs: firstContentAt === null ? null : firstContentAt - started }), final, chunks } +} + +/** + * The answer `ollama_run` prints: the text, then the numbers, then the limits + * of a single run. + * + * The digest is passed in rather than guessed: `/api/generate` echoes the model + * TAG, not its digest, and a number without the digest is a number about "a + * model with that name", which is not the same claim. + */ +function runAnswer(spec, result, digest, config) { + const measured = result.measured + const lines = [] + const headline = [] + headline.push('MODEL ' + spec.model + (digest === '' ? '' : ' \u00b7 digest ' + shortDigest(digest))) + headline.push(numberOr(measured.evalCount, (value) => String(value)) + ' tokens generated in ' + numberOr(measured.evalMs, (value) => formatMilliseconds(value)) + ' \u00b7 ' + formatRate(measured.evalTokensPerSecond)) + headline.push('prompt ' + numberOr(measured.promptEvalCount, (value) => String(value)) + ' tokens \u00b7 ' + numberOr(measured.promptEvalMs, (value) => formatMilliseconds(value)) + ' \u00b7 ' + formatRate(measured.promptTokensPerSecond)) + headline.push( + 'load ' + + numberOr(measured.loadMs, (value) => formatMilliseconds(value)) + + ' \u00b7 first token ' + + (measured.ttftMeasuredMs !== null ? formatMilliseconds(measured.ttftMeasuredMs) + ' (measured)' : 'not measured') + + ' \u00b7 total ' + + numberOr(measured.totalMs, (value) => formatMilliseconds(value)) + + ' (daemon) \u00b7 ' + + numberOr(measured.wallMs, (value) => formatMilliseconds(value)) + + ' (wall)', + ) + if (measured.doneReason !== '') headline.push('stop reason: ' + measured.doneReason) + lines.push(headline.join('\n')) + lines.push('') + const body = result.text === '' ? '(the model generated no text)' : result.text + const capped = body.length > 12_000 ? body.slice(0, 12_000) + '\n\u2026 (cut at 12 000 characters; the view carries the counts)' : body + lines.push(capped) + lines.push('') + lines.push('Conditions: ' + conditionLine(spec) + '.') + if (measured.doneReason === 'length') { + const limit = spec.options.num_predict + lines.push('') + lines.push('The answer hit num_predict (' + limit + ') and was cut at the model\'s stop, not at its own end. Raise `options.num_predict` (ceiling ' + CAPS.maxNumPredict + ') for a longer answer.') + } + lines.push('A single run is a single run: `ollama_bench` makes 3 or more and reports a median with its spread.') + if (config.remote) lines.push('Host: ' + config.origin + ' is not this machine (allowed by DSH_OLLAMA_ALLOW_REMOTE=1), so these timings include the network.') + return lines.join('\n') +} + +/** The view `ollama_run` hands the card, and the check asserts. */ +function runView(spec, result, digest) { + return { + ok: true, + model: spec.model, + digest: digest, + text: result.text, + measured: result.measured, + conditions: { options: spec.options, keepAlive: spec.keepAlive ?? null, timeoutSeconds: Math.round(spec.timeoutMs / 1000) }, + } +} + +/** + * Ollama's `options` map, declared once and used by BOTH generation tools. + * + * It is CLOSED (`additionalProperties: false`) for the reason the validator + * enforces it: the daemon ignores an option it does not know, so `temprature` + * would not fail - it would take the numbers at a setting nobody chose. + */ +const OPTIONS_SCHEMA = { + type: 'object', + description: 'Ollama\'s options map, closed to the names here. A typo would be silently ignored by the daemon and change what the numbers measured.', + additionalProperties: false, + properties: { + num_predict: { type: 'number', description: 'Maximum tokens to generate: 1-8192. Default 512 for a run, 128 for a benchmark (Ollama\'s own default is unbounded).' }, + temperature: { type: 'number', description: '0-2. 0 for a reproducible measurement.' }, + seed: { type: 'number', description: 'Integer 0-2147483647; fixed with temperature 0 for a reproducible run.' }, + num_ctx: { type: 'number', description: 'Context window in tokens: 128-1048576. Changing it changes memory use and speed.' }, + top_k: { type: 'number', description: '0-1000.' }, + top_p: { type: 'number', description: '0-1.' }, + min_p: { type: 'number', description: '0-1.' }, + repeat_penalty: { type: 'number', description: '0-10.' }, + repeat_last_n: { type: 'number', description: '-1 to 8192.' }, + presence_penalty: { type: 'number', description: '-2 to 2.' }, + frequency_penalty: { type: 'number', description: '-2 to 2.' }, + stop: { type: 'array', items: { type: 'string' }, description: 'Up to 8 stop sequences.' }, + num_gpu: { type: 'number', description: 'Layers to offload: 0-999. 0 keeps it on the CPU.' }, + num_thread: { type: 'number', description: 'CPU threads: 1-256.' }, + use_mmap: { type: 'boolean', description: 'Memory-map the weights.' }, + use_mlock: { type: 'boolean', description: 'Lock the weights in RAM.' }, + mirostat: { type: 'number', description: '0 (off), 1, or 2.' }, + }, +} + +/** Build the tools. @param deps - `{ env, fetch, now, runner, cache, log }`. */ +export function buildTools(deps) { + /** `ollama_status`: what is here, whether it answers, and what it holds. */ + const status = { + name: 'ollama_status', + description: [ + 'Report whether this machine\'s Ollama daemon is answering, which version it is, every model on disk (family, parameter size, quantization, bytes, digest), and which models are loaded right now (processor split, context length, when they expire) - plus the CLI version when `ollama` is on PATH.', + 'Call this FIRST, before any other ollama_* tool. It is the only tool that treats a daemon that is not running as an ANSWER rather than a failure: it names the address it tried and the command that starts one (`ollama serve`, or the desktop app).', + 'It never starts the daemon, pulls a model or changes anything. Read-only.', + ].join('\n'), + parameters: { type: 'object', additionalProperties: false, properties: {} }, + output: OUTPUT, + ...present('Ollama status'), + async execute() { + try { + const config = resolveHost(deps.env) + const cli = await cliVersion(deps) + const base = { host: { origin: config.origin, host: config.host, port: config.port, remote: config.remote, optIn: config.optIn }, cli } + if (!config.allowed) return { text: config.reason, view: { ok: false, code: config.code, ...base } } + let version + try { + const answer = await requestJson(deps, '/api/version', { timeoutMs: CAPS.statusTimeoutMs }) + version = typeof answer.data.version === 'string' ? answer.data.version : '' + } catch (error) { + if (error instanceof OllamaError && (error.code === 'DAEMON_DOWN' || error.code === 'TIMEOUT')) { + return { text: downText(config, cli), view: { ok: false, code: 'DAEMON_DOWN', ...base } } + } + throw error + } + const errors = {} + let rawModels = [] + try { + rawModels = await tagsOf(deps, { fresh: true }) + } catch (error) { + errors.tags = messageOf(error) + } + let rawLoaded = [] + try { + rawLoaded = await psOf(deps) + } catch (error) { + errors.ps = messageOf(error) + } + const models = rawModels.map(describeModel) + const totalBytes = models.reduce((sum, entry) => sum + (typeof entry.bytes === 'number' ? entry.bytes : 0), 0) + const loaded = rawLoaded.map((entry) => { + const details = entry && typeof entry.details === 'object' && entry.details !== null ? entry.details : {} + const size = typeof entry.size === 'number' ? entry.size : null + const vram = typeof entry.size_vram === 'number' ? entry.size_vram : null + return { + name: nameOf(entry), + digest: typeof entry.digest === 'string' ? entry.digest : '', + size, + sizeVram: vram, + contextLength: typeof entry.context_length === 'number' ? entry.context_length : null, + expiresAt: typeof entry.expires_at === 'string' ? entry.expires_at : '', + family: typeof details.family === 'string' ? details.family : '', + parameterSize: typeof details.parameter_size === 'string' ? details.parameter_size : '', + quantization: typeof details.quantization_level === 'string' ? details.quantization_level : '', + processor: vram === null || size === null || size === 0 ? '' : vram === 0 ? '100% CPU' : formatShare(vram, size) + ' GPU', + } + }) + + const lines = [] + lines.push('Ollama ' + (version === '' ? '(version not reported)' : version) + (cli === '' ? ' \u00b7 CLI not on PATH' : ' \u00b7 CLI ' + cli) + ' at ' + config.origin + (config.remote ? ' (not this machine; allowed by DSH_OLLAMA_ALLOW_REMOTE=1)' : '')) + lines.push('Models on disk: ' + models.length + (models.length > 0 ? ' (' + formatBytes(totalBytes) + ')' : '')) + if (models.length > 0) { + lines.push(table(['NAME', 'PARAM', 'QUANT', 'SIZE', 'DIGEST'], models.map((entry) => [entry.name, entry.parameterSize, entry.quantization, formatBytes(entry.bytes), shortDigest(entry.digest)]))) + } else { + lines.push('') + lines.push('No models yet. Pull one with `ollama pull ` (gigabytes): `ollama_models` action "pull" shows what it would fetch first.') + } + lines.push('') + if (loaded.length === 0) { + lines.push('Loaded now: none. The first call that names a model loads it (and pays the load time once).') + } else { + lines.push('Loaded now: ' + loaded.length) + lines.push(table(['NAME', 'CONTEXT', 'SIZE', 'PROCESSOR', 'EXPIRES'], loaded.map((entry) => [entry.name, entry.contextLength === null ? 'n/a' : String(entry.contextLength), formatBytes(entry.size), entry.processor || 'n/a', formatExpiry(entry.expiresAt, Date.now())]))) + } + for (const [what, message] of Object.entries(errors)) lines.push('', 'Could not read ' + what + ': ' + message) + lines.push('') + lines.push('Next: `ollama_run` for one inference with its numbers, `ollama_bench` for a measurement (3+ runs, median and spread), `ollama_models` to show or pull one, `ollama_embed` for vectors.') + return { + text: lines.join('\n'), + view: { + ok: true, + host: base.host, + daemon: { up: true, version }, + cli, + counts: { models: models.length, bytes: totalBytes, loaded: loaded.length }, + models, + loaded, + errors, + }, + } + } catch (error) { + return failureOf(error) + } + }, + } + + /** `ollama_run`: ONE bounded inference, with the numbers. */ + const run = { + name: 'ollama_run', + description: [ + 'Run ONE bounded inference against a model that is already pulled: either `prompt` (a single string, `/api/generate`) or `messages` (a chat array, `/api/chat`), plus `options`, `format`, `keep_alive` and a timeout.', + 'It returns the text AND the measurement: prompt and eval token counts, tokens/second from `eval_count / eval_duration`, the load time, time-to-first-token as actually observed while streaming, and both the daemon\'s total and the wall clock. `num_predict` is set for you (512) because Ollama\'s default would generate until the context filled; the ceiling is 8192.', + 'A refused call names the fix: a model that is not pulled lists the tags that ARE on disk and the exact `ollama pull`; a daemon that is not running names `ollama serve`; a non-loopback OLLAMA_HOST names the opt-in variable.', + 'ONE run is not a measurement. For a number worth quoting, use `ollama_bench`.', + ].join('\n'), + parameters: { + type: 'object', + additionalProperties: false, + required: ['model'], + properties: { + model: { type: 'string', description: 'A model tag already on disk, e.g. qwen2.5:7b (as `ollama_status` prints it).' }, + prompt: { type: 'string', description: 'One prompt, sent to /api/generate. Exactly one of prompt / messages.' }, + messages: { + type: 'array', + description: 'A chat array for /api/chat: [{ role: "system"|"user"|"assistant"|"tool", content, images?, tool_calls?, tool_name? }].', + items: { type: 'object' }, + }, + system: { type: 'string', description: 'A system prompt for the `prompt` form. In a `messages` call, use a system message instead.' }, + options: OPTIONS_SCHEMA, + format: { description: 'Structured output: the string "json", or a JSON Schema object the answer must satisfy.' }, + keep_alive: { description: 'How long the weights stay loaded after this call: a duration ("5m", "30s"), 0 (unload when done), or -1 (forever).' }, + think: { type: 'boolean', description: 'Ask a thinking-capable model to reason before answering (models that do not support it ignore the flag).' }, + timeoutSeconds: { type: 'number', description: 'Deadline for the whole call: 10-900 s. Default 300.' }, + }, + }, + output: OUTPUT, + ...present('Ollama run'), + async execute(args, exec) { + try { + const spec = validateRun(args) + const result = await infer(deps, spec, { signal: exec?.signal }) + const config = resolveHost(deps.env) + let digest = '' + try { + const models = await tagsOf(deps) + digest = models.map(describeModel).find((entry) => entry.name === spec.model)?.digest ?? '' + } catch (error) { + /* the run succeeded; only the digest lookup failed */ + } + return { text: runAnswer(spec, result, digest, config), view: runView(spec, result, digest) } + } catch (error) { + if (error instanceof OllamaError && error.code === 'NO_MODEL') return refusal('NO_MODEL', await noModelText(deps, String(args.model ?? '')), { model: args.model }) + return failureOf(error) + } + }, + } + + /** `ollama_bench`: the measurement, not the anecdote. */ + const bench = { + name: 'ollama_bench', + description: [ + 'Measure ONE model (or A/B two or three) with N repetitions of the same call, and report per-run numbers plus a median, p95 and spread for tokens/second, time-to-first-token and wall clock.', + 'It is the only tool here that calls a number a MEASUREMENT: `runs` must be at least 3 (a single run is refused by name), temperature defaults to 0 with a fixed seed, `num_predict` defaults to 128, and every answer carries the model digest, the context length and the run count.', + 'By default the FIRST counted run is cold: the model is unloaded with `keep_alive: 0` before the runs begin, so run 1 pays the load. Its cost shows in TTFT and wall - NEVER in tok/s, because `eval_duration` covers generation only. `cold: false` measures a warm model only.', + '`contexts` sweeps num_ctx (e.g. [2048, 8192]) - one variable at a time, which is the point. `warmup` makes unmeasured calls first.', + 'Bounded: 3-50 runs, 1-3 models, 0-6 contexts, at most 150 measured calls, and a timeout you must state.', + ].join('\n'), + parameters: { + type: 'object', + additionalProperties: false, + required: ['prompt'], + properties: { + model: { type: 'string', description: 'One model tag. Send this OR `models`, not both.' }, + models: { type: 'array', items: { type: 'string' }, description: 'An A/B array of 1-3 distinct model tags.' }, + prompt: { type: 'string', description: 'The prompt every run sends. Keep it identical across runs: that is the experiment.' }, + runs: { type: 'number', description: 'Measured runs: 3-50. Default 5.' }, + warmup: { type: 'number', description: 'Unmeasured calls before the runs: 0-5. Default 0.' }, + cold: { type: 'boolean', description: 'Unload the model before the runs so run 1 is cold. Default true.' }, + contexts: { type: 'array', items: { type: 'number' }, description: 'A num_ctx sweep: up to 6 context lengths, e.g. [2048, 8192]. Default: the model\'s own.' }, + options: OPTIONS_SCHEMA, + format: { description: 'Structured output for every run: "json" or a JSON Schema object.' }, + think: { type: 'boolean', description: 'Pass `think` to each run.' }, + timeoutSeconds: { type: 'number', description: 'Deadline PER CALL: 30-600 s. Default 300.' }, + }, + }, + output: OUTPUT, + ...present('Ollama bench'), + async execute(args, exec) { + try { + const plan = validateBench(args) + const config = resolveHost(deps.env) + const models = await tagsOf(deps, { fresh: true }).catch(() => []) + const digestFor = (tag) => models.map(describeModel).find((entry) => entry.name === tag)?.digest ?? '' + const groups = [] + for (const model of plan.models) { + const contexts = plan.contexts.length > 0 ? plan.contexts : [null] + for (const context of contexts) { + const runs = [] + if (plan.warmup > 0) { + for (let index = 0; index < plan.warmup; index += 1) { + await infer(deps, { ...runSpec(plan, model, context), options: optionsWithContext(plan.options, context) }, { signal: exec?.signal }) + } + } + let unloaded = false + if (plan.cold) { + try { + await requestJson(deps, '/api/generate', { + method: 'POST', + body: { model, keep_alive: 0, stream: false }, + timeoutMs: CAPS.readTimeoutMs, + model, + }) + unloaded = true + } catch (error) { + /* a model that was not loaded cannot be unloaded; run 1 is cold anyway */ + } + } + for (let index = 0; index < plan.runs; index += 1) { + const spec = { ...runSpec(plan, model, context), options: optionsWithContext(plan.options, context) } + const result = await infer(deps, spec, { signal: exec?.signal }) + runs.push({ index: index + 1, measured: result.measured }) + } + groups.push({ model, digest: digestFor(model), context, runs, unloaded }) + } + } + return { text: benchText(plan, groups, config), view: benchView(plan, groups, config) } + } catch (error) { + if (error instanceof OllamaError && error.code === 'NO_MODEL') { + const named = String(args.model ?? (Array.isArray(args.models) ? args.models[0] : '') ?? '') + return refusal('NO_MODEL', await noModelText(deps, named), { model: named }) + } + return failureOf(error) + } + }, + } + + /** `ollama_models`: show one, list, and the two writes behind a confirmation. */ + const models = { + name: 'ollama_models', + description: [ + 'Inspect and manage the models on disk: `list` (every tag with its family, parameter size, quantization and bytes), `show` (one model\'s parameters, template, license, capabilities and architecture facts), `pull` (download one) and `delete` (remove one, freeing its bytes).', + '`pull` and `delete` are WRITES, and both require `confirm` set to the exact tag. Without it the tool answers with what it WOULD do - the size on disk or the size to download, and the exact command - and changes nothing. Models are gigabytes: a delete costs a download to undo.', + 'Pulling a tag that is already on disk re-checks for a newer layer; it is not a no-op. Nothing here pulls a model implicitly.', + ].join('\n'), + parameters: { + type: 'object', + additionalProperties: false, + properties: { + action: { type: 'string', enum: ['list', 'show', 'pull', 'delete'], description: 'What to do. Default list.' }, + model: { type: 'string', description: 'The model tag: required for show, pull and delete.' }, + confirm: { type: 'string', description: 'Repeat the model tag to confirm a pull or a delete. Without it, the tool reports what it would do and does nothing.' }, + timeoutSeconds: { type: 'number', description: 'Deadline for a whole pull: 60-3600 s. Default 1800. Ignored by the other actions.' }, + }, + }, + output: OUTPUT, + ...present('Ollama models'), + async execute(args) { + try { + const spec = validateModels(args) + if (spec.action === 'list') { + const raw = await tagsOf(deps, { fresh: true }) + const models = raw.map(describeModel) + if (models.length === 0) { + return { text: 'No models on disk. `ollama_models` action "pull" with a tag shows what it would download; nothing is pulled without `confirm`.', view: { ok: true, models: [], count: 0, bytes: 0 } } + } + const bytes = models.reduce((sum, entry) => sum + (typeof entry.bytes === 'number' ? entry.bytes : 0), 0) + const rows = models.map((entry) => [entry.name, entry.family, entry.parameterSize, entry.quantization, formatBytes(entry.bytes), shortDigest(entry.digest)]) + return { + text: table(['NAME', 'FAMILY', 'PARAM', 'QUANT', 'SIZE', 'DIGEST'], rows) + '\n\n' + models.length + ' model(s), ' + formatBytes(bytes) + ' on disk.', + view: { ok: true, models, count: models.length, bytes }, + } + } + if (spec.action === 'show') { + const data = await showOf(deps, spec.model) + return { text: showText(spec.model, data), view: { ok: true, model: spec.model, details: data.details ?? {}, capabilities: data.capabilities ?? [] } } + } + if (spec.action === 'pull') { + if (spec.confirm !== spec.model) { + let present = null + let checked = true + try { + present = (await tagsOf(deps, { fresh: true })).map(describeModel).find((entry) => entry.name === spec.model) ?? null + } catch (error) { + // "not on disk yet" is a claim about the store; a silent failure + // to read the store cannot support it. + checked = false + } + const state = !checked ? ' (could not check: the daemon did not answer)' : present === null ? ' (not on disk yet)' : ' (already on disk, ' + formatBytes(present.bytes) + ')' + const lines = ['PULL ' + spec.model + state] + lines.push('') + lines.push('Nothing was downloaded. A pull writes gigabytes to disk and this row does not start one without an explicit confirmation.') + lines.push('To pull it: call `ollama_models` again with action "pull", model "' + spec.model + '", confirm "' + spec.model + '".') + lines.push('In a terminal the same thing is: ollama pull ' + spec.model) + return refusal('CONFIRM_REQUIRED', lines.join('\n'), { model: spec.model, present: present !== null, bytes: present === null ? null : present.bytes }) + } + const started = deps.now() + const streamed = await streamJsonLines(deps, '/api/pull', { model: spec.model, stream: true }, { + timeoutMs: spec.timeoutSeconds * 1000, + model: spec.model, + route: '/api/pull', + }) + const elapsedMs = deps.now() - started + const summary = pullSummary(streamed.items) + const lines = [] + lines.push('PULLED ' + spec.model + ' in ' + formatMilliseconds(elapsedMs, 1) + '.') + if (summary.layers > 0) lines.push('Layers: ' + summary.layers + (summary.total > 0 ? ' \u00b7 ' + formatBytes(summary.completed) + ' of ' + formatBytes(summary.total) : '') + '.') + lines.push('Final status: ' + (summary.status === '' ? '(none reported)' : summary.status)) + lines.push('') + lines.push('Next: `ollama_run` with model "' + spec.model + '" sends one inference; `ollama_bench` measures it over 3+ runs.') + return { text: lines.join('\n'), view: { ok: true, model: spec.model, elapsedMs, bytes: summary.total, layers: summary.layers, status: summary.status } } + } + // delete + if (spec.confirm !== spec.model) { + let present = null + let checked = true + try { + present = (await tagsOf(deps, { fresh: true })).map(describeModel).find((entry) => entry.name === spec.model) ?? null + } catch (error) { + checked = false + } + const state = !checked ? ' (could not check: the daemon did not answer)' : present === null ? ' (not on disk)' : ' (' + formatBytes(present.bytes) + ' on disk)' + const lines = ['DELETE ' + spec.model + state] + lines.push('') + lines.push('Nothing was removed. The bytes are freed immediately and the only way back is a pull.') + lines.push('To delete it: call `ollama_models` again with action "delete", model "' + spec.model + '", confirm "' + spec.model + '".') + lines.push('In a terminal the same thing is: ollama rm ' + spec.model) + return refusal('CONFIRM_REQUIRED', lines.join('\n'), { model: spec.model, present: present !== null, bytes: present === null ? null : present.bytes }) + } + await requestJson(deps, '/api/delete', { method: 'DELETE', body: { model: spec.model }, timeoutMs: CAPS.readTimeoutMs, model: spec.model }) + return { + text: 'Deleted ' + spec.model + '. Its bytes are free; getting it back is `ollama pull ' + spec.model + '`.', + view: { ok: true, model: spec.model, deleted: true }, + } + } catch (error) { + if (error instanceof OllamaError && error.code === 'NO_MODEL') return refusal('NO_MODEL', await noModelText(deps, String(args.model ?? '')), { model: args.model }) + return failureOf(error) + } + }, + } + + /** `ollama_embed`: vectors, their dimension, and comparability. */ + const embed = { + name: 'ollama_embed', + description: [ + 'Turn one or more strings into embeddings with an embedding model (`/api/embed`), and report the dimension and the vectors\' norms. With exactly two inputs it also reports their cosine similarity (1.0 identical, 0 orthogonal, -1 opposite), which is the question a reader actually has.', + 'The model must be an EMBEDDING model: this tool checks the model\'s advertised capabilities and refuses a completion model by name, pointing at `nomic-embed-text`. An older daemon with no `/api/embed` is handled through `/api/embeddings` instead.', + 'Vectors are large: the answer previews the first few dimensions and reports the count and dimension; the view carries nothing you cannot recompute.', + ].join('\n'), + parameters: { + type: 'object', + additionalProperties: false, + required: ['model', 'input'], + properties: { + model: { type: 'string', description: 'An embedding model tag, e.g. nomic-embed-text.' }, + input: { description: 'One string, or an array of 1-64 strings to embed.' }, + similarity: { type: 'boolean', description: 'Compute cosine similarity between the first two inputs. Default true when there are exactly two.' }, + keep_alive: { description: 'How long the embeddings model stays loaded: a duration, 0, or -1.' }, + timeoutSeconds: { type: 'number', description: 'Deadline: 5-600 s. Default 60.' }, + }, + }, + output: OUTPUT, + ...present('Ollama embed'), + async execute(args) { + try { + const spec = validateEmbed(args) + const show = await showOf(deps, spec.model) + const capabilities = Array.isArray(show.capabilities) ? show.capabilities.map(String) : [] + if (capabilities.length > 0 && !capabilities.includes('embedding')) { + return refusal( + 'NOT_EMBEDDING', + '"' + spec.model + '" is not an embedding model: it advertises ' + capabilities.join(', ') + '. An embedding model is different weights - `ollama pull nomic-embed-text` (about 274 MB) is the usual one, then call this with that tag.', + { model: spec.model, capabilities }, + ) + } + const body = { model: spec.model, input: spec.inputs } + if (spec.keepAlive !== undefined) body.keep_alive = spec.keepAlive + let vectors + let route = '/api/embed' + try { + const answer = await requestJson(deps, '/api/embed', { method: 'POST', body, timeoutMs: spec.timeoutMs, model: spec.model }) + const embeddings = Array.isArray(answer.data.embeddings) ? answer.data.embeddings : null + if (embeddings === null) throw new OllamaError('BAD_RESPONSE', 'The daemon answered /api/embed without an `embeddings` array.', { model: spec.model }) + vectors = embeddings + } catch (error) { + if (!(error instanceof OllamaError) || error.code !== 'NOT_FOUND') throw error + // An older daemon: one prompt per call, one vector per answer. + route = '/api/embeddings' + vectors = [] + for (const prompt of spec.inputs) { + const answer = await requestJson(deps, '/api/embeddings', { method: 'POST', body: { model: spec.model, prompt }, timeoutMs: spec.timeoutMs, model: spec.model }) + if (!Array.isArray(answer.data.embedding)) throw new OllamaError('BAD_RESPONSE', 'The daemon answered /api/embeddings without an `embedding` array.', { model: spec.model }) + vectors.push(answer.data.embedding) + } + } + if (!Array.isArray(vectors) || vectors.length !== spec.inputs.length) { + throw new OllamaError('BAD_RESPONSE', 'The daemon returned ' + (Array.isArray(vectors) ? vectors.length : 'no') + ' vector(s) for ' + spec.inputs.length + ' input(s).', { model: spec.model }) + } + const dimension = Array.isArray(vectors[0]) ? vectors[0].length : 0 + const similarity = spec.similarity && vectors.length >= 2 ? cosineSimilarity(vectors[0], vectors[1]) : null + const digest = (await tagsOf(deps).catch(() => [])).map(describeModel).find((entry) => entry.name === spec.model)?.digest ?? '' + const lines = [] + lines.push('EMBED ' + spec.model + (digest === '' ? '' : ' \u00b7 digest ' + shortDigest(digest)) + ' \u00b7 ' + vectors.length + ' input(s) \u00b7 dimension ' + dimension + (route === '/api/embeddings' ? ' \u00b7 via /api/embeddings (older daemon)' : '')) + for (const [index, vector] of vectors.entries()) { + const norm = vectorNorm(vector) + const preview = vector.slice(0, 8).map((value) => (typeof value === 'number' ? value.toFixed(4) : String(value))).join(', ') + const label = (index + 1) + '. "' + (spec.inputs[index].length > 60 ? spec.inputs[index].slice(0, 60) + '\u2026' : spec.inputs[index]) + '"' + lines.push(label + ' \u00b7 norm ' + (norm === null ? 'n/a' : norm.toFixed(4))) + lines.push(' [' + preview + (vector.length > 8 ? ', \u2026 (' + vector.length + ' of ' + dimension + ')' : '') + ']') + } + if (similarity !== null) { + lines.push('') + lines.push('cosine(1, 2) = ' + similarity.toFixed(4) + ' (1.0 identical direction, 0 orthogonal, -1 opposite)') + } + lines.push('') + lines.push('Conditions: ' + (spec.inputs.length === 1 ? '1 input' : spec.inputs.length + ' inputs') + ' \u00b7 ' + (spec.keepAlive === undefined ? 'keep_alive: daemon default' : 'keep_alive ' + String(spec.keepAlive)) + ' \u00b7 timeout ' + Math.round(spec.timeoutMs / 1000) + ' s.') + return { + text: lines.join('\n'), + view: { + ok: true, + model: spec.model, + digest, + dimension, + count: vectors.length, + similarity, + route, + norms: vectors.map((vector) => vectorNorm(vector)), + }, + } + } catch (error) { + if (error instanceof OllamaError && error.code === 'NO_MODEL') return refusal('NO_MODEL', await noModelText(deps, String(args.model ?? '')), { model: args.model }) + return failureOf(error) + } + }, + } + + return [status, run, bench, models, embed] +} + +/** A bench run's inference spec, from the plan. */ +function runSpec(plan, model, context) { + return { + model, + chat: false, + prompt: plan.prompt, + messages: [], + system: '', + options: plan.options, + format: plan.format, + keepAlive: undefined, + think: plan.think, + timeoutMs: plan.timeoutMs, + } +} + +/** The plan's options plus the sweep's context, without mutating the plan. */ +function optionsWithContext(options, context) { + return context === null ? { ...options } : { ...options, num_ctx: context } +} + +/** What a pull's NDJSON progress lines add up to. */ +export function pullSummary(items) { + let status = '' + let layers = 0 + let total = 0 + let completed = 0 + const seen = new Map() + for (const item of items) { + if (item === null || typeof item !== 'object') continue + if (typeof item.status === 'string' && item.status !== '') status = item.status + const digest = typeof item.digest === 'string' ? item.digest : '' + if (digest === '') continue + const layer = seen.get(digest) ?? { total: 0, completed: 0 } + if (typeof item.total === 'number') layer.total = item.total + if (typeof item.completed === 'number') layer.completed = item.completed + seen.set(digest, layer) + } + for (const layer of seen.values()) { + layers += 1 + total += layer.total + completed += layer.completed + } + return { status, layers, total, completed } +} + +/** + * One `model_info` value by SUFFIX. + * + * Ollama prefixes those keys with the architecture (`qwen2.context_length`, + * `nomic-bert.embedding_length`), and the prefix changes with the model - so + * matching only `llama.*` would report "n/a" for every model that is not a + * Llama. The suffix is the stable part. + */ +function infoValue(info, suffix) { + if (info === null || typeof info !== 'object') return null + for (const [key, value] of Object.entries(info)) { + if (typeof value === 'number' && (key === suffix || key.endsWith('.' + suffix))) return value + } + return null +} + +/** One model's metadata as prose: what the template and the parameters say. */ +function showText(model, data) { + const details = data && typeof data.details === 'object' && data.details !== null ? data.details : {} + const info = data && typeof data.model_info === 'object' && data.model_info !== null ? data.model_info : {} + const capabilities = Array.isArray(data.capabilities) ? data.capabilities : [] + const lines = [] + lines.push('MODEL ' + model) + lines.push('family ' + (details.family ?? 'n/a') + (Array.isArray(details.families) && details.families.length > 1 ? ' (' + details.families.join(', ') + ')' : '')) + lines.push('parameters ' + (details.parameter_size ?? 'n/a') + ' \u00b7 quantization ' + (details.quantization_level ?? 'n/a') + ' \u00b7 format ' + (details.format ?? 'n/a')) + lines.push('capabilities ' + (capabilities.length === 0 ? '(not reported)' : capabilities.join(', '))) + const contextLength = infoValue(info, 'context_length') + const embeddingLength = infoValue(info, 'embedding_length') + const blockCount = infoValue(info, 'block_count') + const architecture = typeof info['general.architecture'] === 'string' ? info['general.architecture'] : '' + if (architecture !== '' || contextLength !== null || blockCount !== null) { + lines.push('architecture ' + (architecture || 'n/a') + (contextLength === null ? '' : ' \u00b7 context ' + contextLength + ' tokens') + (blockCount === null ? '' : ' \u00b7 ' + blockCount + ' blocks')) + } + if (embeddingLength !== null) { + // `*.embedding_length` exists on every transformer - it is the hidden size. + // Calling it "embedding" on a completion model would tell a reader the model + // produces vectors, which is a different capability (and a different model). + lines.push((capabilities.includes('embedding') ? 'embedding ' : 'hidden size ') + embeddingLength + ' dimensions') + } + const parameters = typeof data.parameters === 'string' ? data.parameters.trim() : '' + if (parameters !== '') { + lines.push('') + lines.push('PARAMETERS (the model\'s own defaults)') + lines.push(parameters.split('\n').slice(0, 40).join('\n')) + } + const template = typeof data.template === 'string' ? data.template.trim() : '' + if (template !== '') { + lines.push('') + lines.push('TEMPLATE' + (template.length > 1200 ? ' (first 1200 characters)' : '')) + lines.push(template.slice(0, 1200)) + } + const license = typeof data.license === 'string' ? data.license.trim() : '' + if (license !== '') { + lines.push('') + lines.push('LICENSE' + (license.length > 800 ? ' (first 800 characters)' : '')) + lines.push(license.slice(0, 800)) + } + lines.push('') + lines.push('Next: `ollama_run` with this tag sends one inference, `ollama_bench` measures it, `ollama_models` action "delete" removes it.') + return lines.join('\n') +} + +/** + * The benchmark's answer. + * + * The per-run table is the evidence; the aggregate below it is the claim; the + * conditions line is what makes the claim checkable. The cold/warm comparison + * is deliberately stated in WALL and TTFT, not tok/s. + */ +export function benchText(plan, groups, config) { + const lines = [] + const models = [...new Set(groups.map((group) => group.model))] + lines.push('BENCH ' + models.join(' vs ') + ' \u00b7 ' + plan.runs + ' measured run(s) per group') + for (const group of groups) { + const digest = group.digest === '' ? '' : ' \u00b7 digest ' + shortDigest(group.digest) + const context = group.context === null ? 'model default' : String(group.context) + lines.push('') + lines.push('MODEL ' + group.model + digest + ' \u00b7 context ' + context) + lines.push(table( + ['RUN', 'TOK/S', 'TTFT', 'LOAD', 'EVAL', 'TOKENS', 'WALL'], + group.runs.map((entry) => [ + String(entry.index), + numberOr(entry.measured.evalTokensPerSecond, (value) => value.toFixed(1)), + numberOr(entry.measured.ttftMeasuredMs ?? entry.measured.ttftDerivedMs, (value) => formatMilliseconds(value)), + numberOr(entry.measured.loadMs, (value) => formatMilliseconds(value)), + numberOr(entry.measured.evalMs, (value) => formatMilliseconds(value)), + numberOr(entry.measured.evalCount, (value) => String(value)), + numberOr(entry.measured.wallMs, (value) => formatMilliseconds(value)), + ]), + )) + const rates = group.runs.map((entry) => entry.measured.evalTokensPerSecond) + const ttfts = group.runs.map((entry) => entry.measured.ttftMeasuredMs ?? entry.measured.ttftDerivedMs) + const walls = group.runs.map((entry) => entry.measured.wallMs) + const rateSummary = summarize(rates) + const ttftSummary = summarize(ttfts) + const wallSummary = summarize(walls) + const spread = rateSummary.spread === null ? 'n/a' : Math.round(rateSummary.spread * 100) + '%' + lines.push( + ' tok/s median ' + + numberOr(rateSummary.median, (value) => value.toFixed(1)) + + ' \u00b7 p95 ' + + numberOr(rateSummary.p95, (value) => value.toFixed(1)) + + ' \u00b7 min ' + + numberOr(rateSummary.min, (value) => value.toFixed(1)) + + ' \u00b7 max ' + + numberOr(rateSummary.max, (value) => value.toFixed(1)) + + ' \u00b7 spread ' + + spread, + ) + lines.push(' TTFT median ' + numberOr(ttftSummary.median, (value) => formatMilliseconds(value)) + ' \u00b7 p95 ' + numberOr(ttftSummary.p95, (value) => formatMilliseconds(value))) + lines.push(' wall median ' + numberOr(wallSummary.median, (value) => formatMilliseconds(value)) + ' \u00b7 p95 ' + numberOr(wallSummary.p95, (value) => formatMilliseconds(value))) + const first = group.runs[0] + const warmWalls = walls.slice(1).filter((value) => typeof value === 'number' && Number.isFinite(value)) + const warmWall = summarize(warmWalls) + const firstTtft = first === undefined ? null : first.measured.ttftMeasuredMs ?? first.measured.ttftDerivedMs + const warmTtft = summarize(ttfts.slice(1)).median + if (group.runs.length > 1 && typeof firstTtft === 'number' && typeof warmTtft === 'number' && warmTtft > 0) { + // The claim is about the FIRST TOKEN, because that is the phase the load + // delays. A wall-clock ratio would compare two numbers a fast local daemon + // can render as single-digit milliseconds - true, but not about the model. + let sentence = + ' first run first token at ' + + formatMilliseconds(firstTtft) + + ' (load ' + + numberOr(first.measured.loadMs, formatMilliseconds) + + ') \u00b7 warm runs ' + + windowsPhrase(group.runs.length - 1) + + ' at ' + + formatMilliseconds(warmTtft) + + ' median' + const ratio = firstTtft / warmTtft + if (ratio >= 1.05) sentence += ' \u00b7 ' + ratio.toFixed(2) + 'x later' + if (typeof first.measured.wallMs === 'number' && typeof warmWall.median === 'number' && first.measured.wallMs >= 50 && warmWall.median >= 10) { + sentence += ' \u00b7 wall ' + formatMilliseconds(first.measured.wallMs) + ' vs ' + formatMilliseconds(warmWall.median) + ' median' + } + lines.push(sentence) + } + lines.push(' note: tok/s is eval_count / eval_duration - the generation phase only - so a cold load raises TTFT and wall, never tok/s.' + (group.unloaded ? ' The model was unloaded with keep_alive 0 before run 1, so that run paid the load.' : '')) + } + lines.push('') + lines.push('Conditions: ' + conditionLine({ options: plan.options, keepAlive: undefined, timeoutMs: plan.timeoutMs }, { timeoutSeconds: plan.timeoutSeconds }) + '.') + lines.push('Runs: ' + plan.measuredCalls + ' measured call(s)' + (plan.warmupCalls > 0 ? ' plus ' + plan.warmupCalls + ' warmup' : '') + (plan.cold ? ', model unloaded before each group' : ', no unload (warm model)') + '.') + lines.push('Reading it: a median with a spread, at a stated context and a stated digest, is a measurement. The same model on another machine - or at another num_ctx - is a different number.') + if (config.remote) lines.push('Host: ' + config.origin + ' is not this machine (allowed by DSH_OLLAMA_ALLOW_REMOTE=1); every ms above includes the network.') + return lines.join('\n') +} + +/** "2-5" or "2", for the cold/warm sentence. */ +function windowsPhrase(count) { + if (count <= 0) return 'none' + if (count === 1) return 'run 2' + return '2-' + (count + 1) +} + +/** The bench's view: the per-run numbers, without the prose. */ +export function benchView(plan, groups, config) { + return { + ok: true, + runs: plan.runs, + measuredCalls: plan.measuredCalls, + cold: plan.cold, + options: plan.options, + host: { origin: config.origin, remote: config.remote }, + groups: groups.map((group) => ({ + model: group.model, + digest: group.digest, + context: group.context, + unloaded: group.unloaded === true, + runs: group.runs, + rates: summarize(group.runs.map((entry) => entry.measured.evalTokensPerSecond)), + ttft: summarize(group.runs.map((entry) => entry.measured.ttftMeasuredMs ?? entry.measured.ttftDerivedMs)), + wall: summarize(group.runs.map((entry) => entry.measured.wallMs)), + })), + } +} diff --git a/packages/dsh-ollama/lib/validate.js b/packages/dsh-ollama/lib/validate.js new file mode 100644 index 0000000..52b640e --- /dev/null +++ b/packages/dsh-ollama/lib/validate.js @@ -0,0 +1,467 @@ +/** + * dsh-ollama — the argument grammar, one function per tool. + * + * A tool schema already tells the model what it may send; this module is the + * second line, for the calls the registry's validator cannot make sense of: + * a `num_predict` that would run for an hour, a `messages` array with a role + * nobody implements, an option name that is a typo (`temprature` would be + * silently ignored by the daemon and quietly change what the benchmark + * measured), or a model tag with a space in it. + * + * Every refusal is a sentence naming the field, the value and the range - the + * model cannot fix an argument it cannot see. + */ + +import { OllamaError } from './errors.js' + +/** + * The ceilings this row enforces, in one place because the tool descriptions + * and the README quote them. + * + * `maxNumPredict` is the one that makes "ONE bounded inference" true: Ollama's + * own default is `-1` (generate until the stop token or the context fills), so + * every call this row sends carries an explicit `num_predict`. + */ +export const CAPS = { + maxNumPredict: 8192, + defaultNumPredict: 512, + benchNumPredict: 128, + maxPromptChars: 200_000, + maxMessages: 64, + maxMessageImages: 4, + maxImageChars: 8_000_000, + maxToolCallChars: 64_000, + maxFormatChars: 8192, + maxStopSequences: 8, + maxStopChars: 256, + maxModelChars: 200, + minRuns: 3, + defaultRuns: 5, + maxRuns: 50, + maxWarmup: 5, + maxModelsPerBench: 3, + maxContexts: 6, + maxBenchCalls: 150, + maxBenchWarmupCalls: 60, + maxEmbedInputs: 64, + maxEmbedChars: 20_000, + maxEmbedTotalChars: 100_000, + defaultRunTimeoutSeconds: 300, + minRunTimeoutSeconds: 10, + maxRunTimeoutSeconds: 900, + defaultBenchTimeoutSeconds: 300, + minBenchTimeoutSeconds: 30, + maxBenchTimeoutSeconds: 600, + defaultEmbedTimeoutSeconds: 60, + minEmbedTimeoutSeconds: 5, + maxEmbedTimeoutSeconds: 600, + defaultPullTimeoutSeconds: 1800, + maxPullTimeoutSeconds: 3600, + statusTimeoutMs: 4000, + readTimeoutMs: 15_000, +} + +/** A model name: lowercase name segments, an optional tag, an optional digest. */ +const MODEL_PATTERN = /^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?(?:\/[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?)*(?::[A-Za-z0-9._-]{1,128})?(?:@sha256:[a-f0-9]{64})?$/ + +/** Ollama's `options` map: ranges in the units the daemon uses. */ +const OPTION_SPECS = { + num_keep: { kind: 'integer', min: 0, max: 512 }, + seed: { kind: 'integer', min: 0, max: 2_147_483_647 }, + num_predict: { kind: 'integer', min: 1, max: CAPS.maxNumPredict }, + top_k: { kind: 'integer', min: 0, max: 1000 }, + top_p: { kind: 'number', min: 0, max: 1 }, + min_p: { kind: 'number', min: 0, max: 1 }, + tfs_z: { kind: 'number', min: 0, max: 5 }, + typical_p: { kind: 'number', min: 0, max: 1 }, + repeat_last_n: { kind: 'integer', min: -1, max: 8192 }, + temperature: { kind: 'number', min: 0, max: 2 }, + repeat_penalty: { kind: 'number', min: 0, max: 10 }, + presence_penalty: { kind: 'number', min: -2, max: 2 }, + frequency_penalty: { kind: 'number', min: -2, max: 2 }, + mirostat: { kind: 'integer', min: 0, max: 2 }, + mirostat_tau: { kind: 'number', min: 0, max: 10 }, + mirostat_eta: { kind: 'number', min: 0, max: 1 }, + num_ctx: { kind: 'integer', min: 128, max: 1_048_576 }, + num_batch: { kind: 'integer', min: 1, max: 4096 }, + num_gpu: { kind: 'integer', min: 0, max: 999 }, + main_gpu: { kind: 'integer', min: 0, max: 99 }, + num_thread: { kind: 'integer', min: 1, max: 256 }, + penalize_newline: { kind: 'boolean' }, + numa: { kind: 'boolean' }, + low_vram: { kind: 'boolean' }, + f16_kv: { kind: 'boolean' }, + vocab_only: { kind: 'boolean' }, + use_mmap: { kind: 'boolean' }, + use_mlock: { kind: 'boolean' }, + truncate: { kind: 'boolean' }, + shift: { kind: 'boolean' }, + stop: { kind: 'stop' }, +} + +/** Every option name this row sends, in the order the tool schema lists them. */ +export const OPTION_KEYS = Object.keys(OPTION_SPECS) + +/** Refuse with a code and a sentence that names the field. */ +function fail(code, message) { + throw new OllamaError(code, message) +} + +/** A required, non-empty string. */ +function requireString(value, field, maxChars = CAPS.maxModelChars) { + if (typeof value !== 'string') fail('BAD_ARGUMENT', '`' + field + '` must be a string (got ' + (value === null ? 'null' : typeof value) + ').') + const text = value + if (text.trim() === '') fail('BAD_ARGUMENT', '`' + field + '` is empty; give it a value.') + if (text.length > maxChars) fail('BAD_ARGUMENT', '`' + field + '` is ' + text.length + ' characters, over the ' + maxChars + '-character ceiling this row accepts.') + return text +} + +/** + * A model tag. + * + * The grammar is Ollama's own (`name[:tag]`, a `namespace/` prefix, an + * optional `@sha256:` digest for a pull); it is closed here so a value cannot + * carry a space, a flag-looking `-` or a path separator into a later request. + * + * @param value - the caller's value. + * @param field - the argument name, for the refusal. + * @param options - `{ digest }` allows the `@sha256:` form (pull only). + */ +export function validateModelTag(value, field = 'model', options = {}) { + const text = requireString(value, field, CAPS.maxModelChars).trim() + if (text.includes('@') && options.digest !== true) { + fail('BAD_MODEL', '`' + field + '` is "' + text + '"; only a pull may name a digest (@sha256:...), and nothing else may contain "@".') + } + if (!MODEL_PATTERN.test(text)) { + fail( + 'BAD_MODEL', + '`' + + field + + '` is "' + + text + + '", which is not a model tag. Use the tag `ollama list` prints: lowercase name, optional ":tag" (e.g. qwen2.5:7b, llama3.2:1b, hf.co/user/model:Q4_K_M).', + ) + } + return text +} + +/** + * `keep_alive`: seconds (number or numeric string) or a duration string. + * + * `0` unloads the model when the request ends, `-1` keeps it loaded forever; + * both are accepted because both are documented daemon behaviour. + */ +export function validateKeepAlive(value) { + if (value === undefined || value === null) return undefined + if (typeof value === 'number') { + if (!Number.isFinite(value) || value < -1) fail('BAD_KEEP_ALIVE', '`keep_alive` is ' + value + '; use -1 (forever), 0 (unload when this request ends), or a positive number of seconds.') + return value + } + const text = requireString(value, 'keep_alive', 32).trim() + if (/^-?\d+$/.test(text)) { + const seconds = Number(text) + if (seconds < -1) fail('BAD_KEEP_ALIVE', '`keep_alive` is ' + seconds + '; use -1, 0, or a positive number of seconds.') + return seconds + } + if (!/^-?\d+(\.\d+)?(ns|us|ms|s|m|h)$/.test(text)) { + fail('BAD_KEEP_ALIVE', '`keep_alive` is "' + text + '"; use a duration such as "5m", "30s", "1h", or 0 / -1.') + } + return text +} + +/** + * `format`: unset, the word "json", or a JSON Schema object. + * + * A schema is passed straight to the daemon, so it is size-capped and must be + * a plain object - a string that merely looks like JSON would be sent as a + * string and rejected by Ollama with a far worse message. + */ +export function validateFormat(value) { + if (value === undefined || value === null) return undefined + if (typeof value === 'string') { + if (value.trim().toLowerCase() !== 'json') fail('BAD_FORMAT', '`format` is "' + value + '"; use "json" or a JSON Schema object (structured outputs are a schema, not a mode name other than json).') + return 'json' + } + if (typeof value !== 'object' || Array.isArray(value)) fail('BAD_FORMAT', '`format` must be "json" or a JSON Schema object.') + const text = JSON.stringify(value) + if (text.length > CAPS.maxFormatChars) fail('BAD_FORMAT', '`format` is ' + text.length + ' characters, over the ' + CAPS.maxFormatChars + '-character ceiling for a schema.') + return value +} + +/** One `stop` value: a string or up to eight strings. */ +function validateStop(value) { + const list = Array.isArray(value) ? value : [value] + if (list.length === 0) fail('BAD_OPTION', '`options.stop` is an empty array; give it at least one sequence or omit it.') + if (list.length > CAPS.maxStopSequences) fail('BAD_OPTION', '`options.stop` carries ' + list.length + ' sequences, over the ' + CAPS.maxStopSequences + '-sequence ceiling.') + for (const entry of list) { + if (typeof entry !== 'string' || entry === '') fail('BAD_OPTION', '`options.stop` must be a string or an array of non-empty strings.') + if (entry.length > CAPS.maxStopChars) fail('BAD_OPTION', '`options.stop` has a ' + entry.length + '-character sequence, over the ' + CAPS.maxStopChars + '-character ceiling.') + } + return list +} + +/** + * The `options` map, closed to names this row does not know. + * + * Refusing an unknown name is deliberate: a mistyped `temprature` is not an + * error to the daemon - it is IGNORED - so the run would report numbers taken + * at a temperature the caller never chose. + */ +export function validateOptions(value) { + if (value === undefined || value === null) return {} + if (typeof value !== 'object' || Array.isArray(value)) fail('BAD_OPTION', '`options` must be an object: { "temperature": 0, "num_predict": 128 }.') + const normalized = {} + for (const [key, raw] of Object.entries(value)) { + const spec = OPTION_SPECS[key] + if (spec === undefined) { + fail( + 'BAD_OPTION', + '`options.' + + key + + '` is not an option this row sends. Known options: ' + + OPTION_KEYS.join(', ') + + '. (Ollama silently ignores an unknown option, so a typo would change what these numbers measured.)', + ) + } + if (raw === undefined || raw === null) continue + if (spec.kind === 'boolean') { + if (typeof raw !== 'boolean') fail('BAD_OPTION', '`options.' + key + '` must be true or false.') + normalized[key] = raw + continue + } + if (spec.kind === 'stop') { + normalized[key] = validateStop(raw) + continue + } + if (typeof raw !== 'number' || !Number.isFinite(raw)) fail('BAD_OPTION', '`options.' + key + '` must be a number (got ' + (raw === null ? 'null' : typeof raw) + ').') + if (spec.kind === 'integer' && !Number.isInteger(raw)) fail('BAD_OPTION', '`options.' + key + '` must be an integer (got ' + raw + ').') + if (raw < spec.min || raw > spec.max) fail('BAD_OPTION', '`options.' + key + '` is ' + raw + ', outside ' + spec.min + '..' + spec.max + '.') + normalized[key] = raw + } + return normalized +} + +/** A prompt: non-empty, capped. Whitespace is kept - it can be the prompt. */ +export function validatePrompt(value, field = 'prompt') { + if (typeof value !== 'string') fail('BAD_ARGUMENT', '`' + field + '` must be a string.') + if (value.trim() === '') fail('BAD_ARGUMENT', '`' + field + '` is empty; give it something to answer.') + if (value.length > CAPS.maxPromptChars) fail('BAD_ARGUMENT', '`' + field + '` is ' + value.length + ' characters, over the ' + CAPS.maxPromptChars + '-character ceiling for one call.') + return value +} + +/** + * `messages` for `/api/chat`: role, content, optional images and tool fields. + * + * Images are accepted because a vision model's API is the same endpoint; they + * are capped per message and in total because base64 is carried through the + * conversation as text and an unbounded one is a prompt of a different size + * entirely. + */ +export function validateMessages(value) { + if (!Array.isArray(value) || value.length === 0) fail('BAD_MESSAGES', '`messages` must be a non-empty array of { role, content }.') + if (value.length > CAPS.maxMessages) fail('BAD_MESSAGES', '`messages` carries ' + value.length + ' messages, over the ' + CAPS.maxMessages + '-message ceiling.') + const allowed = ['role', 'content', 'images', 'tool_calls', 'tool_name'] + const roles = ['system', 'user', 'assistant', 'tool'] + let totalChars = 0 + const normalized = [] + for (const [index, entry] of value.entries()) { + if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) fail('BAD_MESSAGES', '`messages[' + index + ']` must be an object { role, content }.') + for (const key of Object.keys(entry)) { + if (!allowed.includes(key)) { + fail('BAD_MESSAGES', '`messages[' + index + '].' + key + '` is not a message field. Allowed: ' + allowed.join(', ') + '.') + } + } + const role = String(entry.role ?? '') + if (!roles.includes(role)) fail('BAD_MESSAGES', '`messages[' + index + '].role` is "' + role + '"; use one of ' + roles.join(', ') + '.') + if (typeof entry.content !== 'string') fail('BAD_MESSAGES', '`messages[' + index + '].content` must be a string.') + totalChars += entry.content.length + const message = { role, content: entry.content } + if (entry.images !== undefined) { + if (!Array.isArray(entry.images) || entry.images.length === 0) fail('BAD_MESSAGES', '`messages[' + index + '].images` must be a non-empty array of base64 strings.') + if (entry.images.length > CAPS.maxMessageImages) fail('BAD_MESSAGES', '`messages[' + index + '].images` carries ' + entry.images.length + ' images, over the ' + CAPS.maxMessageImages + '-image ceiling per message.') + for (const image of entry.images) { + if (typeof image !== 'string' || image === '') fail('BAD_MESSAGES', '`messages[' + index + '].images` must contain base64 strings.') + if (!/^[A-Za-z0-9+/=\r\n]+$/.test(image)) fail('BAD_MESSAGES', '`messages[' + index + '].images` must be raw base64 (no data: URL prefix).') + if (image.length > CAPS.maxImageChars) fail('BAD_MESSAGES', '`messages[' + index + ']` has an image of ' + image.length + ' base64 characters, over the ' + CAPS.maxImageChars + '-character ceiling.') + totalChars += image.length + } + message.images = entry.images + } + if (entry.tool_name !== undefined) { + if (typeof entry.tool_name !== 'string' || entry.tool_name === '') fail('BAD_MESSAGES', '`messages[' + index + '].tool_name` must be a non-empty string.') + message.tool_name = entry.tool_name + } + if (entry.tool_calls !== undefined) { + if (!Array.isArray(entry.tool_calls)) fail('BAD_MESSAGES', '`messages[' + index + '].tool_calls` must be an array.') + const text = JSON.stringify(entry.tool_calls) + if (text.length > CAPS.maxToolCallChars) fail('BAD_MESSAGES', '`messages[' + index + '].tool_calls` is ' + text.length + ' characters, over the ' + CAPS.maxToolCallChars + '-character ceiling.') + message.tool_calls = entry.tool_calls + } + normalized.push(message) + } + if (totalChars > CAPS.maxPromptChars) fail('BAD_MESSAGES', 'The whole `messages` array is ' + totalChars + ' characters, over the ' + CAPS.maxPromptChars + '-character ceiling for one call.') + return normalized +} + +/** A timeout in seconds, clamped to the named range. */ +export function validateSeconds(value, field, fallback, min, max) { + if (value === undefined || value === null) return fallback + if (typeof value !== 'number' || !Number.isFinite(value)) fail('BAD_ARGUMENT', '`' + field + '` must be a number of seconds.') + if (value < min || value > max) fail('BAD_ARGUMENT', '`' + field + '` is ' + value + ' s, outside ' + min + '..' + max + ' s. This row will not run a call it cannot bound.') + return value +} + +/** + * One inference: `prompt` XOR `messages`, plus the options map. + * + * `num_predict` gets a value whether or not the caller sent one: Ollama's + * default is unbounded generation, and this row's contract is ONE bounded + * inference. + */ +export function validateRun(args = {}) { + const model = validateModelTag(args.model, 'model') + const hasPrompt = args.prompt !== undefined && args.prompt !== null + const hasMessages = args.messages !== undefined && args.messages !== null + if (hasPrompt === hasMessages) { + fail('BAD_ARGUMENT', '`ollama_run` takes `prompt` (one string) OR `messages` (a chat array), exactly one. ' + (hasPrompt ? 'Both were sent.' : 'Neither was sent.')) + } + if (hasMessages && args.system !== undefined) { + fail('BAD_ARGUMENT', '`system` belongs to `prompt`; in a `messages` call put it in its own message: { "role": "system", "content": "..." }.') + } + const options = validateOptions(args.options) + if (options.num_predict === undefined) options.num_predict = CAPS.defaultNumPredict + return { + model, + chat: hasMessages, + prompt: hasPrompt ? validatePrompt(args.prompt) : '', + messages: hasMessages ? validateMessages(args.messages) : [], + system: args.system === undefined ? '' : requireString(args.system, 'system', CAPS.maxPromptChars), + options, + format: validateFormat(args.format), + keepAlive: validateKeepAlive(args.keep_alive), + think: args.think === undefined ? undefined : args.think === true, + timeoutMs: validateSeconds(args.timeoutSeconds, 'timeoutSeconds', CAPS.defaultRunTimeoutSeconds, CAPS.minRunTimeoutSeconds, CAPS.maxRunTimeoutSeconds) * 1000, + } +} + +/** + * A benchmark plan: 3+ runs, one variable at a time, a bounded call count. + * + * `runs` is refused below three because one run is not a measurement; the + * aggregate the tool prints is a median with a spread, and a median of one + * number is a claim the sample cannot carry. + */ +export function validateBench(args = {}) { + const single = args.model === undefined || args.model === null ? null : validateModelTag(args.model, 'model') + const many = args.models === undefined || args.models === null ? null : args.models + if (single === null && many === null) fail('BAD_ARGUMENT', '`ollama_bench` needs `model` (one tag) or `models` (an A/B array of 1-3 tags).') + if (single !== null && many !== null) fail('BAD_ARGUMENT', 'Send `model` OR `models`, not both: `models` is the A/B form and `model` is the single-model shorthand.') + let models + if (many !== null) { + if (!Array.isArray(many) || many.length === 0) fail('BAD_ARGUMENT', '`models` must be a non-empty array of model tags.') + if (many.length > CAPS.maxModelsPerBench) fail('BAD_ARGUMENT', '`models` carries ' + many.length + ' tags, over the ' + CAPS.maxModelsPerBench + '-model ceiling for one A/B run.') + models = many.map((entry, index) => validateModelTag(entry, 'models[' + index + ']')) + } else { + models = [single] + } + if (new Set(models).size !== models.length) fail('BAD_ARGUMENT', '`models` names the same tag twice; an A/B needs two different models.') + const prompt = validatePrompt(args.prompt, 'prompt') + const runs = args.runs === undefined ? CAPS.defaultRuns : args.runs + if (typeof runs !== 'number' || !Number.isInteger(runs)) fail('BAD_ARGUMENT', '`runs` must be an integer.') + if (runs < CAPS.minRuns) { + fail( + 'BAD_RUNS', + '`runs` is ' + + runs + + '; a benchmark needs at least ' + + CAPS.minRuns + + '. One run is a number, not a measurement - a single tok/s cannot say whether the next run will be faster or slower.', + ) + } + if (runs > CAPS.maxRuns) fail('BAD_ARGUMENT', '`runs` is ' + runs + ', over the ' + CAPS.maxRuns + '-run ceiling for one benchmark.') + const warmup = args.warmup === undefined ? 0 : args.warmup + if (typeof warmup !== 'number' || !Number.isInteger(warmup) || warmup < 0 || warmup > CAPS.maxWarmup) { + fail('BAD_ARGUMENT', '`warmup` must be an integer 0..' + CAPS.maxWarmup + ' (calls made before the measured runs and not counted).') + } + const cold = args.cold !== false + const contexts = args.contexts === undefined || args.contexts === null ? [] : args.contexts + if (!Array.isArray(contexts)) fail('BAD_ARGUMENT', '`contexts` must be an array of context lengths, e.g. [2048, 8192].') + if (contexts.length > CAPS.maxContexts) fail('BAD_ARGUMENT', '`contexts` carries ' + contexts.length + ' values, over the ' + CAPS.maxContexts + '-value ceiling for one sweep.') + const seen = new Set() + for (const entry of contexts) { + if (typeof entry !== 'number' || !Number.isInteger(entry) || entry < 128 || entry > 1_048_576) fail('BAD_ARGUMENT', '`contexts` must be integers 128..1048576 (got ' + entry + ').') + if (seen.has(entry)) fail('BAD_ARGUMENT', '`contexts` names ' + entry + ' twice; a sweep needs distinct values.') + seen.add(entry) + } + const options = validateOptions(args.options) + if (options.num_predict === undefined) options.num_predict = CAPS.benchNumPredict + if (options.temperature === undefined) options.temperature = 0 + if (options.seed === undefined) options.seed = 42 + const timeoutSeconds = validateSeconds(args.timeoutSeconds, 'timeoutSeconds', CAPS.defaultBenchTimeoutSeconds, CAPS.minBenchTimeoutSeconds, CAPS.maxBenchTimeoutSeconds) + const groups = models.length * Math.max(1, contexts.length) + const measured = groups * runs + const warmupCalls = groups * warmup + if (measured > CAPS.maxBenchCalls) { + fail('BAD_ARGUMENT', 'This plan is ' + measured + ' measured calls (' + models.length + ' model(s) x ' + Math.max(1, contexts.length) + ' context(s) x ' + runs + ' runs), over the ' + CAPS.maxBenchCalls + '-call ceiling. Lower `runs`, `contexts` or `models`.') + } + if (warmupCalls > CAPS.maxBenchWarmupCalls) { + fail('BAD_ARGUMENT', 'This plan is ' + warmupCalls + ' warmup calls, over the ' + CAPS.maxBenchWarmupCalls + '-call ceiling. Lower `warmup` or the plan.') + } + return { + models, + prompt, + runs, + warmup, + cold, + contexts, + options, + format: validateFormat(args.format), + think: args.think === undefined ? undefined : args.think === true, + timeoutMs: timeoutSeconds * 1000, + timeoutSeconds, + measuredCalls: measured, + warmupCalls, + } +} + +/** Enumerations this row accepts for the writer tools. */ +export const MODEL_ACTIONS = ['list', 'show', 'pull', 'delete'] + +/** `ollama_models`: the action, the tag it names, and the confirmation. */ +export function validateModels(args = {}) { + const action = args.action === undefined || args.action === null ? 'list' : String(args.action).toLowerCase() + if (!MODEL_ACTIONS.includes(action)) fail('BAD_ARGUMENT', '`action` is "' + action + '"; use one of ' + MODEL_ACTIONS.join(', ') + '.') + if (action === 'list') return { action, model: '', confirm: '', timeoutSeconds: 0 } + const model = validateModelTag(args.model, 'model', { digest: action === 'pull' }) + const confirm = args.confirm === undefined || args.confirm === null ? '' : String(args.confirm) + // A pull downloads gigabytes, so its deadline is the one a person may need to + // raise - and the TIMEOUT refusal names this argument. + const timeoutSeconds = action === 'pull' ? validateSeconds(args.timeoutSeconds, 'timeoutSeconds', CAPS.defaultPullTimeoutSeconds, 60, CAPS.maxPullTimeoutSeconds) : 0 + return { action, model, confirm, timeoutSeconds } +} + +/** `ollama_embed`: one or more strings, and the comparability question. */ +export function validateEmbed(args = {}) { + const model = validateModelTag(args.model, 'model') + const raw = args.input + const list = Array.isArray(raw) ? raw : [raw] + if (raw === undefined || raw === null) fail('BAD_ARGUMENT', '`input` is required: one string, or an array of strings.') + if (list.length === 0) fail('BAD_ARGUMENT', '`input` is an empty array; give it at least one string.') + if (list.length > CAPS.maxEmbedInputs) fail('BAD_ARGUMENT', '`input` carries ' + list.length + ' strings, over the ' + CAPS.maxEmbedInputs + '-input ceiling for one call.') + let total = 0 + const inputs = list.map((entry, index) => { + if (typeof entry !== 'string') fail('BAD_ARGUMENT', '`input[' + index + ']` must be a string.') + if (entry === '') fail('BAD_ARGUMENT', '`input[' + index + ']` is empty; an embedding of nothing is not an embedding.') + if (entry.length > CAPS.maxEmbedChars) fail('BAD_ARGUMENT', '`input[' + index + ']` is ' + entry.length + ' characters, over the ' + CAPS.maxEmbedChars + '-character ceiling.') + total += entry.length + return entry + }) + if (total > CAPS.maxEmbedTotalChars) fail('BAD_ARGUMENT', 'The whole `input` is ' + total + ' characters, over the ' + CAPS.maxEmbedTotalChars + '-character ceiling for one call.') + return { + model, + inputs, + similarity: args.similarity === undefined ? inputs.length === 2 : args.similarity === true, + timeoutMs: validateSeconds(args.timeoutSeconds, 'timeoutSeconds', CAPS.defaultEmbedTimeoutSeconds, CAPS.minEmbedTimeoutSeconds, CAPS.maxEmbedTimeoutSeconds) * 1000, + keepAlive: validateKeepAlive(args.keep_alive), + } +} diff --git a/packages/dsh-ollama/package.json b/packages/dsh-ollama/package.json new file mode 100644 index 0000000..ec5d596 --- /dev/null +++ b/packages/dsh-ollama/package.json @@ -0,0 +1,29 @@ +{ + "name": "dsh-ollama", + "version": "0.1.0-alpha.1", + "description": "Ollama as a measured surface: five `ollama_*` tools over the machine's own daemon (status, one bounded run with its token counts and durations, a benchmark that reports 3+ runs as a median with spread, models with confirmed pull/delete, embeddings with cosine similarity), the loopback fence that refuses a daemon on another machine by default, and three skills that teach the REST API, the metric arithmetic and the Python client. Host-only: one row, no route, no browser half, zero npm dependencies, no bundled engine, nothing started on its own.", + "type": "module", + "main": "lib/index.js", + "exports": { + ".": "./lib/index.js", + "./package.json": "./package.json" + }, + "files": [ + "lib", + "skills", + "checks", + "cordis.patch.yml" + ], + "scripts": { + "check": "node checks/check-ollama-node.mjs" + }, + "engines": { + "node": ">=22" + }, + "dsh": { + "bundle": { + "patch": "./cordis.patch.yml" + } + }, + "license": "AGPL-3.0-only" +} diff --git a/packages/dsh-ollama/skills/ollama-api/SKILL.md b/packages/dsh-ollama/skills/ollama-api/SKILL.md new file mode 100644 index 0000000..8dd8d7e --- /dev/null +++ b/packages/dsh-ollama/skills/ollama-api/SKILL.md @@ -0,0 +1,366 @@ +--- +name: ollama-api +description: The Ollama REST API as it is on this machine - /api/generate and /api/chat with streaming NDJSON, /api/embed vs the older /api/embeddings, /api/tags, /api/ps, /api/show, pull progress, create/copy/delete, the OpenAI-compatible /v1 endpoints, the options map, keep_alive, the error shapes, and the environment variables that change what the daemon does. +whenToUse: Use before writing any code that calls Ollama over HTTP, before debugging a request that returns 404 / 500 / an empty body, when a stream seems to end early, when structured output or tool calling is wanted, or when choosing between the native API and the OpenAI-compatible one. +--- + +# The Ollama REST API + +The daemon is a plain HTTP server on the loopback interface. Everything below is +the shape this install answers with - not a client library's wrapper. + +## The address + +Ollama binds `127.0.0.1:11434` unless `OLLAMA_HOST` says otherwise. The value is +`host:port` (a scheme is accepted). Two different readers use it: + +- the **daemon** reads `OLLAMA_HOST` as the address to BIND (`0.0.0.0:11434` to + serve the whole network); +- a **client** reads it as the address to CONNECT to. + +That asymmetry is the classic mistake: setting `OLLAMA_HOST=0.0.0.0:11434` for +the server and then pointing a client at `http://0.0.0.0:11434` works on Linux +and fails on Windows. Clients should default to `http://127.0.0.1:11434`. + +The first request of any session should be the version, because every other +behaviour is version-dependent: + +```bash +curl -s http://127.0.0.1:11434/api/version +``` + +```json +{"version":"0.40.3"} +``` + +`dsh-ollama`'s `ollama_status` answers this and the model inventory in one call; +when you are writing a program, this is the endpoint to probe. + +## The three request shapes + +### One prompt: `/api/generate` + +```bash +curl -s http://127.0.0.1:11434/api/generate -d '{"model":"qwen2.5:7b","prompt":"Why is the sky blue?","stream":false}' +``` + +`stream: false` answers ONE JSON object carrying the whole completion and every +timing field. `stream: true` (the daemon's default) answers NDJSON - see below. + +### A conversation: `/api/chat` + +```bash +curl -s http://127.0.0.1:11434/api/chat -d '{"model":"qwen2.5:7b","messages":[{"role":"system","content":"Be brief."},{"role":"user","content":"Why is the sky blue?"}],"stream":false}' +``` + +Roles are `system`, `user`, `assistant` and `tool`. The same model, template and +options apply; the difference is that the daemon applies the chat template to +the whole array instead of wrapping one prompt. + +### Vectors: `/api/embed` + +```bash +curl -s http://127.0.0.1:11434/api/embed -d '{"model":"nomic-embed-text","input":["the cat sat","a cat was sitting"]}' +``` + +`input` is a string or an array. The answer is `{"embeddings": [[...], [...]], +"model": "...", ...}` - one vector per input, in order. Older daemons only have +the single-prompt form: + +```bash +curl -s http://127.0.0.1:11434/api/embeddings -d '{"model":"nomic-embed-text","prompt":"the cat sat"}' +``` + +That answers `{"embedding": [...]}`. Prefer `/api/embed`: one request, many +inputs, and a dimension you can rely on. If `/api/embed` returns 404, fall back. + +## Streaming is NDJSON + +A streaming answer is one JSON object per line, each line terminated by `\n`, +with `done: false` on every partial object and `done: true` on the last one. The +partial objects carry `response` (generate) or `message.content` (chat); the +FINAL object carries the token counts and the durations. Nothing before the last +line has a measurement in it. + +```bash +curl -s http://127.0.0.1:11434/api/generate -d '{"model":"qwen2.5:7b","prompt":"Say hello"}' +``` + +Three properties follow from that shape, and each one is a bug waiting to +happen: + +1. **A line is not a document.** Parse per line; do not buffer the whole body + and `JSON.parse` it. +2. **The last line is the measurement.** If your reader stops at the first + `done: true` it has the numbers; if it stops at end-of-stream it may have + them too, but the `done` object is the contract. +3. **A stream can be cut.** A cancelled request, a killed daemon or a timeout + ends the stream with no `done` object at all. Treat "no done object" as a + failed measurement, not as an empty answer. + +A reader that keeps the text and the final object: + +```javascript +const response = await fetch('http://127.0.0.1:11434/api/generate', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'qwen2.5:7b', prompt: 'Say hello', stream: true }), +}) +if (!response.ok) throw new Error(`HTTP ${response.status}: ${await response.text()}`) +const reader = response.body.getReader() +const decoder = new TextDecoder() +let buffer = '' +let text = '' +let final = null +for (;;) { + const { value, done } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + let newline = buffer.indexOf('\n') + while (newline >= 0) { + const line = buffer.slice(0, newline).trim() + buffer = buffer.slice(newline + 1) + newline = buffer.indexOf('\n') + if (line === '') continue + const item = JSON.parse(line) + if (item.response) text += item.response + if (item.done === true) final = item + } +} +if (final === null) throw new Error('the stream ended without a done object') +console.log(text, final.eval_count, final.eval_duration) +``` + +(The arithmetic that turns those two numbers into tok/s is the +`ollama-inference-metrics` skill's subject.) + +## The final object + +`/api/generate` with `stream: false` and the last line of a streaming call have +the same fields: + +| field | unit | what it covers | +|---|---|---| +| `response` | string | the generated text (chat: `message.content`) | +| `done` | bool | `true` on the final object | +| `done_reason` | string | `stop` (the model stopped) or `length` (it hit `num_predict`/context) | +| `context` | int[] | the token ids of prompt+answer - pass it back as `context` to continue | +| `total_duration` | ns | the whole daemon-side request | +| `load_duration` | ns | loading the weights (0 when they were resident) | +| `prompt_eval_count` | int | prompt tokens evaluated | +| `prompt_eval_duration` | ns | evaluating the prompt | +| `eval_count` | int | tokens generated | +| `eval_duration` | ns | generating them | + +`done_reason: "length"` is the one field a reader must not ignore: the answer +was CUT by your `num_predict` or the context window, not finished by the model. + +## Options and `keep_alive` + +`options` is the sampling/runtime map. The ones that change the measurement: + +| option | range | effect | +|---|---|---| +| `num_predict` | -1..N | max tokens to generate; `-1` (the default) means "until the model stops or the context fills" | +| `temperature` | 0..2 | 0 = greedy; the only defensible setting for a reproducible benchmark | +| `seed` | int | fixes sampling when temperature > 0; set it anyway so a rerun is comparable | +| `num_ctx` | 128.. | context window in tokens; the single biggest memory/speed knob | +| `top_k`, `top_p`, `min_p` | | sampling cutoffs | +| `repeat_penalty`, `repeat_last_n` | | repetition control | +| `stop` | string[] | stop sequences | +| `num_gpu` | int | layers offloaded to the GPU (0 = CPU only) | +| `num_thread` | int | CPU threads | +| `use_mmap`, `use_mlock` | bool | weight paging | + +Anything the daemon does not recognise in `options` is **silently ignored** - +which is why a typo is worse than an error: the run reports numbers taken at a +setting nobody chose. `dsh-ollama` closes the map for exactly that reason. + +`keep_alive` is a TOP-LEVEL sibling of `options`, not an option: + +- `"5m"`, `"30s"`, `"1h"` - keep the weights loaded that long after the request; +- `0` - unload as soon as this request finishes; +- `-1` - never unload. + +```bash +curl -s http://127.0.0.1:11434/api/generate -d '{"model":"qwen2.5:7b","prompt":"hi","keep_alive":"10m"}' +``` + +Unloading is the documented use of an empty request: + +```bash +curl -s http://127.0.0.1:11434/api/generate -d '{"model":"qwen2.5:7b","keep_alive":0}' +``` + +That is the call that makes a "cold" benchmark honest: without it, run 1 is +as warm as run 2 and the load time you meant to measure never appears. + +## Models: list, inspect, pull, delete + +```bash +curl -s http://127.0.0.1:11434/api/tags +``` + +`/api/tags` is the inventory: `name`, `size` (bytes), `digest` (`sha256:...`), +`modified_at`, and `details` (`family`, `families`, `parameter_size`, +`quantization_level`, `format`). + +```bash +curl -s http://127.0.0.1:11434/api/ps +``` + +`/api/ps` is what is LOADED right now: `size`, `size_vram`, `context_length`, +`expires_at`, and the same `details`. `size_vram` is 0 for a CPU-resident model; +`size_vram / size` is the offload share. + +```bash +curl -s http://127.0.0.1:11434/api/show -d '{"model":"qwen2.5:7b"}' +``` + +`/api/show` is the model's own metadata: `details`, `parameters` (the Modelfile +defaults - read them before assuming `temperature`), `template`, `license`, +`model_info` (`general.architecture`, `*.context_length`, `*.embedding_length`, +`*.block_count`) and `capabilities` (`completion`, `tools`, `embedding`, +`vision`, `thinking` depending on the model). `capabilities` is what tells you +whether a tag can embed at all. + +```bash +curl -s http://127.0.0.1:11434/api/pull -d '{"model":"qwen2.5:7b","stream":true}' +``` + +A pull is NDJSON progress, not a completion: `{"status":"pulling manifest"}`, +then one line per layer with `digest`, `total` and `completed`, then +`{"status":"verifying sha256 digest"}`, then `{"status":"success"}`. It is +measured in gigabytes and minutes; there is no partial-success state - a cut +pull leaves layers cached and the next pull resumes them. + +```bash +curl -s -X DELETE http://127.0.0.1:11434/api/delete -d '{"model":"llama3.2:1b"}' +``` + +Delete answers `200` with an empty body. It frees the bytes immediately; the +model comes back only by pulling it again. `dsh-ollama` requires the tag +repeated as `confirm` for this reason. + +The other model verbs, for completeness: + +```bash +curl -s http://127.0.0.1:11434/api/copy -d '{"source":"qwen2.5:7b","destination":"qwen2.5:7b-mine"}' +curl -s http://127.0.0.1:11434/api/create -d '{"model":"my-model","from":"qwen2.5:7b","system":"You answer in one sentence."}' +``` + +`copy` and `create` are not tools in this pack - they mutate the model store in +ways an agent should not do implicitly - but they are part of the surface and +worth knowing when a user asks why a tag exists. + +## Errors, and what they mean + +Ollama reports a problem as a non-2xx status with a JSON body `{"error": "..."}`. + +| status | body | meaning | the fix | +|---|---|---|---| +| 404 | `model "x" not found, try pulling it first` | the tag is not on disk | `ollama pull x` | +| 404 | `404 page not found` | the route does not exist on this daemon (an older build, or a typo) | check `/api/version`, use the older shape | +| 400 | `invalid options` / `json: cannot unmarshal ...` | the body's shape is wrong | compare with this page | +| 500 | `...` | the model failed while loading or running | read the message; often memory (`num_ctx` too large) | +| (none) | `fetch failed` / `ECONNREFUSED` | the DAEMON is not running | `ollama serve`, or the desktop app | + +A 500 whose message mentions memory is not a bug in your code: `num_ctx` sizes +the KV cache, and a context the machine cannot hold fails at load. Try a smaller +`num_ctx` or a smaller quantization. + +## Structured output + +`format` is a top-level field. `"json"` asks for any valid JSON; a JSON Schema +asks for that shape and is the reliable form: + +```bash +curl -s http://127.0.0.1:11434/api/generate -d '{"model":"qwen2.5:7b","prompt":"Give me one user: name and age.","format":{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"]},"stream":false}' +``` + +The answer's text is a JSON document (not a Markdown fence). Parse it and fail +loudly when it does not parse - a schema is a strong request, not a guarantee. + +## Tool calling + +`/api/chat` accepts `tools` (a JSON-Schema function list) in the same shape as +the OpenAI tools array. The model answers with +`message.tool_calls: [{ function: { name, arguments } }]` and no content; you run +the function, append a `{role: "tool", content: "...", tool_name: "..."}` +message, and call again. + +```bash +curl -s http://127.0.0.1:11434/api/chat -d '{"model":"qwen2.5:7b","messages":[{"role":"user","content":"What is the weather in Lisbon?"}],"tools":[{"type":"function","function":{"name":"weather","description":"Current weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}],"stream":false}' +``` + +Not every model supports tools: check `capabilities` from `/api/show` first, or +you will get a plain-text answer that looks like a refusal. + +## The OpenAI-compatible endpoints + +Ollama also serves `/v1`: the same daemon, the OpenAI wire format. Use it when a +client library or SDK already speaks OpenAI and you cannot change it. + +```bash +curl -s http://127.0.0.1:11434/v1/chat/completions -d '{"model":"qwen2.5:7b","messages":[{"role":"user","content":"hi"}]}' +curl -s http://127.0.0.1:11434/v1/completions -d '{"model":"qwen2.5:7b","prompt":"hi"}' +curl -s http://127.0.0.1:11434/v1/embeddings -d '{"model":"nomic-embed-text","input":"hi"}' +curl -s http://127.0.0.1:11434/v1/models +``` + +Two cost-saving facts about this surface: + +- the **timings are not in the OpenAI body**; the `usage` object carries token + counts only. If you need the durations, use the native `/api` route; +- `stream: true` here is Server-Sent Events (`data: {...}` lines ending + `data: [DONE]`), not NDJSON. A reader written for `/api` will not parse it. + +## The CLI, for the same operations + +| task | API | CLI | +|---|---|---| +| list models | `GET /api/tags` | `ollama list` | +| loaded models | `GET /api/ps` | `ollama ps` | +| inspect | `POST /api/show` | `ollama show qwen2.5:7b` | +| pull | `POST /api/pull` | `ollama pull qwen2.5:7b` | +| delete | `DELETE /api/delete` | `ollama rm qwen2.5:7b` | +| one prompt | `POST /api/generate` | `ollama run qwen2.5:7b "hi"` | +| daemon | - | `ollama serve` | + +`ollama run` starts an interactive session when given no prompt; it is not a +drop-in for a scripted call, and it prints the finished text without the timing +fields. + +## Environment variables that change the daemon + +These are read by the SERVER at start, not per request: + +| variable | default | what it does | +|---|---|---| +| `OLLAMA_HOST` | `127.0.0.1:11434` | the bind address | +| `OLLAMA_MODELS` | `~/.ollama/models` | where the weights live | +| `OLLAMA_KEEP_ALIVE` | `5m` | default residency for a loaded model | +| `OLLAMA_NUM_PARALLEL` | 1 (auto) | concurrent requests one model serves; each gets its own KV cache slot | +| `OLLAMA_MAX_LOADED_MODELS` | 3 x GPUs (auto) | how many models may be resident at once | +| `OLLAMA_MAX_QUEUE` | 512 | queued requests before the daemon answers 503 | +| `OLLAMA_FLASH_ATTENTION` | off | a speed/memory trade at long context | +| `OLLAMA_KV_CACHE_TYPE` | `f16` | `q8_0` / `q4_0` shrink the KV cache | +| `OLLAMA_ORIGINS` | localhost only | CORS origins the daemon accepts from a browser | +| `OLLAMA_DEBUG` | off | verbose server logging | + +Two settings interact in a way that ruins benchmarks: with +`OLLAMA_NUM_PARALLEL=2`, one model serves two requests at once, so two +"sequential" runs can overlap and both report a lower tok/s. Measure with +parallelism at its default, or state it in the result. + +## What not to do + +- Do not assume the daemon is running. Probe `/api/version` and say the address + you tried. +- Do not buffer a streaming response and parse it as one JSON document. +- Do not invent an `options` key: an unknown one is ignored, and the numbers + become a claim about a setting the daemon never applied. +- Do not quote `total_duration` as latency: it is the daemon's own span, and it + excludes the transport. +- Do not treat a 404 for a model as "the daemon is broken": the fix is + `ollama pull`, and the error body says so. diff --git a/packages/dsh-ollama/skills/ollama-inference-metrics/SKILL.md b/packages/dsh-ollama/skills/ollama-inference-metrics/SKILL.md new file mode 100644 index 0000000..d7d4327 --- /dev/null +++ b/packages/dsh-ollama/skills/ollama-inference-metrics/SKILL.md @@ -0,0 +1,213 @@ +--- +name: ollama-inference-metrics +description: What every Ollama timing field means and the arithmetic that turns it into a claim - tokens per second, time-to-first-token, why total_duration is not latency, why a second run is faster, what changes with context length, quantization, GPU offload and parallel requests - plus the reporting rules that make a benchmark a measurement instead of an anecdote. +whenToUse: Use when reporting how fast a model is, when comparing two models or two settings, when a benchmark's numbers look surprising or unstable, when writing a bench script, or before quoting any tok/s figure to a person. +--- + +# Inference metrics + +A tok/s number without its conditions is not a measurement; it is a sentence +about one moment. This skill is the arithmetic and the reporting contract. + +## The fields, and what each one covers + +Every field below comes from the daemon's final response object, in +NANOSECONDS. Nothing has to be inferred: + +| field | unit | covers | +|---|---|---| +| `load_duration` | ns | loading the weights into memory. **0 when they were already resident.** | +| `prompt_eval_count` | tokens | the prompt, tokenized | +| `prompt_eval_duration` | ns | evaluating the prompt | +| `eval_count` | tokens | tokens generated | +| `eval_duration` | ns | generating them | +| `total_duration` | ns | the daemon's whole span for the request | + +The two rates a reader cares about are divisions of those pairs: + + tok/s = eval_count / (eval_duration / 1e9) + prompt tok/s = prompt_eval_count / (prompt_eval_duration / 1e9) + +and time-to-first-token is the wait before the first generated token: + + TTFT = load_duration + prompt_eval_duration + +with one caveat that matters: that formula is the daemon's account. A client +that streams can measure the real first token, and the difference between the +two is the transport plus the queue - which is why `ollama_run` reports both. + +```javascript +/** The arithmetic, in one place, with the nulls an absent field deserves. */ +export function measure(final) { + const ns = (value) => (typeof value === 'number' && Number.isFinite(value) ? value : null) + const rate = (count, duration) => (ns(count) === null || ns(duration) === null || duration <= 0 ? null : count / (duration / 1e9)) + const loadMs = ns(final.load_duration) === null ? null : final.load_duration / 1e6 + const promptMs = ns(final.prompt_eval_duration) === null ? null : final.prompt_eval_duration / 1e6 + return { + evalCount: ns(final.eval_count), + evalTokensPerSecond: rate(final.eval_count, final.eval_duration), + promptTokensPerSecond: rate(final.prompt_eval_count, final.prompt_eval_duration), + evalMs: ns(final.eval_duration) === null ? null : final.eval_duration / 1e6, + loadMs, + promptEvalMs: promptMs, + ttftMs: loadMs === null && promptMs === null ? null : (loadMs ?? 0) + (promptMs ?? 0), + totalMs: ns(final.total_duration) === null ? null : final.total_duration / 1e6, + cut: final.done_reason === 'length', + } +} +``` + +A zero duration yields `null`, not `Infinity`: the daemon did not measure a +phase it reported as zero, and an infinite tok/s is not a fast model. + +## A worked example + +Given a final object: + +```json +{"done":true,"done_reason":"stop","eval_count":100,"eval_duration":2000000000,"prompt_eval_count":20,"prompt_eval_duration":200000000,"load_duration":1000000000,"total_duration":3500000000} +``` + +- tok/s = 100 / 2.0 s = **50.0** +- prompt tok/s = 20 / 0.2 s = **100.0** +- load = **1.00 s**, prompt eval = **0.20 s**, eval = **2.00 s** +- TTFT = 1.00 + 0.20 = **1.20 s** +- `total_duration` = **3.50 s**, which is 1.00 + 0.20 + 2.00 plus 0.30 s of + load/setup the four fields do not name. + +Those numbers are exactly what the fake daemon in `packages/dsh-ollama/checks` +serves, so the plugin's own arithmetic is asserted against them. + +## `total_duration` is not latency + +Three different spans get called "the time": + +1. **server span** = `total_duration` - what the daemon spent, including setup; +2. **model span** = `load_duration + prompt_eval_duration + eval_duration` - + the sum of the named phases; +3. **wall clock** - what the caller waited, including connection, queueing, + serialization and the transport. + +`total_duration` is (1), and it is NEITHER of the other two. A benchmark that +quotients `eval_count` by `total_duration` reports a rate a third lower than the +real one, because it charges the load and the prompt to the generation. Report +(1) as "daemon total", (3) as "wall", and compute tok/s only from the eval pair. + +## Cold and warm: what actually changes + +The first call that names a model pays `load_duration`; later calls with the +weights resident pay ~0. What that changes: + +| | cold (first call) | warm (resident) | +|---|---|---| +| `load_duration` | seconds | ~0 | +| TTFT | higher by the load | lower | +| wall clock | higher by the load | lower | +| **tok/s** | **the same** | **the same** | +| `eval_duration` | generation only | generation only | + +This is the trap in "the second run was 2.4x faster": the run is faster to +START, not faster per token. `eval_duration` never includes the load, so a cold +run's tok/s is not penalised. If a report says tok/s improved 2.4x on a second +run, it is measuring something else (a different `num_ctx`, a different prompt, +a cache that was cold, or parallelism). + +To measure a cold run honestly, unload first: + + POST /api/generate {"model": "", "keep_alive": 0} + +then run. Without that unload, "run 1" is warm and the load number is a lie. + +## What changes the numbers, and by roughly how much + +- **Context length (`num_ctx`)** - the KV cache grows linearly with it. A larger + context costs memory and, at long prompts, attention time. It is the single + biggest knob after the model itself, and it is why a number without a stated + context is not comparable to another number. +- **Prompt length** - prompt eval is quadratic-ish in tokens for attention, so a + 16k-token prompt has a much higher `prompt_eval_duration` than a 200-token + one, and TTFT scales with it. +- **Quantization** - Q4_K_M is smaller and faster than Q8_0 or F16, at some + quality cost. Comparing a Q4 to an F16 is a comparison of two artifacts, so + the digest (which encodes the layers) must accompany the number. +- **GPU offload (`num_gpu`, `size_vram / size`)** - a partial offload moves the + slowest layers to the CPU and the tok/s drops non-linearly. Report the + offload share from `/api/ps`, not just "GPU". +- **Parallel requests (`OLLAMA_NUM_PARALLEL`)** - one model serving N requests + splits its capacity; each reports a lower tok/s while total throughput may + rise. A benchmark run with parallelism enabled is not a single-stream number. +- **Model size and architecture** - the obvious one; the reason A/B needs the + same prompt, context and options. +- **Thermal state and background load** - real, and why a spread is reported + rather than a single figure. + +## The reporting rules + +A number is honest when a reader could reproduce it. That means all of: + +1. **At least 5 runs** for a number you will quote. 3 is the floor for a quick + look; 5 is a sample; 1 is an anecdote. State the count. +2. **A median, not a mean**, and the spread beside it (relative standard + deviation, or min-max). A mean is dominated by one slow run. +3. **p95 with the sample size** - with 5 runs p95 IS the max, and saying so is + more honest than implying a tail you did not measure. +4. **One variable at a time.** Two models at the same prompt/context/options, + or one model at two contexts - never both at once. +5. **Temperature 0 and a fixed seed** when the comparison is about speed; a + sampled answer changes `eval_count` and pollutes the rate. +6. **The model digest**, not just the tag: `qwen2.5:7b` can be re-pulled as a + different set of layers. +7. **The context length** (`num_ctx`) and whether the model was resident + (`load_duration` ~ 0 or not). +8. **The hardware and the offload share** - the same model on a CPU and on a + GPU are different numbers, and both are true. +9. **The prompt**, or at least its token count: `prompt_eval_count` is in the + answer, so quote it. +10. **What was measured**: single-stream, no concurrent requests, unless stated. + +A headline that obeys the rules looks like this: + +> `qwen2.5:7b` (digest a3de86cd1c13), 5 runs, 8k context, Q4_K_M, temperature 0, +> seed 42, 100% GPU: **41.2 tok/s median** (spread 6%), TTFT 0.41 s median, +> cold load 3.1 s. Single stream, no parallel requests. + +A headline that breaks them looks like this: + +> Ollama is fast - 40 tok/s. + +It is missing the model, the digest, the context, the run count, the spread, +the hardware and the definition of "fast"; and 40 could be any of the numbers +above. + +## Reading a `dsh-ollama` bench + +`ollama_bench` prints one section per model (and per context, in a sweep): + + RUN TOK/S TTFT LOAD EVAL TOKENS WALL + 1 50.0 1.20 s 1.00 s 2.00 s 100 3.61 s + 2 40.0 0.20 s 0.00 s 2.50 s 100 2.71 s + ... + tok/s median 50.0 · p95 100.0 · min 33.3 · max 100.0 · spread 41% + TTFT median 0.20 s · p95 1.20 s + wall median 2.71 s · p95 3.61 s + first run 3.61 s wall · 1.20 s TTFT; warm runs 2-5 median 2.71 s wall · 1.33x the warm median + note: tok/s is eval_count / eval_duration - the generation phase only - so a + cold load raises TTFT and wall, never tok/s. + +Read it in this order: the digest and context (is this the artifact I meant?), +the run count (is this a sample?), the median and spread (is it stable?), then +the per-run table (was one run an outlier, and does `load_duration` explain +it?). The cold/warm line is in wall and TTFT on purpose. + +## Claims a single run cannot support + +| claim | why one run cannot carry it | +|---|---| +| "this model is faster than that one" | no second sample, no spread, possibly a different prompt | +| "it is 2x faster on the second run" | that is the load, not the rate | +| "it does 40 tok/s" | at what context, on what hardware, at which quantization? | +| "it fits in 8 GB" | memory is a property of the loaded artifact + context, read `/api/ps` | +| "this is the model's speed" | it is one prompt's speed, on one machine, at one moment | + +The correction is always the same: more runs, one variable, and the conditions +beside the number. diff --git a/packages/dsh-ollama/skills/ollama-python/SKILL.md b/packages/dsh-ollama/skills/ollama-python/SKILL.md new file mode 100644 index 0000000..ee30220 --- /dev/null +++ b/packages/dsh-ollama/skills/ollama-python/SKILL.md @@ -0,0 +1,612 @@ +--- +name: ollama-python +description: The `ollama` Python package used properly - chat and generate with streaming, embeddings, tool calling, structured outputs with a JSON Schema, the async client, keep_alive, timeouts and cancellation - and a project shape whose benchmark arithmetic is unit-tested against a fake transport while the live half skips loudly when no daemon answers. +whenToUse: Use when writing a Python program, script or service that talks to Ollama, when a Python benchmark has to be trustworthy, when tool calling or structured output is wanted from Python, or when deciding how to test code that needs a model. +--- + +# Ollama from Python + +`pip install ollama` is the official client: a thin wrapper over the REST API in +this pack's `ollama-api` skill, with pydantic response models. It ships a sync +`Client`, an `AsyncClient`, and module-level helpers that use a default client. + +Every example below is self-contained: it reads `OLLAMA_HOST` itself, so it can +be copied out of this page on its own. + +## Connect without hardcoding a host + +The daemon is on `127.0.0.1:11434` by default, but a person may have moved it. +Read the environment, do not bake the address into the code: + +```python +import os + +from ollama import Client + +HOST = os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434") +client = Client(host=HOST) +print(client.list().models) +``` + +`OLLAMA_HOST` may arrive as `host:port` without a scheme, because the daemon +accepts that spelling for its BIND address. Normalise it before a client uses +it, or an error message will read `host:port` and look like a hostname: + +```python +import os + + +def daemon_url() -> str: + raw = os.environ.get("OLLAMA_HOST", "127.0.0.1:11434").strip() + if "://" not in raw: + raw = "http://" + raw + return raw.rstrip("/") +``` + +## One prompt, one conversation + +```python +import os + +from ollama import Client + +client = Client(host=os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434")) + +# generate: one string in, text and timings out +answer = client.generate(model="qwen2.5:7b", prompt="Why is the sky blue?", options={"num_predict": 128}) +print(answer.response) +print(answer.eval_count, answer.eval_duration) # tokens, NANOSECONDS + +# chat: messages in, the same timings out +chat = client.chat( + model="qwen2.5:7b", + messages=[ + {"role": "system", "content": "Answer in one sentence."}, + {"role": "user", "content": "Why is the sky blue?"}, + ], + options={"temperature": 0, "seed": 42}, +) +print(chat.message.content) +``` + +Every response model carries the same metric fields as the API: +`eval_count`, `eval_duration`, `prompt_eval_count`, `prompt_eval_duration`, +`load_duration`, `total_duration` and `done_reason`. They are the daemon's +numbers, in nanoseconds; the arithmetic in the `ollama-inference-metrics` skill +applies unchanged. + +## Streaming + +With `stream=True` the call returns a generator of partial responses. Each part +has `.message.content` (chat) or `.response` (generate); only the FINAL part +carries the counts and durations: + +```python +import os + +from ollama import Client + +client = Client(host=os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434")) + +stream = client.chat( + model="qwen2.5:7b", + messages=[{"role": "user", "content": "Write one paragraph about the sea."}], + stream=True, +) +text = "" +final = None +try: + for part in stream: + text += part.message.content + if part.done: + final = part +finally: + stream.close() +print(text) +if final is None: + raise RuntimeError("the stream ended without a done part") +print(final.eval_count / (final.eval_duration / 1e9), "tok/s") +``` + +Two habits worth keeping: + +- **close the generator.** Breaking out of a `for` over `stream` leaves the HTTP + response open until garbage collection; `stream.close()` in a `finally` is + what tells the daemon to stop generating into a socket nobody reads; +- **a stream is not a retryable unit.** If the connection drops mid-stream you + have partial text and no timings; treat that as a failed measurement, not as + an answer. + +## Embeddings + +```python +import os + +from ollama import Client + +client = Client(host=os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434")) +response = client.embed(model="nomic-embed-text", input=["the cat sat", "a cat was sitting"]) +first, second = response.embeddings +print(len(first), "dimensions") + +dot = sum(a * b for a, b in zip(first, second)) +norm = (sum(a * a for a in first) ** 0.5) * (sum(b * b for b in second) ** 0.5) +print("cosine", dot / norm) +``` + +`client.embed` uses `/api/embed` (many inputs, one request). The older +`client.embeddings(model=..., prompt=...)` is the single-prompt form. Check the +model's capability first - a completion model refuses, and the client raises: + +```python +import os + +from ollama import Client, ResponseError + +client = Client(host=os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434")) +try: + client.embed(model="qwen2.5:7b", input="hello") +except ResponseError as error: + print(error.status_code, error.error) + # -> the model is not an embedding model; pull nomic-embed-text +``` + +## Tool calling + +Pass `tools` as JSON-Schema function definitions. The model answers with +`message.tool_calls` and no content; run the function and append a `tool` +message. This is a loop, and it must be bounded: + +```python +import os + +from ollama import Client + +client = Client(host=os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434")) + + +def get_weather(city: str) -> str: + return f"18 C and clear in {city}" + + +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } +] + +messages = [{"role": "user", "content": "What is the weather in Lisbon?"}] +for _ in range(4): # bound the loop: a model can call a tool forever + response = client.chat(model="qwen2.5:7b", messages=messages, tools=TOOLS) + messages.append(response.message) + if not response.message.tool_calls: + print(response.message.content) + break + for call in response.message.tool_calls: + result = get_weather(**call.function.arguments) + messages.append({"role": "tool", "content": result, "tool_name": call.function.name}) +else: + raise RuntimeError("the model kept calling tools; stop and inspect the conversation") +``` + +## Structured output + +`format="json"` asks for JSON; a JSON Schema asks for a shape. With pydantic the +schema comes from the model itself: + +```python +import os + +from ollama import Client +from pydantic import BaseModel + + +class Person(BaseModel): + name: str + age: int + + +client = Client(host=os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434")) +response = client.chat( + model="qwen2.5:7b", + messages=[{"role": "user", "content": "Give me one fictional person."}], + format=Person.model_json_schema(), + options={"temperature": 0}, +) +person = Person.model_validate_json(response.message.content) +print(person.name, person.age) +``` + +The schema is a strong request, not a guarantee: `model_validate_json` is what +turns "the model probably obeyed" into an exception you can handle. + +## Async, timeouts and cancellation + +```python +import asyncio +import os + +from ollama import AsyncClient + +HOST = os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434") + + +async def ask(question: str) -> str: + async with AsyncClient(host=HOST, timeout=120.0) as client: + response = await client.chat( + model="qwen2.5:7b", + messages=[{"role": "user", "content": question}], + options={"num_predict": 256}, + ) + return response.message.content + + +async def main() -> None: + try: + print(await asyncio.wait_for(ask("hi"), timeout=60)) + except asyncio.TimeoutError: + print("cancelled; the daemon drops the request when the socket closes") + + +asyncio.run(main()) +``` + +Three rules: + +- **a timeout is a cancellation, not a kill.** The client closes the response; + the daemon notices the closed socket and stops. That is not instant, and a + long `num_predict` can keep a GPU busy for a moment after you left; +- **`timeout=` on the client** bounds connect AND read; pass a value larger than + the slowest cold load you accept, or set it per call; +- **do not share one `AsyncClient` across an event-loop boundary.** Create it in + the loop that uses it (`async with` is the simplest way to get that right). + +## `keep_alive` and residency + +`keep_alive` is a keyword argument on every generation call: a duration string, +`0` to unload when the call ends, or `-1` to keep the weights forever. + +```python +import os + +from ollama import Client + +client = Client(host=os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434")) +client.generate(model="qwen2.5:7b", prompt="warm up", keep_alive="10m", options={"num_predict": 1}) +# ... many calls reuse the resident weights ... +client.generate(model="qwen2.5:7b", keep_alive=0, options={"num_predict": 0}) +``` + +The last call is the unload (`num_predict: 0` keeps it from generating, so the +warm-up and the unload each cost one prompt pass rather than an answer). Without +it, a "cold" measurement is not cold. + +## A project shape that can be tested + +A benchmark needs arithmetic you can test without a model, and a live half that +skips when no daemon answers. That is four small files: + +``` +pyproject.toml +src/ollama_lab/__init__.py +src/ollama_lab/client.py +src/ollama_lab/bench.py +tests/test_bench.py +tests/test_live.py +``` + +```toml +[project] +name = "ollama-lab" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["ollama>=0.4"] + +[project.optional-dependencies] +dev = ["pytest>=8"] + +[tool.pytest.ini_options] +markers = ["live: needs a running Ollama daemon and a pulled model"] +``` + +`client.py` owns the transport seam, so the benchmark is testable: + +```python +"""A small layer over the daemon: metrics, and a transport you can fake.""" +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any, Protocol + + +def daemon_url() -> str: + raw = os.environ.get("OLLAMA_HOST", "127.0.0.1:11434").strip() + if "://" not in raw: + raw = "http://" + raw + return raw.rstrip("/") + + +class Transport(Protocol): + def post(self, path: str, payload: dict[str, Any], *, timeout: float) -> dict[str, Any]: ... + + +class HttpTransport: + def __init__(self, host: str | None = None, timeout: float = 120.0) -> None: + self.host = (host or daemon_url()).rstrip("/") + self.timeout = timeout + + def post(self, path: str, payload: dict[str, Any], *, timeout: float | None = None) -> dict[str, Any]: + request = urllib.request.Request( + self.host + path, + data=json.dumps(payload).encode("utf-8"), + headers={"content-type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout or self.timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + detail = error.read().decode("utf-8", "replace") + raise RuntimeError(f"HTTP {error.code} from {path}: {detail}") from error + + +@dataclass(frozen=True) +class Metrics: + eval_count: int | None + eval_duration_ns: int | None + prompt_eval_count: int | None + prompt_eval_duration_ns: int | None + load_duration_ns: int | None + total_duration_ns: int | None + done_reason: str + + @classmethod + def from_response(cls, body: dict[str, Any]) -> "Metrics": + return cls( + eval_count=body.get("eval_count"), + eval_duration_ns=body.get("eval_duration"), + prompt_eval_count=body.get("prompt_eval_count"), + prompt_eval_duration_ns=body.get("prompt_eval_duration"), + load_duration_ns=body.get("load_duration"), + total_duration_ns=body.get("total_duration"), + done_reason=str(body.get("done_reason", "")), + ) + + @property + def tokens_per_second(self) -> float | None: + if not self.eval_count or not self.eval_duration_ns: + return None + return self.eval_count / (self.eval_duration_ns / 1e9) + + @property + def seconds_to_first_token(self) -> float | None: + if self.load_duration_ns is None and self.prompt_eval_duration_ns is None: + return None + return ((self.load_duration_ns or 0) + (self.prompt_eval_duration_ns or 0)) / 1e9 + + +class OllamaClient: + """generate() over a Transport; stream=False keeps the measurement whole.""" + + def __init__(self, transport: Transport | None = None) -> None: + self.transport = transport or HttpTransport() + + def generate(self, model: str, prompt: str, **options: Any) -> tuple[str, Metrics]: + body = self.transport.post( + "/api/generate", + {"model": model, "prompt": prompt, "stream": False, "options": options}, + timeout=300.0, + ) + return str(body.get("response", "")), Metrics.from_response(body) +``` + +`bench.py` holds the statistics: + +```python +"""Median, p95 and spread: the only summary a small sample can carry.""" +from __future__ import annotations + +import math +import statistics + +from .client import Metrics, OllamaClient + + +def nearest_rank(values: list[float], pct: float) -> float | None: + """The smallest value at or above the rank - p95 of 5 is one of the 5.""" + ordered = sorted(v for v in values if math.isfinite(v)) + if not ordered: + return None + rank = math.ceil((pct / 100) * len(ordered)) + return ordered[max(0, min(len(ordered) - 1, rank - 1))] + + +def summarize(values: list[float]) -> dict[str, float | int | None]: + sample = [v for v in values if math.isfinite(v)] + if not sample: + return {"count": 0, "median": None, "p95": None, "spread": None} + mean = statistics.fmean(sample) + stdev = statistics.pstdev(sample) if len(sample) > 1 else 0.0 + return { + "count": len(sample), + "median": statistics.median(sample), + "p95": nearest_rank(sample, 95), + "spread": (stdev / mean) if mean else None, + } + + +def bench(model: str, prompt: str, runs: int = 5, client: OllamaClient | None = None) -> dict: + """runs measured calls after an unload, so run 1 is genuinely cold.""" + if runs < 3: + raise ValueError("a benchmark needs at least 3 runs; one run is not a measurement") + client = client or OllamaClient() + client.transport.post("/api/generate", {"model": model, "keep_alive": 0}, timeout=30.0) + measured: list[Metrics] = [] + for _ in range(runs): + _text, metrics = client.generate(model, prompt, num_predict=128, temperature=0, seed=42) + if metrics.done_reason == "length": + raise RuntimeError("the answer was cut by num_predict; raise it for a benchmark") + if metrics.tokens_per_second is None: + raise RuntimeError("the daemon reported no eval_duration; the rate is unknown") + measured.append(metrics) + return { + "model": model, + "runs": runs, + "tokens_per_second": summarize([m.tokens_per_second or 0.0 for m in measured]), + "seconds_to_first_token": summarize([m.seconds_to_first_token or 0.0 for m in measured]), + "cold_load_ns": measured[0].load_duration_ns, + } +``` + +`tests/test_bench.py` proves the arithmetic without a model: a fake transport +returns canned bodies with fixed nanosecond durations, so 100 tokens in 2 s is +exactly 50 tok/s. + +```python +"""The arithmetic, against a fake transport - no daemon, no model, no GPU.""" +from __future__ import annotations + +import pytest + +from ollama_lab.bench import bench, nearest_rank, summarize +from ollama_lab.client import Metrics, OllamaClient + + +class FakeTransport: + def __init__(self, responses: list[dict]) -> None: + self.responses = list(responses) + self.calls: list[tuple[str, dict]] = [] + + def post(self, path: str, payload: dict, *, timeout: float) -> dict: + self.calls.append((path, payload)) + return self.responses.pop(0) + + +def body(eval_duration: int, load_duration: int = 0) -> dict: + return { + "response": "ok", + "done": True, + "done_reason": "stop", + "eval_count": 100, + "eval_duration": eval_duration, + "prompt_eval_count": 20, + "prompt_eval_duration": 200_000_000, + "load_duration": load_duration, + "total_duration": eval_duration + 200_000_000 + load_duration, + } + + +def test_rate_is_the_eval_pair() -> None: + metrics = Metrics.from_response(body(2_000_000_000)) + assert metrics.tokens_per_second == 50 + assert metrics.seconds_to_first_token == pytest.approx(0.2) + + +def test_a_zero_duration_is_unknown_not_infinite() -> None: + metrics = Metrics.from_response({**body(0), "eval_count": 0}) + assert metrics.tokens_per_second is None + + +def test_p95_is_a_real_run() -> None: + assert nearest_rank([50, 40, 66.7, 33.3, 100], 95) == 100 + assert nearest_rank([], 95) is None + assert summarize([7])["spread"] == 0 + + +def test_bench_unloads_then_measures() -> None: + transport = FakeTransport([{"done": True}] + [body(2_000_000_000 + index) for index in range(3)]) + result = bench("test:1b", "bench", runs=3, client=OllamaClient(transport)) + assert transport.calls[0][1]["keep_alive"] == 0 + assert result["runs"] == 3 + assert result["tokens_per_second"]["median"] == pytest.approx(50) + + +def test_a_short_sample_is_refused() -> None: + with pytest.raises(ValueError): + bench("test:1b", "bench", runs=2) +``` + +`tests/test_live.py` is the half that needs a machine: it skips unless a daemon +answers AND a completion model is pulled, and says which condition failed. + +```python +"""The live half: skipped loudly when there is no daemon or no model.""" +from __future__ import annotations + +import json +import urllib.request + +import pytest + +from ollama_lab.client import daemon_url + + +def daemon_version() -> str | None: + try: + with urllib.request.urlopen(daemon_url() + "/api/version", timeout=1.5) as response: + return json.loads(response.read().decode("utf-8")).get("version") + except Exception: + return None + + +def smallest_model() -> str | None: + try: + with urllib.request.urlopen(daemon_url() + "/api/tags", timeout=2.0) as response: + models = json.loads(response.read().decode("utf-8")).get("models", []) + except Exception: + return None + candidates = [m for m in models if "embed" not in str(m.get("name", ""))] + if not candidates: + return None + return sorted(candidates, key=lambda m: m.get("size", 0))[0]["name"] + + +VERSION = daemon_version() +MODEL = smallest_model() if VERSION is not None else None + +pytestmark = pytest.mark.skipif( + VERSION is None or MODEL is None, + reason=f"no live Ollama (version={VERSION}, model={MODEL}); start one with `ollama serve` and pull a model", +) + + +def test_one_tiny_generation() -> None: + from ollama_lab.client import OllamaClient + + text, metrics = OllamaClient().generate(MODEL, "Reply with the single word: ok", num_predict=8, temperature=0) + assert text.strip() != "" + assert metrics.eval_count is not None and metrics.eval_count >= 0 +``` + +Run `pytest -m "not live"` in CI and `pytest -m live` on a machine with the +daemon. The rule the fake transport encodes is the important one: **the half +that needs a model is never the half that proves your arithmetic.** + +## Pitfalls, in the order they bite + +1. **Quoting `eval_duration` as milliseconds.** It is nanoseconds; 2e9 is 2 s. +2. **Treating `client.chat(stream=True)` as a string.** It is a generator of + parts, and only the last part has the timings. +3. **Assuming `OLLAMA_HOST` is a connect address.** The daemon may bind + `0.0.0.0`; a client should use `127.0.0.1`. +4. **Benchmarking without an unload.** Run 1 is then warm, and `load_duration` + is 0 for every run - the cold number you meant to report does not exist. +5. **Comparing two models with `temperature` at its default.** Sampling changes + `eval_count`; fix `temperature=0` and `seed`. +6. **Trusting a `format=` schema blindly.** Validate the JSON; a model can still + return a shape that parses but does not match. +7. **Letting a tool loop run unbounded.** Cap the iterations and inspect the + conversation when the cap is hit. diff --git a/scripts/checks/README.md b/scripts/checks/README.md index 7a52300..2ce3e11 100644 --- a/scripts/checks/README.md +++ b/scripts/checks/README.md @@ -41,12 +41,17 @@ Windows PowerShell: `$env:DSH_CHECK_LAUNCH='1'; node scripts/checks/check-node-r ## The checks that live inside a package `dsh-docker`, `dsh-wsl`, `dsh-ocr`, `dsh-diagrams`, `dsh-frontend-craft`, -`dsh-research`, `dsh-supercollider` and `dsh-proto-lab` each ship their own under +`dsh-research`, `dsh-supercollider`, `dsh-proto-lab` and `dsh-ollama` each ship +their own under `packages//checks/`, run with `npm run check` in that folder. They are additional to the list above, never a replacement for it: a host half's fixtures and its live skips belong beside the package (see `packages/dsh-docker/checks/check-docker-node.mjs`, which drives the runner -against a real child process and every engine state against a scripted one), and +against a real child process and every engine state against a scripted one, and +`packages/dsh-ollama/checks/check-ollama-node.mjs`, which serves a **fake +daemon** on `127.0.0.1:0` with fixed nanosecond durations so tok/s and TTFT are +assertions rather than observations, parses every skill example, and runs one +generation against a real daemon only when one answers), and the pack-wide checks still own the routes, the bundle and the manifest. `dsh-proto-lab`'s three carry the pack's **verification ladder** — the claim that a diff --git a/scripts/checks/check-node-routes.mjs b/scripts/checks/check-node-routes.mjs index ae7bb7c..86421aa 100644 --- a/scripts/checks/check-node-routes.mjs +++ b/scripts/checks/check-node-routes.mjs @@ -1710,7 +1710,7 @@ try { // whose registration nothing pinned. It is here now. { const registrations = [] - for (const packageName of ['dsh-media', 'dsh-diagrams', 'dsh-pdf', 'dsh-supercollider', 'dsh-proto-lab', 'dsh-research', 'dsh-frontend-craft']) { + for (const packageName of ['dsh-media', 'dsh-diagrams', 'dsh-pdf', 'dsh-supercollider', 'dsh-proto-lab', 'dsh-research', 'dsh-frontend-craft', 'dsh-ollama']) { registrations.push({ packageName, source: await fsp.readFile(path.join(repo, 'packages', packageName, 'lib', 'index.js'), 'utf8'), @@ -1720,7 +1720,7 @@ try { check( 'every bundled-skill registration names its file and source bucket', named.map((entry) => entry.packageName).join(','), - 'dsh-media,dsh-diagrams,dsh-pdf,dsh-supercollider,dsh-proto-lab,dsh-research,dsh-frontend-craft', + 'dsh-media,dsh-diagrams,dsh-pdf,dsh-supercollider,dsh-proto-lab,dsh-research,dsh-frontend-craft,dsh-ollama', ) } @@ -1983,6 +1983,62 @@ try { check('docker: ...and names what is deliberately absent', healthBody.notBuilt.includes('exec') && healthBody.tools.length, 9) } +// ------------------------------------------------------------- dsh-ollama +// The model row is host-only: five tools, NO route, and one refusal that has to +// happen BEFORE a socket is opened. The arithmetic and every other refusal are +// the package's own check (packages/dsh-ollama/checks/check-ollama-node.mjs, +// against a fake daemon with fixed nanosecond durations); what is pinned here is +// the pack-wide contract - the registration, the absence of a route, and the +// loopback fence counted at `fetch`. +{ + const ollamaModule = await import('../../packages/dsh-ollama/lib/index.js') + const seen = { tools: [], routes: 0 } + ollamaModule.apply({ + get: (name) => (name === 'connection' ? { fetch: { register: () => (seen.routes += 1, () => {}) } } : undefined), + tools: { register: (tool) => (seen.tools.push(tool), () => {}) }, + effect: (fn) => fn(), + logger: { info() {}, warn() {} }, + }) + check('ollama: the row registers five tools', seen.tools.length, 5) + check( + 'ollama: ...named for the surface the README promises', + seen.tools.map((tool) => tool.name).join(','), + 'ollama_status,ollama_run,ollama_bench,ollama_models,ollama_embed', + ) + check( + 'ollama: ...each with a closed schema and an execute', + seen.tools.every((tool) => tool.parameters && tool.parameters.type === 'object' && tool.parameters.additionalProperties === false && typeof tool.execute === 'function' && tool.output && tool.output.schema), + true, + ) + check('ollama: ...and NO route, because the row is host-only', seen.routes, 0) + check('ollama: the skill registration is the one this check drives', ollamaModule.SKILL_FILES.length, 3) + + // The fence, counted: an OLLAMA_HOST on another machine is refused and the + // fetch spy is never reached. `runner` is a stub so this check spawns nothing. + let fetches = 0 + const ollamaDeps = ollamaModule.__internals.createDeps( + { logger: { info() {}, warn() {} }, get: () => undefined, effect: (fn) => fn() }, + { + env: { OLLAMA_HOST: 'http://10.0.0.5:11434' }, + fetch: async () => { + fetches += 1 + return new Response('{}') + }, + runner: async () => ({ status: 0, stdout: '', stderr: '' }), + }, + ) + const fenced = await ollamaModule.__internals.buildTools(ollamaDeps) + .find((tool) => tool.name === 'ollama_status') + .execute({}, {}) + check('ollama: an off-machine daemon is refused', fenced.view.code, 'REMOTE_REFUSED') + check('ollama: ...before any request is made', fetches, 0) + check( + 'ollama: ...and the sentence names the opt-in variable', + fenced.text.includes('DSH_OLLAMA_ALLOW_REMOTE=1'), + true, + ) +} + // --------------------------------------------------------- the repo manifest // `.dsh-version.json` is documentation, but it is documentation a PERSON reads // to know what is installed and at which version, and nothing else keeps it