Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ connection the process makes, providers included.
`/metrics` is a Prometheus scrape target, off by default because an
open daemon has no authentication; with keys configured it requires one
like every other route. `LLMMAN_METRICS=1 llmman serve` turns it on;
the fifteen metric families and how to read them are in
the seventeen metric families and how to read them are in
[metrics.md](metrics.md).

## CORS
Expand Down
4 changes: 3 additions & 1 deletion docs/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ scrape_configs:

## Metric families

Fifteen metric families:
Seventeen metric families:

| Metric | Type | Labels | What it tells you |
|--------|------|--------|-------------------|
Expand All @@ -37,6 +37,8 @@ Fifteen metric families:
| `llmman_models_loaded` | gauge | — | Backends currently running, the set `/api/ps` reports. |
| `llmman_models_loading` | gauge | — | Loads under way; `loaded + loading` is what `LLMMAN_MAX_LOADED_MODELS` caps. |
| `llmman_model_up` | gauge | `model`, `engine` | 1 while the backend process is alive, 0 once it has died but llmman has not noticed. |
| `llmman_requests_in_flight` | gauge | `model` | Requests a loaded model is serving or holding, including responses still being sent. |
| `llmman_requests_queued` | gauge | `model` | Estimate of those waiting for a free `LLMMAN_NUM_PARALLEL` slot. Absent where llmman does not know the slot count (diffusion, `--parallel` in `LLMMAN_LLAMA_ARGS`, no `--ctx-size`). |
| `llmman_model_loads_total` | counter | `model` | Cold starts per model — the churn a too-small `LLMMAN_MAX_LOADED_MODELS` produces. |
| `llmman_model_load_duration_seconds` | histogram | `model` | How long a cold start takes, from admission to ready. |
| `llmman_model_load_oom_retries_total` | counter | `model`, `strategy` | Loads that hit an out-of-memory failure and retried: `evict_others`, `split_mode`, `ctx_shrink`. |
Expand Down
14 changes: 14 additions & 0 deletions src/cmd/serve.rs
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,9 @@ struct RunningModel {
/// `ActivityGuard`'s doc comment for why a generation slower than its
/// own `keep_alive` must not be killed mid-stream.
in_flight: u32,
/// llama-server's slot count when llmman knows it, so `in_flight`
/// beyond it is queued. `None` otherwise.
parallel_slots: Option<u32>,
/// The `"model"` field a request must carry to reach this model on
/// its backend when that differs from the canonical reference — see
/// `backend_wire_model`. `Some(<absolute model directory>)` for
Expand Down Expand Up @@ -2751,6 +2754,12 @@ async fn resolve_target(
inherits_env.then_some("LLAMA_ARG_THINK_BUDGET_MESSAGE"),
);

// Not diffusion (one generation at a time) or a user-set `--parallel`.
let parallel_slots = num_parallel.filter(|_| {
process.engine() == Engine::LlamaServer
&& !matches!(model_path, ModelPath::Diffusion(_))
&& !backend::llama_sets_parallel(&llama_extra_args, inherits_env)
});
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
model_ref.to_string(),
Expand All @@ -2767,6 +2776,7 @@ async fn resolve_target(
keep_alive: default_keep_alive(),
// 1, not 0 — see this function's own doc comment.
in_flight: 1,
parallel_slots,
decision,
effort_budget,
effort_budget_message,
Expand Down Expand Up @@ -4242,6 +4252,10 @@ async fn handle_metrics(State(state): State<AppState>) -> impl IntoResponse {
// only notices a dead backend when a request arrives for
// that model, so `models_loaded` counts it until then.
up: m.process.is_alive(),
requests_in_flight: u64::from(m.in_flight),
requests_queued: m
.parallel_slots
.map(|slots| u64::from(m.in_flight.saturating_sub(slots))),
Comment thread
ericcurtin marked this conversation as resolved.
Resolved
})
.collect();
(mgr.running.len() as u64, mgr.pending_loads as u64, models)
Expand Down
20 changes: 20 additions & 0 deletions src/cmd/serve/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,15 @@ pub(super) fn check_extra_args_env() -> anyhow::Result<()> {
Ok(())
}

/// Whether `extra` or, if `inherits_env`, the environment sets llama-server's
/// slot count, overriding llmman's `--parallel`.
pub(super) fn llama_sets_parallel(extra: &[String], inherits_env: bool) -> bool {
let env = inherits_env.then_some("LLAMA_ARG_N_PARALLEL");
["-np", "--parallel"]
.iter()
.any(|flag| llama_sets(extra, flag, env))
}

/// Appends `extra` (a split `LLMMAN_*_ARGS`), then `--port port --host
/// host`. vllm, sglang and ds4-server keep each flag's last value, so
/// llmman's win however `extra` spells them (argparse takes abbreviations).
Expand Down Expand Up @@ -1622,6 +1631,17 @@ mod tests {
}
}

#[test]
fn llama_sets_parallel_sees_either_spelling() {
let sets = |args: &[&str]| {
let args: Vec<String> = args.iter().map(|a| a.to_string()).collect();
llama_sets_parallel(&args, false)
};
assert!(sets(&["-np", "1"]));
assert!(sets(&["--parallel=1"]));
assert!(!sets(&["--jinja", "--ctx-size", "8192"]));
}

#[test]
fn only_generation_ggufs_go_to_ds4() {
let gguf = ModelPath::Gguf(PathBuf::from("/m/ds4.gguf"), None);
Expand Down
2 changes: 2 additions & 0 deletions src/cmd/serve/test_support.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ pub(super) fn running_model_fixture(
context_window: None,
keep_alive,
in_flight,
parallel_slots: None,
decision: false,
effort_budget: false,
effort_budget_message: true,
Expand Down Expand Up @@ -233,6 +234,7 @@ pub(super) fn running_model_fixture_with_engine(
context_window: None,
keep_alive: None,
in_flight: 0,
parallel_slots: None,
decision: false,
effort_budget: false,
effort_budget_message: true,
Expand Down
27 changes: 27 additions & 0 deletions src/cmd/serve/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4680,6 +4680,33 @@ async fn the_metrics_route_label_is_the_template_not_the_request_path() {
assert!(!rendered.contains("abc123"), "{rendered}");
}

/// Three in flight on one slot means two are queued.
#[tokio::test]
async fn the_scrape_shows_requests_queued_behind_the_parallel_slots() {
let state = test_state();
let mut model = running_model_fixture(None, Duration::ZERO, 3);
model.parallel_slots = Some(1);
state
.0
.manager
.lock()
.await
.running
.insert("queued-model:latest".into(), model);

let resp = handle_metrics(State(state)).await.into_response();
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let scrape = String::from_utf8(body.to_vec()).unwrap();
for expected in [
"llmman_requests_in_flight{model=\"queued-model:latest\"} 3\n",
"llmman_requests_queued{model=\"queued-model:latest\"} 2\n",
] {
assert!(scrape.contains(expected), "{expected} in:\n{scrape}");
}
}

/// Any localhost page is an allowed origin, so one that can reach the
/// daemon could otherwise read a scrape out of a browser. What makes
/// this true is `/metrics` being merged *after* `.layer(cors_layer())`
Expand Down
89 changes: 77 additions & 12 deletions src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,9 @@ pub(crate) struct ModelState {
/// Whether the backend process is still alive right now
/// (`ModelProcess::is_alive`), not whether llmman still lists it.
pub(crate) up: bool,
pub(crate) requests_in_flight: u64,
/// `requests_in_flight` beyond the parallel slots; `None` if unknown.
pub(crate) requests_queued: Option<u64>,
}

/// The values [`Store`] cannot accumulate on its own: they are read from
Expand Down Expand Up @@ -479,6 +482,22 @@ fn gauge(out: &mut String, name: &str, help: &str, labels: &Labels, value: impl
out.push_str(&format!("{name}{} {value}\n", labels.braces()));
}

/// Emits a gauge family with a series per model in label order; `series`
/// returns `None` to skip one.
fn model_gauge(
out: &mut String,
name: &str,
help: &str,
models: &[ModelState],
series: impl Fn(&ModelState) -> Option<(Labels, u64)>,
) {
header(out, name, "gauge", help);
let samples: BTreeMap<Labels, u64> = models.iter().filter_map(series).collect();
for (labels, value) in &samples {
out.push_str(&format!("{name}{} {value}\n", labels.braces()));
}
}

/// Emits every accumulated series of one counter family, in key order.
///
/// A family with no samples yet still gets its `HELP` and `TYPE`, but no
Expand Down Expand Up @@ -597,22 +616,30 @@ fn render_from(store: &Store, snapshot: &Snapshot) -> String {
snapshot.models_loading,
);

header(
model_gauge(
&mut out,
"llmman_model_up",
"gauge",
"1 while a loaded model's backend process is alive, 0 once it has exited and llmman has not yet noticed.",
&snapshot.models,
|m| Some((Labels::new(&[("model", &m.model), ("engine", m.engine)]), u64::from(m.up))),
);
model_gauge(
&mut out,
"llmman_requests_in_flight",
"Requests a loaded model is serving or holding, including responses still being sent.",
&snapshot.models,
|m| Some((Labels::new(&[("model", &m.model)]), m.requests_in_flight)),
);
model_gauge(
&mut out,
"llmman_requests_queued",
"Estimated requests waiting for a free parallel slot; absent where llmman does not know the slot count.",
&snapshot.models,
|m| {
m.requests_queued
.map(|q| (Labels::new(&[("model", &m.model)]), q))
},
);
let mut model_up: BTreeMap<Labels, u8> = BTreeMap::new();
for model in &snapshot.models {
model_up.insert(
Labels::new(&[("model", &model.model), ("engine", model.engine)]),
u8::from(model.up),
);
}
for (labels, up) in &model_up {
out.push_str(&format!("llmman_model_up{} {up}\n", labels.braces()));
}

counter(
&mut out,
Expand Down Expand Up @@ -729,6 +756,8 @@ mod tests {
"llmman_models_loaded",
"llmman_models_loading",
"llmman_model_up",
"llmman_requests_in_flight",
"llmman_requests_queued",
"llmman_model_loads_total",
"llmman_model_load_duration_seconds",
"llmman_model_load_oom_retries_total",
Expand Down Expand Up @@ -878,6 +907,38 @@ mod tests {
}
}

/// No queue figure without a known slot count.
#[test]
fn a_waiting_request_reads_as_queued_only_when_the_slots_are_known() {
let mut snapshot = snapshot();
snapshot.models = vec![
ModelState {
model: "a:latest".into(),
engine: "llama-server",
up: true,
requests_in_flight: 2,
requests_queued: Some(1),
},
ModelState {
model: "b:latest".into(),
engine: "llama-server",
up: true,
requests_in_flight: 3,
requests_queued: None,
},
];

let rendered = rendered(&store(), &snapshot);
for expected in [
"llmman_requests_in_flight{model=\"a:latest\"} 2\n",
"llmman_requests_in_flight{model=\"b:latest\"} 3\n",
"llmman_requests_queued{model=\"a:latest\"} 1\n",
] {
assert!(rendered.contains(expected), "{expected} in:\n{rendered}");
}
assert!(!rendered.contains("llmman_requests_queued{model=\"b:latest\"}"));
}

/// A backend that exited without llmman noticing is the one case no
/// other metric here can show.
#[test]
Expand All @@ -889,11 +950,15 @@ mod tests {
model: "a:latest".into(),
engine: "llama-server",
up: true,
requests_in_flight: 0,
requests_queued: None,
},
ModelState {
model: "b:latest".into(),
engine: "vllm",
up: false,
requests_in_flight: 0,
requests_queued: None,
},
];

Expand Down
Loading