Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 21 additions & 23 deletions engine/crates/core/src/calc.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
// VRAM budget calculator from the product spec, with the one fit rule every load path uses.

const SAFETY_MARGIN_GB: f64 = 1.5;
const MAX_MEM_FRACTION: f64 = 0.95;
const CUDA_CONTEXT_GB: f64 = crate::engine_overhead::DEFAULT_CONTEXT_GB;
const MAX_MEM_FRACTION: f64 = 1.0;
pub const BASE_OVERHEAD_GB: f64 = 1.8;
pub const METAL_OVERHEAD_GB: f64 = 0.5;

Expand Down Expand Up @@ -30,10 +29,9 @@ impl Need {
}

pub fn missing_gb(need: &Need, free_gb: f64) -> f64 {
// GB a budget lacks: the whole budget leaves the safety margin, the static pool stays within 95% of free minus context.
(need.total() - (free_gb - SAFETY_MARGIN_GB))
.max(need.pool() - MAX_MEM_FRACTION * (free_gb - CUDA_CONTEXT_GB))
.max(need.weights + need.load_peak + need.overhead - (free_gb - SAFETY_MARGIN_GB))
// Weights+peak in free; total leaves SAFETY_MARGIN_GB. (ponytail: pool-cap == free is dead under the margin.)
(need.weights + need.load_peak - free_gb)
.max(need.total() - (free_gb - SAFETY_MARGIN_GB))
.max(0.0)
}

Expand All @@ -58,7 +56,7 @@ pub fn shortfall(need: &Need, gpu: u32, free_gb: f64) -> Option<String> {
let (w, k, s, o, f) = (tenth(need.weights), tenth(need.kv) + tenth(need.window), tenth(need.state), tenth(need.overhead), tenth(free_gb));
let n = w + k + s + o;
let head = format!("needs {}; GPU {gpu} has {f:.1} GB free", describe(need));
let by_load = tenth(need.weights + need.load_peak + need.overhead) - (f - SAFETY_MARGIN_GB);
let by_load = tenth(need.weights + need.load_peak) - f;
if need.load_peak > 0.0 && by_load >= n - (f - SAFETY_MARGIN_GB) && by_load > 0.0 {
return Some(format!(
"{head}; while loading, SGLang also holds {:.1} GB of reordered expert copies: missing {:.1} GB (split it across more GPUs with --gpu)",
Expand All @@ -67,16 +65,6 @@ pub fn shortfall(need: &Need, gpu: u32, free_gb: f64) -> Option<String> {
));
}
let by_margin = n - (f - SAFETY_MARGIN_GB);
let cap = tenth(MAX_MEM_FRACTION * (f - CUDA_CONTEXT_GB));
let by_cap = w + k + s - cap;
if by_cap > by_margin {
return Some(format!(
"{head}, and its static pool ({}, {:.1} GB) may use at most 95% of it after the CUDA context ({cap:.1} GB): missing {:.1} GB",
if s > 0.0 { "weights, KV and state" } else { "weights + KV" },
w + k + s,
by_cap.max(0.1)
));
}
Some(format!("{head}, minus a {SAFETY_MARGIN_GB} GB safety margin: missing {:.1} GB", by_margin.max(0.1)))
}

Expand Down Expand Up @@ -135,7 +123,7 @@ mod tests {
}

#[test]
fn mem_fraction_rejects_over_095_with_missing() {
fn mem_fraction_rejects_when_budget_misses() {
let err = mem_fraction_from_free(&need(45.0, 3.2, 1.8), 50.0, 0.0).unwrap_err();
assert!(err.contains("missing"), "{err}");
}
Expand Down Expand Up @@ -168,17 +156,27 @@ mod tests {
minus a 1.5 GB safety margin: missing 1.3 GB"
);
let capped = shortfall(&need(70.0, 9.5, 1.8), 2, 75.6).unwrap();
assert!(
capped.ends_with("its static pool (weights + KV, 79.5 GB) may use at most 95% of it after the CUDA context (70.5 GB): missing 9.0 GB"),
"{capped}"
);
assert!(capped.ends_with("minus a 1.5 GB safety margin: missing 7.2 GB"), "{capped}");
assert_eq!(shortfall(&need(10.0, 1.0, 1.8), 2, 75.6), None);
}

#[test]
fn missing_uses_margin_and_static_cap() {
assert_eq!(missing_gb(&need(8.0, 0.2, 1.8), 20.0), 0.0);
assert!((missing_gb(&need(15.0, 2.2, 1.8), 20.0) - 0.5).abs() < 1e-9);
assert!((missing_gb(&need(88.0, 5.0, 1.8), 96.0) - (93.0 - 0.95 * (96.0 - CUDA_CONTEXT_GB))).abs() < 1e-9);
assert!((missing_gb(&need(88.0, 5.0, 1.8), 96.0) - 0.3).abs() < 1e-9);
}

#[test]
fn mem_fraction_may_use_the_full_free_card() {
let n = need(26.0, 0.0, 1.5);
assert!((mem_fraction_from_free(&n, 30.0, 4.0).unwrap() - 1.0).abs() < 1e-9);
assert!(mem_fraction_from_free(&n, 27.0, 0.0).is_err());
}

#[test]
fn exact_fill_with_overhead_still_refuses() {
assert!(missing_gb(&need(20.0, 0.1, 1.8), 20.0) > 0.0);
assert!(missing_gb(&need(20.0, 1.0, 0.5), 21.0) > 0.0);
}
}
114 changes: 104 additions & 10 deletions engine/crates/core/src/chooser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ pub struct LoadPlan {
pub cc: String,
pub ctx: u64,
pub max_conc: u64,
pub disable_marlin: bool,
}

#[derive(Debug, Clone, PartialEq, Eq)]
Expand Down Expand Up @@ -163,10 +164,10 @@ pub fn need(input: &ChooserInput, tp: usize, cc: &str) -> Need {
pub const KV_SHARE_OF_FREE: f64 = 0.4;

pub fn cap_default_ctx(input: &ChooserInput, tp: usize, cc: &str, free_gb: f64) -> u64 {
// A context nobody asked for is halved (down to 8192) until its KV cache is at most 40% of the free VRAM.
// A context nobody asked for is halved (down to SHRINK_CTX_FLOOR) until its KV cache is at most 40% of the free VRAM.
let mut trial = input.clone();
while trial.ctx > 8192 && need(&trial, tp, cc).kv > KV_SHARE_OF_FREE * free_gb {
trial.ctx = (trial.ctx / 2).max(8192);
while trial.ctx > SHRINK_CTX_FLOOR && need(&trial, tp, cc).kv > KV_SHARE_OF_FREE * free_gb {
trial.ctx = (trial.ctx / 2).max(SHRINK_CTX_FLOOR);
}
trial.ctx
}
Expand Down Expand Up @@ -211,7 +212,27 @@ pub fn plan_with(
pinned_gpus: Option<&[u32]>,
learned: LearnedAt,
) -> Result<LoadPlan, PlanError> {
// Pinned GPUs as given, else a pinned or the smallest fitting tp over GPUs whose architecture runs the model.
let with_extra = plan_once(input, gpus, hw, pinned_tp, pinned_gpus, learned);
if input.marlin_extra_gb <= 0.0 {
return with_extra;
}
let want = (input.ctx.min(input.max_ctx), input.max_conc.max(1));
if matches!(&with_extra, Ok(p) if (p.ctx, p.max_conc) == want) {
return with_extra;
}
let mut bare = input.clone();
bare.marlin_extra_gb = 0.0;
prefer_capacity(with_extra, plan_once(&bare, gpus, hw, pinned_tp, pinned_gpus, learned))
}

fn plan_once(
input: &ChooserInput,
gpus: &[GpuPlacementInput],
hw: Option<&HardwareState>,
pinned_tp: Option<usize>,
pinned_gpus: Option<&[u32]>,
learned: LearnedAt,
) -> Result<LoadPlan, PlanError> {
if let Some(pg) = pinned_gpus.filter(|p| !p.is_empty()) {
let tp = pinned_tp.unwrap_or(pg.len());
if tp != pg.len() {
Expand Down Expand Up @@ -258,6 +279,21 @@ pub fn plan_with(
auto_plan(input, &eligible, hw, learned)
}

fn prefer_capacity(with_extra: Result<LoadPlan, PlanError>, without: Result<LoadPlan, PlanError>) -> Result<LoadPlan, PlanError> {
match (with_extra, without) {
(Ok(a), Ok(mut b)) if (b.ctx, b.max_conc) > (a.ctx, a.max_conc) => {
b.disable_marlin = true;
Ok(b)
}
(Ok(a), _) => Ok(a),
(Err(_), Ok(mut b)) => {
b.disable_marlin = true;
Ok(b)
}
(Err(e), Err(_)) => Err(e),
}
}

const MAX_TP: usize = 64;

fn or_list(v: &[usize]) -> String {
Expand Down Expand Up @@ -305,17 +341,20 @@ fn plan(input: &ChooserInput, tp: usize, set: SetRank) -> LoadPlan {
cc: set.cc,
ctx: input.ctx,
max_conc: input.max_conc,
disable_marlin: false,
}
}

pub const SHRINK_CTX_FLOOR: u64 = 2048;

fn shrink_until(input: &ChooserInput, mut fits: impl FnMut(&ChooserInput) -> Option<LoadPlan>) -> Option<LoadPlan> {
// Context halved down to 8192, then one request fewer at full context, until a memory shortfall goes away.
// Context halved down to SHRINK_CTX_FLOOR, then concurrency, until a shortfall clears.
let mut trial = input.clone();
trial.ctx = input.ctx.min(input.max_ctx);
trial.max_conc = input.max_conc.max(1);
loop {
if trial.ctx > 8192 {
trial.ctx /= 2;
if trial.ctx > SHRINK_CTX_FLOOR {
trial.ctx = (trial.ctx / 2).max(SHRINK_CTX_FLOOR);
} else if trial.max_conc > 1 {
trial.max_conc -= 1;
trial.ctx = input.ctx.min(input.max_ctx);
Expand Down Expand Up @@ -432,8 +471,8 @@ fn auto_plan(
return Ok(plan(&trial, tp, set));
}
}
if trial.ctx > 8192 {
trial.ctx /= 2;
if trial.ctx > SHRINK_CTX_FLOOR {
trial.ctx = (trial.ctx / 2).max(SHRINK_CTX_FLOOR);
} else if trial.max_conc > 1 {
trial.max_conc -= 1;
trial.ctx = input.ctx.min(input.max_ctx);
Expand Down Expand Up @@ -501,7 +540,7 @@ mod tests {
assert!(capped < 131072 && kv(capped) <= KV_SHARE_OF_FREE * 10.0, "{capped}");
assert!(kv(capped * 2) > KV_SHARE_OF_FREE * 10.0, "the largest halving that fits");
assert_eq!(cap_default_ctx(&m, 1, "12.0", 4000.0), 131072, "plenty of room keeps the default");
assert_eq!(cap_default_ctx(&m, 1, "12.0", 0.1), 8192, "never below 8192");
assert_eq!(cap_default_ctx(&m, 1, "12.0", 0.1), SHRINK_CTX_FLOOR, "never below the shrink floor");
}

#[test]
Expand Down Expand Up @@ -547,6 +586,61 @@ mod tests {
assert_eq!(need(&m, 2, "9.0").weights, 10.0);
}

#[test]
fn weight_extra_that_wont_fit_is_dropped_so_the_card_still_places() {
let mut m = inp();
m.static_vram_gb = 23.0;
m.ctx = 262_144;
m.max_ctx = 262_144;
m.max_conc = 8;
m.marlin_extra_gb = 7.85;
m.kv_bytes_per_token = 32768;
m.num_attention_heads = 24;
m.num_kv_heads = 4;
m.attention = "hybrid".into();
m.supported_sm = vec!["12.0".into()];
let plan = pick_load_plan(&m, &[gpu(0, 31.3, false)], None, None, None)
.expect("dropping the weight-extra copy must still place on one 32 GB GPU");
assert_eq!(plan.tp, 1);
assert!(plan.disable_marlin, "planned without the dual-copy weight overhead");
assert!(plan.ctx < m.ctx, "ctx reduced: {}", plan.ctx);
assert!(plan.ctx >= SHRINK_CTX_FLOOR, "{}", plan.ctx);
assert!(plan.max_conc <= m.max_conc);
}

#[test]
fn auto_fit_across_gpu_states_and_model_shapes() {
// free / foreign / arch / marlin_extra → place, drop, shrink, or refuse.
for (name, static_gb, extra, kv, free, cc, foreign, want_ok, want_drop, want_full) in [
("roomy", 10.0, 0.0, 4096, 40.0, "9.0", false, true, false, true),
("keeps-marlin", 20.0, 4.0, 4096, 40.0, "12.0", false, true, false, true),
("drops-marlin", 23.0, 7.85, 32768, 31.3, "12.0", false, true, true, false),
("shrink", 20.0, 0.0, 65536, 31.3, "9.0", false, true, false, false),
("extra-off-arch", 23.0, 7.85, 4096, 40.0, "9.0", false, true, false, true),
("empty", 20.0, 0.0, 4096, 0.0, "9.0", false, false, false, false),
("foreign", 10.0, 0.0, 4096, 40.0, "9.0", true, true, false, true),
("partial", 18.0, 0.0, 32768, 22.0, "8.6", false, true, false, false),
] {
let mut m = inp();
m.static_vram_gb = static_gb;
m.marlin_extra_gb = extra;
m.kv_bytes_per_token = kv;
m.ctx = 131072;
m.max_ctx = 131072;
m.max_conc = 8;
m.supported_sm = vec![cc.into()];
let gpus = [GpuPlacementInput { foreign_process: foreign, ..gpu_on(0, cc, free) }];
let got = pick_load_plan(&m, &gpus, None, None, None);
if !want_ok {
assert!(got.is_err(), "{name}");
continue;
}
let plan = got.unwrap_or_else(|e| panic!("{name}: {}", e.message));
assert_eq!(plan.disable_marlin, want_drop, "{name}");
assert_eq!((plan.ctx, plan.max_conc) == (m.ctx, m.max_conc), want_full, "{name}");
}
}

#[test]
fn valid_tp_divisors_and_kv_rule() {
let tps = valid_tp_values(&inp(), 8);
Expand Down
20 changes: 13 additions & 7 deletions engine/crates/core/src/engine_overhead.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,18 @@ pub fn parse_shortfall_gb(log_text: &str) -> Option<f64> {
None
}

fn size_gb_after(line: &str, key: &str) -> Option<f64> {
// "2.00 GiB" / "272.00 MiB" / "3.00 GB" → GB. Unknown unit kept as today (GiB).
let mut words = line.split(key).nth(1)?.split_whitespace();
let n: f64 = words.next()?.trim_end_matches(['.', ',']).parse().ok()?;
let unit = words.next().unwrap_or("GiB").to_ascii_lowercase();
Some(if unit.starts_with("mi") { n / 1024.0 } else { n })
}

fn torch_oom_gap_gb(line: &str) -> Option<f64> {
// "Tried to allocate 2.00 GiB. GPU 0 has a total capacity of 178.34 GiB of which 1.15 GiB is free": what was missing.
let number_after = |key: &str| -> Option<f64> {
let rest = line.split(key).nth(1)?;
rest.split_whitespace().next()?.trim_end_matches(['.', ',']).parse().ok()
};
let wanted = number_after("Tried to allocate ")?;
let free = number_after("of which ").unwrap_or(0.0);
// "Tried to allocate 2.00 GiB ... of which 1.15 GiB is free": what was missing.
let wanted = size_gb_after(line, "Tried to allocate ")?;
let free = size_gb_after(line, "of which ").unwrap_or(0.0);
Some(wanted - free)
}

Expand Down Expand Up @@ -237,6 +241,8 @@ mod tests {
fn a_torch_oom_names_what_was_missing() {
let line = "torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 12.00 GiB. GPU 0 has a total capacity of 178.34 GiB of which 1.15 GiB is free.";
assert!((parse_shortfall_gb(line).unwrap() - (12.0 - 1.15 + 1.0)).abs() < 1e-9);
let mib = "Tried to allocate 272.00 MiB. GPU 0 has a total capacity of 31.36 GiB of which 53.56 MiB is free.";
assert!((parse_shortfall_gb(mib).unwrap() - 2.0).abs() < 1e-6);
assert_eq!(parse_shortfall_gb("CUDA out of memory"), Some(4.0));
}

Expand Down
28 changes: 23 additions & 5 deletions engine/crates/core/src/engines/sglang.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,17 @@ fn memory_args(spec: &EngineSpec, extras: &[String]) -> Vec<String> {
const SGLANG_BOOT: &str = "import os, runpy, sys\n\nif __name__ == '__main__':\n sys.argv = ['sglang.launch_server', *sys.argv[1:], '--api-key', os.environ['GOETHE_ENGINE_API_KEY']]\n runpy.run_module('sglang.launch_server', run_name='__main__')\n";

fn graph_args(max_conc: u64, extras: &[String]) -> Vec<String> {
// Decode CUDA graphs captured only up to the batch the model runs (its concurrency), not SGLang's default range: a shorter load and less VRAM; graph flags the user set win.
// Decode graphs up to concurrency. Prefill capture lives outside the static pool; skip it (ponytail: else OOM on tight cards).
let theirs = extras.iter().any(|a| a.starts_with("--cuda-graph") || a == "--disable-cuda-graph");
if theirs {
return Vec::new();
}
vec!["--cuda-graph-max-bs-decode".to_string(), max_conc.max(1).to_string()]
vec![
"--cuda-graph-max-bs-decode".into(),
max_conc.max(1).to_string(),
"--cuda-graph-backend-prefill".into(),
"disabled".into(),
]
}

fn sglang_command(spec: &EngineSpec) -> Command {
Expand Down Expand Up @@ -102,7 +107,14 @@ fn sglang_command(spec: &EngineSpec) -> Command {
.arg(spec.served_name)
.arg("--uvicorn-access-log-exclude-prefixes")
.args(QUIET_PATHS);
let extras: Vec<String> = card_extra_args(spec.card).into_iter().chain(spec.load_extra.iter().cloned()).collect();
let card_extras = {
let raw = card_extra_args(spec.card);
match crate::model_facts::read_config(spec.model_path) {
Ok(cfg) => crate::sglang_infer::drop_quantization_when_config_declares(&raw, &cfg).0,
Err(_) => raw,
}
};
let extras: Vec<String> = card_extras.into_iter().chain(spec.load_extra.iter().cloned()).collect();
cmd.args(memory_args(spec, &extras));
cmd.args(graph_args(spec.max_conc, &extras));
if let Some(dtype) = spec
Expand Down Expand Up @@ -164,8 +176,14 @@ mod tests {

#[test]
fn graphs_are_captured_up_to_the_concurrency_unless_the_user_says_otherwise() {
assert_eq!(graph_args(4, &[]), ["--cuda-graph-max-bs-decode", "4"], "the pinned SGLang splits the flag into -decode and -prefill");
assert_eq!(graph_args(0, &[]), ["--cuda-graph-max-bs-decode", "1"]);
assert_eq!(
graph_args(4, &[]),
["--cuda-graph-max-bs-decode", "4", "--cuda-graph-backend-prefill", "disabled"]
);
assert_eq!(
graph_args(0, &[]),
["--cuda-graph-max-bs-decode", "1", "--cuda-graph-backend-prefill", "disabled"]
);
for theirs in ["--cuda-graph-max-bs-decode=16", "--cuda-graph-bs-prefill", "--cuda-graph-config", "--disable-cuda-graph"] {
assert!(graph_args(4, &[theirs.to_string()]).is_empty(), "{theirs}");
}
Expand Down
6 changes: 6 additions & 0 deletions engine/crates/core/src/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1617,6 +1617,8 @@ fn plan(home: &Path, opts: &InstallOpts, t: &Target) -> Result<Planned, String>
if let Some(msg) = sgl::reserved_flag_error(&registry_args) {
return Err(format!("registry entry for {}: {msg}", t.model_id));
}
let (registry_args, quant_note) = sgl::drop_quantization_when_config_declares(&registry_args, &cfg);
notes.extend(quant_note);
let (mut base_args, dir_notes) = sgl::resolve_model_dir_args(&registry_args, dir)?;
notes.extend(dir_notes);
if opts.trust_remote_code {
Expand All @@ -1629,6 +1631,10 @@ fn plan(home: &Path, opts: &InstallOpts, t: &Target) -> Result<Planned, String>
.and_then(|v| v.as_array())
.map(|a| a.iter().filter_map(|x| x.as_str().map(String::from)).collect())
.unwrap_or_default();
let (old_args, old_quant_note) = sgl::drop_quantization_when_config_declares(&old_args, &cfg);
if old_quant_note.is_some() && !sgl::has_flag(&cli_args, "--quantization") {
notes.extend(old_quant_note);
}
let mut inferred = Vec::new();
for (flag, value, why) in sgl::infer_parsers(&cfg, dir) {
if [&base_args, &old_args, &cli_args].iter().all(|a| !sgl::has_flag(a, &flag)) {
Expand Down
Loading
Loading