diff --git a/scripts/install-k8s.sh b/scripts/install-k8s.sh index ce36343..4b99de9 100755 --- a/scripts/install-k8s.sh +++ b/scripts/install-k8s.sh @@ -23,6 +23,8 @@ # HOST_DATA_DIR=~/.tracebloc default: ~/.tracebloc # CLIENT_ENV=dev optional; if not set, CLIENT_ENV is not added to env in values # TRACEBLOC_SKIP_REBOOT_PROMPT=1 (Linux) skip "Reboot now?" after NVIDIA driver install +# TRACEBLOC_TRAINING_RESOURCES="cpu=4,memory=16Gi" CPU/RAM each training run +# may use (default cpu=2,memory=8Gi; sets requests==limits) # ============================================================================= set -euo pipefail diff --git a/scripts/lib/install-client-helm.sh b/scripts/lib/install-client-helm.sh index 4a36d1f..efcca11 100644 --- a/scripts/lib/install-client-helm.sh +++ b/scripts/lib/install-client-helm.sh @@ -518,8 +518,11 @@ install_client_helm() { env: $([ -n "${CLIENT_ENV:-}" ] && printf ' CLIENT_ENV: "%s"\n' "$CLIENT_ENV")${proxy_env_yaml} - RESOURCE_LIMITS: "cpu=2,memory=8Gi" - RESOURCE_REQUESTS: "cpu=2,memory=8Gi" + # Training size: how much CPU/RAM each training run gets. One knob sets + # requests == limits (Guaranteed QoS; client-runtime keeps them in lockstep). + # Set at install time with TRACEBLOC_TRAINING_RESOURCES="cpu=4,memory=16Gi". + RESOURCE_LIMITS: "${TRACEBLOC_TRAINING_RESOURCES:-cpu=2,memory=8Gi}" + RESOURCE_REQUESTS: "${TRACEBLOC_TRAINING_RESOURCES:-cpu=2,memory=8Gi}" GPU_LIMITS: "$gpu_val" GPU_REQUESTS: "$gpu_val" RUNTIME_CLASS_NAME: "" diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256 index f2b4ac9..7a766dd 100644 --- a/scripts/manifest.sha256 +++ b/scripts/manifest.sha256 @@ -1,4 +1,4 @@ -b946562948a7e6581147ec5a661456d841872464dd49df924236ac7825db70d9 scripts/install-k8s.sh +42995d6a5085e650e9a7ed88dda3d4b34f412d1e8b721e8e35bef571fdb2213e scripts/install-k8s.sh dd0d6c1a533cc30a63524d1cd5796009e70be055c30ec9022d2267dee7be29c5 scripts/lib/common.sh a370fc0c170f34f21d36d67e5434443b4ac8c671fba259826878b9840838b9c4 scripts/lib/preflight.sh 19be2771df0e1a41b4fa9678e1cf6a77492304f66f73cf705a2ae42b1dac2ba3 scripts/lib/detect-gpu.sh @@ -8,7 +8,7 @@ fd8d37d698c242ab081095625aa430bb91971db646e35393a02bbadd41325898 scripts/lib/gp 2e64a2c47bdd14c861f8b5e86b06d1dfe4eb349ea6f76d9d22ecf3bb1ec12e04 scripts/lib/setup-linux.sh dd2e1f517b0f6bb665eacebfb7885c6a81443f69c2ebc772433a6baed53c7a8e scripts/lib/cluster.sh 5de1afac8b1062a7f8d0691b46ad39ccdc2287e0e422b53c48628ecfd47d852e scripts/lib/gpu-plugins.sh -ae327ea8530930cce34d18d4001b5321bb6615d24060eebc81004f6e2fd7e1f5 scripts/lib/install-client-helm.sh +8c2a6c20e7384280690bc0bc0fda72aa7ab0628624280c49f676465e1fc96f94 scripts/lib/install-client-helm.sh f890cc3917b09b4b779fcd1a53b902370cb291488a7905f9d710daa871a9eb72 scripts/lib/install-cli.sh fac5860148ccd199fca2f17042227a33046c3f18df23edf8a67a232d07d2cda3 scripts/lib/provision.sh 7c5ca5fc1b3f3d5b2b699811da603b40b8879b317b1c756b8ed15da104192828 scripts/lib/summary.sh diff --git a/scripts/tests/install-client-helm.bats b/scripts/tests/install-client-helm.bats index 1b42342..1fa106d 100644 --- a/scripts/tests/install-client-helm.bats +++ b/scripts/tests/install-client-helm.bats @@ -588,3 +588,31 @@ setup() { [ "$status" -eq 0 ] ! grep -q 'HTTP_PROXY_HOST' "$HOST_DATA_DIR/values.yaml" } + +@test "install_client_helm: TRACEBLOC_TRAINING_RESOURCES overrides the training size in generated values" { + HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" + _ensure_tracebloc_dirs() { :; } + _ensure_release_dirs() { :; } + _ensure_helm_runnable() { :; } + helm() { record "helm $*"; return 0; } + verify_credentials() { printf valid; } + export TRACEBLOC_TRAINING_RESOURCES="cpu=4,memory=16Gi" + run install_client_helm <<< $'myid\nmypw' + [ "$status" -eq 0 ] + grep -q 'RESOURCE_LIMITS: "cpu=4,memory=16Gi"' "$HOST_DATA_DIR/values.yaml" + grep -q 'RESOURCE_REQUESTS: "cpu=4,memory=16Gi"' "$HOST_DATA_DIR/values.yaml" +} + +@test "install_client_helm: default training size stays cpu=2,memory=8Gi" { + HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" + _ensure_tracebloc_dirs() { :; } + _ensure_release_dirs() { :; } + _ensure_helm_runnable() { :; } + helm() { record "helm $*"; return 0; } + verify_credentials() { printf valid; } + unset TRACEBLOC_TRAINING_RESOURCES + run install_client_helm <<< $'myid\nmypw' + [ "$status" -eq 0 ] + grep -q 'RESOURCE_LIMITS: "cpu=2,memory=8Gi"' "$HOST_DATA_DIR/values.yaml" + grep -q 'RESOURCE_REQUESTS: "cpu=2,memory=8Gi"' "$HOST_DATA_DIR/values.yaml" +}