Skip to content

fix(installer): make the training size configurable at install (TRACEBLOC_TRAINING_RESOURCES) #308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LukasWodka wants to merge 1 commit into
base: develop
Choose a base branch
from
+37 −4
Open

fix(installer): make the training size configurable at install (TRACEBLOC_TRAINING_RESOURCES) #308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions scripts/install-k8s.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
# HOST_DATA_DIR=~/.tracebloc default: ~/.tracebloc
# CLIENT_ENV=dev optional; if not set, CLIENT_ENV is not added to env in values
# TRACEBLOC_SKIP_REBOOT_PROMPT=1 (Linux) skip "Reboot now?" after NVIDIA driver install
# TRACEBLOC_TRAINING_RESOURCES="cpu=4,memory=16Gi" CPU/RAM each training run
# may use (default cpu=2,memory=8Gi; sets requests==limits)
# =============================================================================

set -euo pipefail
Expand Down
7 changes: 5 additions & 2 deletions scripts/lib/install-client-helm.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,8 +518,11 @@ install_client_helm() {

env:
$([ -n "${CLIENT_ENV:-}" ] && printf ' CLIENT_ENV: "%s"\n' "$CLIENT_ENV")${proxy_env_yaml}
RESOURCE_LIMITS: "cpu=2,memory=8Gi"
RESOURCE_REQUESTS: "cpu=2,memory=8Gi"
# Training size: how much CPU/RAM each training run gets. One knob sets
# requests == limits (Guaranteed QoS; client-runtime keeps them in lockstep).
# Set at install time with TRACEBLOC_TRAINING_RESOURCES="cpu=4,memory=16Gi".
RESOURCE_LIMITS: "${TRACEBLOC_TRAINING_RESOURCES:-cpu=2,memory=8Gi}"
RESOURCE_REQUESTS: "${TRACEBLOC_TRAINING_RESOURCES:-cpu=2,memory=8Gi}"
GPU_LIMITS: "$gpu_val"
GPU_REQUESTS: "$gpu_val"
RUNTIME_CLASS_NAME: ""
Expand Down
4 changes: 2 additions & 2 deletions scripts/manifest.sha256
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
b946562948a7e6581147ec5a661456d841872464dd49df924236ac7825db70d9 scripts/install-k8s.sh
42995d6a5085e650e9a7ed88dda3d4b34f412d1e8b721e8e35bef571fdb2213e scripts/install-k8s.sh
5bdcbae06d6fddca16ca616bdbd83ba68d4206361ea5c2e73c09ac492ea2dfa0 scripts/lib/common.sh
a370fc0c170f34f21d36d67e5434443b4ac8c671fba259826878b9840838b9c4 scripts/lib/preflight.sh
e14e4dfc43f0ac8648a01b09f597d8c6b9e3d3dfb5a2953b099de7a09b4c29f5 scripts/lib/detect-gpu.sh
Expand All @@ -8,7 +8,7 @@ fd8d37d698c242ab081095625aa430bb91971db646e35393a02bbadd41325898 scripts/lib/gp
2e64a2c47bdd14c861f8b5e86b06d1dfe4eb349ea6f76d9d22ecf3bb1ec12e04 scripts/lib/setup-linux.sh
dd2e1f517b0f6bb665eacebfb7885c6a81443f69c2ebc772433a6baed53c7a8e scripts/lib/cluster.sh
5de1afac8b1062a7f8d0691b46ad39ccdc2287e0e422b53c48628ecfd47d852e scripts/lib/gpu-plugins.sh
3e5bc6dac689f269a9ba06e1be5a831465ff74e8d78465ec7f893ea097a5b785 scripts/lib/install-client-helm.sh
06de18a5025fad9fe5490185006f2cec2dac60a007788a7d4d86a5fe32b0bc7f scripts/lib/install-client-helm.sh
f2350fbafbc5ad22d7a87ec55c83b5138e2d039a6eb5580bb8410fd01b3536e2 scripts/lib/install-cli.sh
63ac8ab924a453f2ae0840c2087a97d7c592c8083260a162c6c51322df96876e scripts/lib/provision.sh
58bd83c06ca16e1d9346700a3f6bcafc67f20130f0bc1f160d2faedab19fd2cf scripts/lib/summary.sh
Expand Down
28 changes: 28 additions & 0 deletions scripts/tests/install-client-helm.bats
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,3 +588,31 @@ setup() {
[ "$status" -eq 0 ]
! grep -q 'HTTP_PROXY_HOST' "$HOST_DATA_DIR/values.yaml"
}

@test "install_client_helm: TRACEBLOC_TRAINING_RESOURCES overrides the training size in generated values" {
HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR"
_ensure_tracebloc_dirs() { :; }
_ensure_release_dirs() { :; }
_ensure_helm_runnable() { :; }
helm() { record "helm $*"; return 0; }
verify_credentials() { printf valid; }
export TRACEBLOC_TRAINING_RESOURCES="cpu=4,memory=16Gi"
run install_client_helm <<< $'myid\nmypw'
[ "$status" -eq 0 ]
grep -q 'RESOURCE_LIMITS: "cpu=4,memory=16Gi"' "$HOST_DATA_DIR/values.yaml"
grep -q 'RESOURCE_REQUESTS: "cpu=4,memory=16Gi"' "$HOST_DATA_DIR/values.yaml"
}

@test "install_client_helm: default training size stays cpu=2,memory=8Gi" {
HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR"
_ensure_tracebloc_dirs() { :; }
_ensure_release_dirs() { :; }
_ensure_helm_runnable() { :; }
helm() { record "helm $*"; return 0; }
verify_credentials() { printf valid; }
unset TRACEBLOC_TRAINING_RESOURCES
run install_client_helm <<< $'myid\nmypw'
[ "$status" -eq 0 ]
grep -q 'RESOURCE_LIMITS: "cpu=2,memory=8Gi"' "$HOST_DATA_DIR/values.yaml"
grep -q 'RESOURCE_REQUESTS: "cpu=2,memory=8Gi"' "$HOST_DATA_DIR/values.yaml"
}
Loading