Add k8s-admin user

Create a non-root k8s-admin account on instances and update Terraform/Ansible to use it.

Signed-off-by: Sudharshan Muralidharan <sudharshan.muralidharan1@ibm.com>

Author: sudharshanibm

kubetest2-tf/data/vpc/main.tf

@@ -37,6 +37,25 @@ resource "ibm_is_instance_template" "node_template" {
     subnet          = local.subnet_id
     security_groups = [local.security_group_id]
   }
+
+  user_data = <<-EOT
+#cloud-config
+users:
+  - default
+  - name: k8s-admin
+    shell: /bin/bash
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    groups: [sudo]
+    ssh_authorized_keys:
+      - ${data.ibm_is_ssh_key.ssh_key.public_key}
+runcmd:
+  - |
+    # Ensure k8s-admin SSH dir has correct permissions
+    mkdir -p /home/k8s-admin/.ssh
+    chown -R k8s-admin:k8s-admin /home/k8s-admin/.ssh
+    chmod 700 /home/k8s-admin/.ssh
+    chmod 600 /home/k8s-admin/.ssh/authorized_keys
+EOT
 }
 
 module "master" {
@@ -59,32 +78,104 @@ module "workers" {
 }
 
 resource "null_resource" "wait-for-master-completes" {
+  depends_on = [module.master]
+  
+  # First wait for cloud-init to complete using root user (still available during boot)
+  provisioner "local-exec" {
+    command = <<-EOT
+      max_attempts=60
+      attempt=0
+      while [ $attempt -lt $max_attempts ]; do
+        # Try k8s-admin first (root SSH is disabled on new IBM Cloud VPC-VSIs)
+        if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
+               -i ${var.ssh_private_key} k8s-admin@${module.master.public_ip} \
+               "sudo cloud-init status --wait" 2>/dev/null; then
+          echo "Cloud-init completed on master (via k8s-admin)"
+          break
+        fi
+        # Fallback to root for older images that still have root SSH enabled
+        if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
+               -i ${var.ssh_private_key} root@${module.master.public_ip} \
+               "cloud-init status --wait" 2>/dev/null; then
+          echo "Cloud-init completed on master (via root)"
+          break
+        fi
+        attempt=$((attempt + 1))
+        echo "Waiting for cloud-init on master (attempt $attempt/$max_attempts)..."
+        sleep 10
+      done
+      if [ $attempt -eq $max_attempts ]; then
+        echo "ERROR: Timed out waiting for cloud-init on master"
+        exit 1
+      fi
+    EOT
+  }
+  
+  # Then verify k8s-admin user is accessible
   connection {
     type        = "ssh"
-    user        = "root"
+    user        = "k8s-admin"
     host        = module.master.public_ip
     private_key = file(var.ssh_private_key)
-    timeout     = "20m"
+    timeout     = "5m"
   }
   provisioner "remote-exec" {
     inline = [
-      "cloud-init status -w"
+      "echo 'k8s-admin user is ready on master'"
     ]
   }
 }
 
 resource "null_resource" "wait-for-workers-completes" {
-  count = var.workers_count
+  count      = var.workers_count
+  depends_on = [module.workers]
+  
+  # First wait for cloud-init to complete using root user (still available during boot)
+  provisioner "local-exec" {
+    command = <<-EOT
+      max_attempts=60
+      attempt=0
+      worker_ip="${module.workers[count.index].public_ip}"
+      worker_index="${count.index}"
+      ssh_key="${var.ssh_private_key}"
+      
+      while [ $attempt -lt $max_attempts ]; do
+        # Try k8s-admin first (root SSH is disabled on new IBM Cloud VPC-VSIs)
+        if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
+               -i "$ssh_key" k8s-admin@"$worker_ip" \
+               "sudo cloud-init status --wait" 2>/dev/null; then
+          echo "Cloud-init completed on worker $worker_index (via k8s-admin)"
+          break
+        fi
+        # Fallback to root for older images that still have root SSH enabled
+        if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
+               -i "$ssh_key" root@"$worker_ip" \
+               "cloud-init status --wait" 2>/dev/null; then
+          echo "Cloud-init completed on worker $worker_index (via root)"
+          break
+        fi
+        attempt=$((attempt + 1))
+        echo "Waiting for cloud-init on worker $worker_index (attempt $attempt/$max_attempts)..."
+        sleep 10
+      done
+      if [ $attempt -eq $max_attempts ]; then
+        echo "ERROR: Timed out waiting for cloud-init on worker $worker_index"
+        exit 1
+      fi
+    EOT
+  }
+  
+  # Then verify k8s-admin user is accessible
   connection {
     type        = "ssh"
-    user        = "root"
+    user        = "k8s-admin"
     host        = module.workers[count.index].public_ip
     private_key = file(var.ssh_private_key)
-    timeout     = "15m"
+    timeout     = "5m"
   }
   provisioner "remote-exec" {
     inline = [
-      "cloud-init status -w"
+      "echo 'k8s-admin user is ready on worker ${count.index}'"
     ]
   }
 }

kubetest2-tf/deployer/deployer.go

@@ -50,6 +50,17 @@ const (
 [workers]
 {{range .Workers}}{{.}}
 {{end}}
+{{if .IsVPC}}
+[masters:vars]
+ansible_user=k8s-admin
+ansible_become=true
+ansible_become_method=sudo
+
+[workers:vars]
+ansible_user=k8s-admin
+ansible_become=true
+ansible_become_method=sudo
+{{end}}
 `
 )
 
@@ -58,6 +69,7 @@ var GitTag string
 type AnsibleInventory struct {
 	Masters []string
 	Workers []string
+	IsVPC   bool
 }
 
 // Add additional Linux package dependencies here, used by checkDependencies()
@@ -269,7 +281,9 @@ func (d *deployer) Up() error {
 			break
 		}
 	}
-	inventory := AnsibleInventory{}
+	inventory := AnsibleInventory{
+		IsVPC: d.TargetProvider == "vpc",
+	}
 	tfMetaOutput, err := terraform.Output(d.tmpDir, d.TargetProvider)
 	if err != nil {
 		return err
@@ -343,6 +357,17 @@ func (d *deployer) Up() error {
 
 	// Add-in the extra-vars set to the final set.
 	maps.Insert(combinedAnsibleVars, maps.All(d.ExtraVars))
+
+	// For VPC deployments, override ansible_user to k8s-admin since IBM Cloud
+	// has disabled root SSH access for new VPC-VSI instances.
+	// Extra-vars have the highest precedence in Ansible, so this overrides
+	// the ansible_user: root in group_vars/all (shared with PowerVS).
+	if d.TargetProvider == "vpc" {
+		combinedAnsibleVars["ansible_user"] = "k8s-admin"
+		combinedAnsibleVars["ansible_become"] = "true"
+		combinedAnsibleVars["ansible_become_method"] = "sudo"
+	}
+
 	klog.Infof("Updated ansible variables with extra vars: %+v", combinedAnsibleVars)
 	if err = ansible.Playbook(d.tmpDir, filepath.Join(d.tmpDir, "hosts"), d.Playbook, combinedAnsibleVars); err != nil {
 		return fmt.Errorf("failed to run ansible playbook: %v", err)