Add daemon-driven HostOSReboot with restart-safe result reporting

api/machina/v1alpha3/machineoperation_types.go

@@ -51,7 +51,7 @@ type MachineOperationList struct {
 // OperationKind identifies the kind of operation to perform. Predefined
 // operations cover common lifecycle actions; custom operations may be
 // supported by individual cloud controllers.
-// +kubebuilder:validation:Enum=NodeReboot;AgentUpgrade;AgentReset;HostReboot;HostPowerOff;HostPowerOn;HostReplace
+// +kubebuilder:validation:Enum=NodeReboot;AgentUpgrade;AgentReset;HostOSReboot;HostReboot;HostPowerOff;HostPowerOn;HostReplace
 type OperationKind string
 
 const (
@@ -69,6 +69,10 @@ const (
 	// all associated resources. Handled by the in-VM agent.
 	OperationAgentReset OperationKind = "AgentReset"
 
+	// OperationHostOSReboot reboots the host operating system through the
+	// host-side daemon agent, without an out-of-band power cycle.
+	OperationHostOSReboot OperationKind = "HostOSReboot"
+
 	// OperationHostReboot triggers a full hardware power cycle of the host
 	// via BMC (e.g. Redfish). Handled by the machina controller or cloud
 	// controller.
@@ -207,6 +211,7 @@ type ProviderMachineSnapshot struct {
 }
 
 // MachineOperationSpec defines the desired state of a MachineOperation.
+// +kubebuilder:validation:XValidation:rule="self.operationKind != 'HostOSReboot' || (has(self.machineRef) && size(self.machineRef) > 0)",message="HostOSReboot requires machineRef"
 type MachineOperationSpec struct {
 	// MachineRef is the name of the Machine CR this operation targets.
 	// Either machineRef or machineSelector must be set.
@@ -267,6 +272,11 @@ type MachineOperationStatus struct {
 	// +optional
 	ObservedMachineGeneration int64 `json:"observedMachineGeneration,omitempty"`
 
+	// HostBootID is the host boot ID observed before issuing HostOSReboot.
+	// The daemon completes the operation only after this ID changes.
+	// +optional
+	HostBootID string `json:"hostBootID,omitempty"`
+
 	// Targets records the per-machine execution state for this operation.
 	// For single-machine operations this contains one entry. For selector
 	// operations, targets are snapshotted when execution begins and remain

api/machina/v1alpha3/machineoperation_validation_test.go

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha3
+
+import (
+	"os"
+	"testing"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"sigs.k8s.io/yaml"
+)
+
+func TestHostOSRebootSchema(t *testing.T) {
+	t.Parallel()
+
+	data, err := os.ReadFile("../../../deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var crd apiextensionsv1.CustomResourceDefinition
+	if err := yaml.Unmarshal(data, &crd); err != nil {
+		t.Fatal(err)
+	}
+
+	schema := crd.Spec.Versions[0].Schema.OpenAPIV3Schema
+	spec := schema.Properties["spec"]
+	assertSchemaValidations(t, spec, map[string]string{
+		"self.operationKind != 'HostOSReboot' || (has(self.machineRef) && size(self.machineRef) > 0)": "HostOSReboot requires machineRef",
+	})
+
+	if got := schema.Properties["status"].Properties["hostBootID"].Type; got != "string" {
+		t.Errorf("hostBootID type = %q, want string", got)
+	}
+
+	for _, kind := range spec.Properties["operationKind"].Enum {
+		if string(kind.Raw) == `"HostOSReboot"` {
+			return
+		}
+	}
+
+	t.Error("operationKind enum is missing HostOSReboot")
+}

cmd/agent/internal/daemon/controller.go

@@ -80,6 +80,7 @@ func runController(
 	machineOperations := &machineOperationTarget{
 		installation:         installation,
 		Client:               c,
+		operationReader:      mgr.GetAPIReader(),
 		log:                  log,
 		machineName:          machineName,
 		nodeOperator:         nodeOperator,
@@ -94,6 +95,7 @@ func runController(
 		nodeName,
 		daemon.MachineOperationHandlers{
 			v1alpha3.OperationNodeReboot:   machineOperations.reconcileNodeReboot,
+			v1alpha3.OperationHostOSReboot: machineOperations.reconcileHostOSReboot,
 			v1alpha3.OperationAgentUpgrade: machineOperations.reconcileAgentUpgrade,
 			v1alpha3.OperationAgentReset:   machineOperations.reconcileAgentReset,
 		},

cmd/agent/internal/daemon/controller_machineoperation.go

@@ -26,6 +26,7 @@ const agentUpgradeLockRetryDelay = 2 * time.Second
 type machineOperationTarget struct {
 	installation *installstate.Store
 	client.Client
+	operationReader      client.Reader
 	log                  *slog.Logger
 	machineName          string
 	nodeOperator         nodeOperator

cmd/agent/internal/daemon/controller_test.go

@@ -41,6 +41,12 @@ type fakeNodeOperator struct {
 	restartActive *ActiveMachine
 	restartErr    error
 
+	hostBootID       string
+	hostBootIDErr    error
+	hostRebootCalls  int
+	hostRebootErr    error
+	hostRebootDuring func()
+
 	resetCalled bool
 	resetErr    error
 
@@ -101,6 +107,19 @@ func (op *fakeNodeOperator) ResetAgentResources(context.Context, *slog.Logger) e
 	return op.resetErr
 }
 
+func (op *fakeNodeOperator) HostBootID() (string, error) {
+	return op.hostBootID, op.hostBootIDErr
+}
+
+func (op *fakeNodeOperator) RebootHost(context.Context, *slog.Logger) error {
+	op.hostRebootCalls++
+	if op.hostRebootDuring != nil {
+		op.hostRebootDuring()
+	}
+
+	return op.hostRebootErr
+}
+
 func (op *fakeNodeOperator) StopDaemon(context.Context, *slog.Logger) error {
 	op.stopCalled = true
 
@@ -169,6 +188,7 @@ func newTestMachinaMachineOperationReconcilerWithLockPath(
 	target := &machineOperationTarget{
 		installation:         installstate.NewStore(filepath.Join(t.TempDir(), "state"), filepath.Join(t.TempDir(), "install.lock")),
 		Client:               c,
+		operationReader:      c,
 		log:                  discardLogger(),
 		machineName:          "test-machine",
 		nodeOperator:         op,
@@ -186,6 +206,7 @@ func newTestMachinaMachineOperationReconcilerWithLockPath(
 		"test-node",
 		daemon.MachineOperationHandlers{
 			v1alpha3.OperationNodeReboot:   target.reconcileNodeReboot,
+			v1alpha3.OperationHostOSReboot: target.reconcileHostOSReboot,
 			v1alpha3.OperationAgentUpgrade: target.reconcileAgentUpgrade,
 			v1alpha3.OperationAgentReset:   target.reconcileAgentReset,
 		},

cmd/agent/internal/daemon/hostreboot.go

@@ -0,0 +1,149 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package daemon
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"strings"
+	"time"
+
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3"
+	"github.com/Azure/unbounded/cmd/agent/internal/installstate"
+	"github.com/Azure/unbounded/internal/executil"
+	daemon "github.com/Azure/unbounded/pkg/agent/daemon"
+)
+
+const (
+	hostRebootPollInterval = 5 * time.Second
+	hostRebootTimeout      = 5 * time.Minute
+)
+
+func (nspawnNodeOperator) HostBootID() (string, error) {
+	data, err := os.ReadFile("/proc/sys/kernel/random/boot_id")
+	if err != nil {
+		return "", fmt.Errorf("read host boot ID: %w", err)
+	}
+
+	id := strings.TrimSpace(string(data))
+	if id == "" {
+		return "", fmt.Errorf("host boot ID is empty")
+	}
+
+	return id, nil
+}
+
+func (nspawnNodeOperator) RebootHost(ctx context.Context, log *slog.Logger) error {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	_, err := executil.OutputCmd(ctx, log, "reboot")
+
+	return err
+}
+
+func (t *machineOperationTarget) reconcileHostOSReboot(ctx context.Context, store daemon.MachineOperationStore[int64], op daemon.MachineOperation) (ctrl.Result, error) {
+	lock, err := t.installation.AcquireMutationLock()
+	if errors.Is(err, installstate.ErrLockHeld) {
+		return ctrl.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil
+	}
+
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	defer releaseInstallationLock(t.log, lock)
+
+	// Read uncached state so retries cannot reissue a reboot while the informer
+	// is still catching up with the status written before the command.
+	var latest v1alpha3.MachineOperation
+	if err := t.operationReader.Get(ctx, client.ObjectKey{Name: op.Name}, &latest); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if latest.Status.IsTerminal() {
+		return ctrl.Result{}, nil
+	}
+
+	if latest.Spec.MachineRef == "" {
+		return ctrl.Result{}, store.Finish(ctx, op, daemon.MachineOperationResult[int64]{
+			Phase:   v1alpha3.OperationPhaseFailed,
+			Reason:  "InvalidSpec",
+			Message: "HostOSReboot requires machineRef",
+		})
+	}
+
+	if latest.Spec.MachineRef != t.machineName {
+		return ctrl.Result{}, nil
+	}
+
+	bootID, err := t.nodeOperator.HostBootID()
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if latest.Status.HostBootID != "" {
+		if latest.Status.HostBootID != bootID {
+			return ctrl.Result{}, store.Finish(ctx, op, daemon.MachineOperationResult[int64]{
+				Phase:                     v1alpha3.OperationPhaseComplete,
+				Reason:                    "Succeeded",
+				Message:                   "HostOSReboot completed",
+				ObservedMachineGeneration: latest.Status.ObservedMachineGeneration,
+			})
+		}
+
+		if latest.Status.StartedAt == nil || time.Since(latest.Status.StartedAt.Time) >= hostRebootTimeout {
+			return ctrl.Result{}, store.Finish(ctx, op, daemon.MachineOperationResult[int64]{
+				Phase:   v1alpha3.OperationPhaseFailed,
+				Reason:  "RebootTimeout",
+				Message: "host boot ID did not change after requesting reboot",
+			})
+		}
+
+		return ctrl.Result{RequeueAfter: hostRebootPollInterval}, nil
+	}
+
+	machine, err := getLocalMachine(ctx, t.Client, t.machineName)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Persist the boot ID and execution intent together before reboot can stop
+	// this process. A daemon restart on the same boot must never replay it.
+	now := metav1.Now()
+	latest.Status.HostBootID = bootID
+	latest.Status.Phase = v1alpha3.OperationPhaseInProgress
+	latest.Status.StartedAt = &now
+	latest.Status.ObservedMachineGeneration = machine.Generation
+	latest.Status.Message = "requesting host OS reboot"
+	apimeta.SetStatusCondition(&latest.Status.Conditions, metav1.Condition{
+		Type:               v1alpha3.MachineOperationConditionCompleted,
+		Status:             metav1.ConditionFalse,
+		Reason:             "InProgress",
+		Message:            latest.Status.Message,
+		ObservedGeneration: latest.Generation,
+	})
+
+	if err := t.Status().Update(ctx, &latest); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if latest.Status.HostBootID != bootID {
+		return ctrl.Result{}, fmt.Errorf("host boot ID was not persisted; update the MachineOperation CRD before using HostOSReboot")
+	}
+
+	if err := t.nodeOperator.RebootHost(ctx, t.log); err != nil {
+		return finishFailedMachineOperation(ctx, store, op, err)
+	}
+
+	return ctrl.Result{RequeueAfter: hostRebootPollInterval}, nil
+}