Reserve taints

sats-23 · sats-23 · commit b5bb6e837e76 · 2026-03-06T12:25:56.000+05:30
diff --git a/CONTEXT.md b/CONTEXT.md
@@ -257,7 +257,7 @@ spec:
     - type: "network.kubernetes.io/NetworkProxyReady"
       requiredStatus: "True"
   taint:
-    key: "readiness.k8s.io/NetworkReady"
+    key: "readiness.k8s.io/network/not-ready"
     effect: "NoSchedule"
     value: "pending"
   enforcementMode: "bootstrap-only"
@@ -277,7 +277,7 @@ spec:
     - type: "storage.kubernetes.io/CSIReady"
       requiredStatus: "True"
   taint:
-    key: "readiness.k8s.io/StorageReady"
+    key: "readiness.k8s.io/storage/not-ready"
     effect: "NoSchedule"
   enforcementMode: "continuous"
   gracePeriod: "60s"
@@ -420,4 +420,4 @@ rules:
 ### Networking
 - Cluster internal only
 - Health check endpoints on :8081
-- Metrics endpoint on :8080
+- Metrics endpoint on :8080
diff --git a/README.md b/README.md
@@ -51,7 +51,7 @@ spec:
     - type: "example.com/CNIReady"
       requiredStatus: "True"
   taint:
-    key: "readiness.k8s.io/NetworkReady"
+    key: "readiness.k8s.io/network/not-ready"
     effect: "NoSchedule"
     value: "pending"
   enforcementMode: "bootstrap-only"
@@ -62,6 +62,19 @@ spec:
 
 Find a more detailed walkthrough of setting up Node Readiness Controller in your Kind cluster [here](https://github.com/kubernetes-sigs/node-readiness-controller/blob/main/docs/TEST_README.md).
 
+### Taint Key Conventions
+
+- Reserved core prefixes are not allowed for user rules (except the controller-owned `readiness.k8s.io/network/not-ready` and `readiness.k8s.io/storage/not-ready`):
+  - readiness.k8s.io/system/*
+  - readiness.k8s.io/core/*
+  - readiness.k8s.io/node/*
+  - readiness.k8s.io/device/*
+  - readiness.k8s.io/network/* (except readiness.k8s.io/network/not-ready)
+  - readiness.k8s.io/storage/* (except readiness.k8s.io/storage/not-ready)
+- Use user-space keys under readiness.k8s.io/* with a DNS-style component, for example:
+  - readiness.k8s.io/network/not-ready
+  - readiness.k8s.io/storage/not-ready
+
 ## High-level Roadmap
 
 - [X] Release v0.1.0
diff --git a/api/v1alpha1/nodereadinessrule_types.go b/api/v1alpha1/nodereadinessrule_types.go
@@ -75,6 +75,12 @@ type NodeReadinessRuleSpec struct {
 	//
 	// +required
 	// +kubebuilder:validation:XValidation:rule="self.key.startsWith('readiness.k8s.io/')",message="taint key must start with 'readiness.k8s.io/'"
+	// +kubebuilder:validation:XValidation:rule="!self.key.startsWith('readiness.k8s.io/system/')",message="reserved taint prefix 'readiness.k8s.io/system/*' is not allowed"
+	// +kubebuilder:validation:XValidation:rule="!self.key.startsWith('readiness.k8s.io/core/')",message="reserved taint prefix 'readiness.k8s.io/core/*' is not allowed"
+	// +kubebuilder:validation:XValidation:rule="!self.key.startsWith('readiness.k8s.io/node/')",message="reserved taint prefix 'readiness.k8s.io/node/*' is not allowed"
+	// +kubebuilder:validation:XValidation:rule="!self.key.startsWith('readiness.k8s.io/device/')",message="reserved taint prefix 'readiness.k8s.io/device/*' is not allowed"
+	// +kubebuilder:validation:XValidation:rule="!self.key.startsWith('readiness.k8s.io/network/') || self.key == 'readiness.k8s.io/network/not-ready'",message="reserved taint prefix 'readiness.k8s.io/network/*' is not allowed except 'readiness.k8s.io/network/not-ready'"
+	// +kubebuilder:validation:XValidation:rule="!self.key.startsWith('readiness.k8s.io/storage/') || self.key == 'readiness.k8s.io/storage/not-ready'",message="reserved taint prefix 'readiness.k8s.io/storage/*' is not allowed except 'readiness.k8s.io/storage/not-ready'"
 	// +kubebuilder:validation:XValidation:rule="self.key.size() <= 253",message="taint key length must be at most 253 characters"
 	// +kubebuilder:validation:XValidation:rule="!has(self.value) || self.value.size() <= 63",message="taint value length must be at most 63 characters"
 	// +kubebuilder:validation:XValidation:rule="self.effect in ['NoSchedule', 'PreferNoSchedule', 'NoExecute']",message="taint effect must be one of 'NoSchedule', 'PreferNoSchedule', 'NoExecute'"
diff --git a/config/crd/bases/readiness.node.x-k8s.io_nodereadinessrules.yaml b/config/crd/bases/readiness.node.x-k8s.io_nodereadinessrules.yaml
@@ -186,6 +186,24 @@ spec:
                 x-kubernetes-validations:
                 - message: taint key must start with 'readiness.k8s.io/'
                   rule: self.key.startsWith('readiness.k8s.io/')
+                - message: reserved taint prefix 'readiness.k8s.io/system/*' is not
+                    allowed
+                  rule: '!self.key.startsWith(''readiness.k8s.io/system/'')'
+                - message: reserved taint prefix 'readiness.k8s.io/core/*' is not
+                    allowed
+                  rule: '!self.key.startsWith(''readiness.k8s.io/core/'')'
+                - message: reserved taint prefix 'readiness.k8s.io/node/*' is not
+                    allowed
+                  rule: '!self.key.startsWith(''readiness.k8s.io/node/'')'
+                - message: reserved taint prefix 'readiness.k8s.io/device/*' is not
+                    allowed
+                  rule: '!self.key.startsWith(''readiness.k8s.io/device/'')'
+                - message: reserved taint prefix 'readiness.k8s.io/network/*' is not
+                    allowed
+                  rule: '!self.key.startsWith(''readiness.k8s.io/network/'')'
+                - message: reserved taint prefix 'readiness.k8s.io/storage/*' is not
+                    allowed
+                  rule: '!self.key.startsWith(''readiness.k8s.io/storage/'')'
                 - message: taint key length must be at most 253 characters
                   rule: self.key.size() <= 253
                 - message: taint value length must be at most 63 characters
diff --git a/config/samples/v1alpha1_nodereadinessrule.yaml b/config/samples/v1alpha1_nodereadinessrule.yaml
@@ -10,7 +10,7 @@ spec:
     - type: "network.kubernetes.io/CNIReady"
       requiredStatus: "True"
   taint:
-    key: "readiness.k8s.io/NetworkReady"
+    key: "readiness.k8s.io/network/not-ready"
     effect: "NoSchedule"
     value: "pending"
   enforcementMode: "bootstrap-only"
diff --git a/config/testing/kind/kind-3node-config.yaml b/config/testing/kind/kind-3node-config.yaml
@@ -18,4 +18,4 @@ nodes:
     nodeRegistration:
       kubeletExtraArgs:
         node-labels: "reserved-for=worker"
-        register-with-taints: "readiness.k8s.io/NetworkReady=pending:NoSchedule"
+        register-with-taints: "readiness.k8s.io/network/not-ready=pending:NoSchedule"
diff --git a/config/testing/kind/test-config.yaml b/config/testing/kind/test-config.yaml
@@ -9,4 +9,4 @@ nodes:
     kind: JoinConfiguration
     nodeRegistration:
       kubeletExtraArgs:
-        register-with-taints: "readiness.k8s.io/NetworkReady=pending:NoSchedule"
+        register-with-taints: "readiness.k8s.io/network/not-ready=pending:NoSchedule"
diff --git a/docs/TEST_README.md b/docs/TEST_README.md
@@ -9,7 +9,7 @@ The test demonstrates a realistic, production-aligned scenario where critical ad
 The test uses a 3-node Kind cluster:
 1.  **`nrr-test-control-plane`**: The Kubernetes control plane. The NRR controller will run here unless specifically configured.
 2.  **`nrr-test-worker` (Platform Node)**: A dedicated node for running cluster-critical addons. It is labeled `reserved-for=platform` and has a corresponding taint to repel normal application workloads. Cert-manager will run here.
-3.  **`nrr-test-worker2` (Application Node)**: A standard worker node that starts with a `readiness.k8s.io/NetworkReady=pending:NoSchedule` taint, simulating a node that is not yet ready for application traffic.
+3.  **`nrr-test-worker2` (Application Node)**: A standard worker node that starts with a `readiness.k8s.io/network/not-ready=pending:NoSchedule` taint, simulating a node that is not yet ready for application traffic.
 
 ## Running the Test
 
@@ -71,7 +71,7 @@ kubectl apply -f examples/cni-readiness/network-readiness-rule.yaml
 Check that the application worker node (`nrr-test-worker2`) has the `NetworkReady` taint.
 
 ```bash
-# The output should include 'readiness.k8s.io/NetworkReady'
+# The output should include 'readiness.k8s.io/network/not-ready'
 kubectl get node nrr-test-worker2 -o jsonpath='Taints:{"\n"}{range .spec.taints[*]}{.key}{"\n"}{end}'
 ```
 
@@ -97,7 +97,7 @@ examples/cni-readiness/apply-calico.sh
 
 2.  **Verify the taint has been removed from the application node:**
     ```bash
-    # The output should NO LONGER include 'readiness.k8s.io/NetworkReady'
+# The output should NO LONGER include 'readiness.k8s.io/network/not-ready'
     kubectl get node nrr-test-worker2 -o jsonpath='Taints:{"\n"}{range .spec.taints[*]}{.key}{"\n"}{end}'
     ```
 
diff --git a/docs/book/src/examples/cni-readiness.md b/docs/book/src/examples/cni-readiness.md
@@ -8,7 +8,7 @@ In many Kubernetes clusters, the CNI plugin runs as a DaemonSet. When a new node
 This guide demonstrates how to use the Node Readiness Controller to prevent pods from being scheduled on a node until the Container Network Interface (CNI) plugin (e.g., Calico) is fully initialized and ready.
 
 The high-level steps are:
-1.  Node is bootstrapped with a [startup taint](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) `readiness.k8s.io/NetworkReady=pending:NoSchedule` immediately upon joining.
+1.  Node is bootstrapped with a [startup taint](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) `readiness.k8s.io/network/not-ready=pending:NoSchedule` immediately upon joining.
 2.  A reporter DaemonSet is deployed to monitor the CNI's health and report it to the API server as node-condition (`projectcalico.org/CalicoReady`). 
 3. Node Readiness Controller will untaint the node only when the CNI reports it is ready.
 
@@ -85,7 +85,7 @@ subjects:
 
 ### 3. Create the Node Readiness Rule
 
-Now define the rule that enforces the requirement. This tells the controller: *"Keep the `readiness.k8s.io/NetworkReady` taint on the node until `projectcalico.org/CalicoReady` is True."*
+Now define the rule that enforces the requirement. This tells the controller: *"Keep the `readiness.k8s.io/network/not-ready` taint on the node until `projectcalico.org/CalicoReady` is True."*
 
 ```yaml
 # network-readiness-rule.yaml
@@ -101,7 +101,7 @@ spec:
   
   # The taint to manage
   taint:
-    key: "readiness.k8s.io/NetworkReady"
+    key: "readiness.k8s.io/network/not-ready"
     effect: "NoSchedule"
     value: "pending"
   
@@ -135,11 +135,11 @@ To test this, add a new node to the cluster.
 
 1.  **Check the Node Taints**:
     Immediately upon joining, the node should have the taint:
-    `readiness.k8s.io/NetworkReady=pending:NoSchedule`.
+    `readiness.k8s.io/network/not-ready=pending:NoSchedule`.
 
 2.  **Check Node Conditions**:
     Watch the node conditions. You will initially see `projectcalico.org/CalicoReady` as `False` or missing.
     Once Calico starts, the reporter will update it to `True`.
 
 3.  **Check Taint Removal**:
-    As soon as the condition becomes `True`, the Node Readiness Controller will remove the taint, and workloads will be scheduled.
+    As soon as the condition becomes `True`, the Node Readiness Controller will remove the taint, and workloads will be scheduled.
diff --git a/docs/book/src/introduction.md b/docs/book/src/introduction.md
@@ -49,7 +49,7 @@ spec:
     - type: "example.com/CNIReady"
       requiredStatus: "True"
   taint:
-    key: "readiness.k8s.io/NetworkReady"
+    key: "readiness.k8s.io/network/not-ready"
     effect: "NoSchedule"
     value: "pending"
   enforcementMode: "bootstrap-only"
@@ -72,4 +72,4 @@ See the Kubernetes community on the [community page](http://kubernetes.io/commun
 
 ## Project Status
 
-This project is currently in **alpha**. The API may change in future releases.
+This project is currently in **alpha**. The API may change in future releases.
diff --git a/docs/book/src/reference/api-spec.md b/docs/book/src/reference/api-spec.md
@@ -162,6 +162,7 @@ _Appears in:_
 | `conditions` _[ConditionRequirement](#conditionrequirement) array_ | conditions contains a list of the Node conditions that defines the specific<br />criteria that must be met for taints to be managed on the target Node.<br />The presence or status of these conditions directly triggers the application or removal of Node taints. |  | MaxItems: 32 <br />MinItems: 1 <br /> |
 | `enforcementMode` _[EnforcementMode](#enforcementmode)_ | enforcementMode specifies how the controller maintains the desired state.<br />enforcementMode is one of bootstrap-only, continuous.<br />"bootstrap-only" applies the configuration once during initial setup.<br />"continuous" ensures the state is monitored and corrected throughout the resource lifecycle. |  | Enum: [bootstrap-only continuous] <br /> |
 | `taint` _[Taint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#taint-v1-core)_ | taint defines the specific Taint (Key, Value, and Effect) to be managed<br />on Nodes that meet the defined condition criteria. |  |  |
+<br />Key must start with `readiness.k8s.io/`. Reserved core prefixes are forbidden: `readiness.k8s.io/{system,core,node,device,network,storage}/*`. Use user-space keys such as `readiness.k8s.io/<dns.subdomain>/<component>`.
 | `nodeSelector` _[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#labelselector-v1-meta)_ | nodeSelector limits the scope of this rule to a specific subset of Nodes. |  |  |
 | `dryRun` _boolean_ | dryRun when set to true, The controller will evaluate Node conditions and log intended taint modifications<br />without persisting changes to the cluster. Proposed actions are reflected in the resource status. |  |  |
 
@@ -204,4 +205,3 @@ _Appears in:_
 | `Present` | TaintStatusPresent represent the taint present on the Node.<br /> |
 | `Absent` | TaintStatusAbsent represent the taint absent on the Node.<br /> |
 
-
diff --git a/docs/book/src/user-guide/concepts.md b/docs/book/src/user-guide/concepts.md
@@ -22,6 +22,16 @@ Typical taint keys look like:
 - `readiness.k8s.io/csi.vendor.com/storage-driver-not-ready`
 - `readiness.k8s.io/<dns.subdomain>/<component-name>`
 
+Reserved core prefixes (not allowed for user rules):
+- `readiness.k8s.io/system/*`
+- `readiness.k8s.io/core/*`
+- `readiness.k8s.io/node/*`
+- `readiness.k8s.io/device/*`
+- `readiness.k8s.io/network/*` (except `readiness.k8s.io/network/not-ready`)
+- `readiness.k8s.io/storage/*` (except `readiness.k8s.io/storage/not-ready`)
+
+Use vendor/user-space paths under `readiness.k8s.io/*` that include a DNS-style component to avoid conflicts.
+
 The segment after `readiness.k8s.io/` should describe the dependency or subsystem whose readiness is being guarded (for example, a CNI plugin, storage backend, or security agent). Treat this domain as reserved for the controller and closely related components, and avoid reusing it for unrelated taints.
 
 ## Enforcement Modes
diff --git a/docs/book/src/user-guide/getting-started.md b/docs/book/src/user-guide/getting-started.md
@@ -135,8 +135,18 @@ taint:
 taint:
   key: "network-ready"              # Missing prefix
   key: "node.kubernetes.io/ready"   # Wrong prefix
+  key: "readiness.k8s.io/system/foo"   # Reserved prefix
+  key: "readiness.k8s.io/network/bar"  # Reserved prefix (except readiness.k8s.io/network/not-ready)
 ```
 
+Reserved core prefixes under `readiness.k8s.io/*` are forbidden to avoid conflicts with controller-owned keys:
+- `readiness.k8s.io/system/*`
+- `readiness.k8s.io/core/*`
+- `readiness.k8s.io/node/*`
+- `readiness.k8s.io/device/*`
+- `readiness.k8s.io/network/*` (except `readiness.k8s.io/network/not-ready`)
+- `readiness.k8s.io/storage/*`
+
 
 ## Testing with Dry Run
 
diff --git a/examples/cni-readiness/README.md b/examples/cni-readiness/README.md
@@ -3,7 +3,7 @@
 This example demonstrates how to use the Node Readiness Controller to ensure nodes are only marked ready for workloads after the CNI (Calico) has fully initialized.
 
 ### How it works:
-1. Nodes join with a `readiness.k8s.io/NetworkReady=pending:NoSchedule` taint.
+1. Nodes join with a `readiness.k8s.io/network/not-ready=pending:NoSchedule` taint.
 2. A lightweight DaemonSet (`cni-reporter-ds.yaml`)
    monitors Calico's health endpoint (`localhost:9099/readiness`) and updates a
    node condition `projectcalico.org/CalicoReady`.
diff --git a/examples/cni-readiness/network-readiness-dryrun-rule.yaml b/examples/cni-readiness/network-readiness-dryrun-rule.yaml
@@ -8,7 +8,7 @@ spec:
     - type: "projectcalico.org/CalicoReady"
       requiredStatus: "True"
   taint:
-    key: "readiness.k8s.io/NetworkReady"
+    key: "readiness.k8s.io/network/not-ready"
     effect: "NoSchedule"
     value: "pending"
   enforcementMode: "bootstrap-only"
diff --git a/examples/cni-readiness/network-readiness-rule.yaml b/examples/cni-readiness/network-readiness-rule.yaml
@@ -7,7 +7,7 @@ spec:
     - type: "projectcalico.org/CalicoReady"
       requiredStatus: "True"
   taint:
-    key: "readiness.k8s.io/NetworkReady"
+    key: "readiness.k8s.io/network/not-ready"
     effect: "NoSchedule"
     value: "pending"
   enforcementMode: "continuous"
diff --git a/internal/webhook/nodereadinessgaterule_webhook.go b/internal/webhook/nodereadinessgaterule_webhook.go
@@ -19,6 +19,7 @@ package webhook
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -95,6 +96,31 @@ func (w *NodeReadinessRuleWebhook) validateSpec(spec readinessv1alpha1.NodeReadi
 	if spec.Taint.Key == "" {
 		allErrs = append(allErrs, field.Required(taintField.Child("key"), "taint key cannot be empty"))
 	}
+	// Enforce readiness.k8s.io/ prefix and forbid reserved subpaths
+	if spec.Taint.Key != "" {
+		if !strings.HasPrefix(spec.Taint.Key, "readiness.k8s.io/") {
+			allErrs = append(allErrs, field.Invalid(taintField.Child("key"), spec.Taint.Key, "taint key must start with 'readiness.k8s.io/'"))
+		} else {
+			reserved := []string{
+				"readiness.k8s.io/system/",
+				"readiness.k8s.io/core/",
+				"readiness.k8s.io/node/",
+				"readiness.k8s.io/device/",
+				"readiness.k8s.io/network/",
+				"readiness.k8s.io/storage/",
+			}
+			for _, p := range reserved {
+				if strings.HasPrefix(spec.Taint.Key, p) {
+					if (p == "readiness.k8s.io/network/" && spec.Taint.Key == "readiness.k8s.io/network/not-ready") ||
+						(p == "readiness.k8s.io/storage/" && spec.Taint.Key == "readiness.k8s.io/storage/not-ready") {
+						continue
+					}
+					allErrs = append(allErrs, field.Invalid(taintField.Child("key"), spec.Taint.Key, fmt.Sprintf("reserved taint prefix '%s*' is not allowed", p)))
+					break
+				}
+			}
+		}
+	}
 	if spec.Taint.Effect == "" {
 		allErrs = append(allErrs, field.Required(taintField.Child("effect"), "taint effect cannot be empty"))
 	}
diff --git a/internal/webhook/nodereadinessgaterule_webhook_test.go b/internal/webhook/nodereadinessgaterule_webhook_test.go
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
diff --git a/test/e2e/testdata/continuous-rule.yaml b/test/e2e/testdata/continuous-rule.yaml
