diff --git a/.github/workflows/garage-contract.yml b/.github/workflows/garage-contract.yml index 8c961941..48ab97a1 100644 --- a/.github/workflows/garage-contract.yml +++ b/.github/workflows/garage-contract.yml @@ -8,6 +8,11 @@ name: Garage Contract # rules and the status/health fields. The released versions gate pull # requests; the main-v2 entry is informational (an upstream change there # should be triaged, not block unrelated work). +# +# The replication contract runs the GarageReplication Job script in the +# operator's pinned rclone image with Docker, as the Job's non-root user on a +# read-only root filesystem: copy/sync semantics, the safety guards and their +# exit codes, the failure classification and the termination message. on: push: @@ -70,5 +75,16 @@ jobs: } >> "$GITHUB_ENV" echo "### Garage contract: \`${image}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Pull the replication image + run: | + set -euo pipefail + image="$(sed -n 's/^const DefaultReplicationImage = "\(.*\)"$/\1/p' internal/controller/garagereplication_job.go)" + test -n "${image}" + docker pull --quiet "${image}" + echo "### Replication image: \`${image}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Run contract tests + env: + # Fail rather than skip when the container runtime is missing. + REPLICATION_CONTRACT_REQUIRED: "true" run: make test-contract diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 223a2995..166bee10 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -138,8 +138,8 @@ jobs: grep -q -- 'port: 9443' <<<"$rendered" grep -q -- '--cosi-namespace=shadows' <<<"$rendered" webhook_render="$(sed -n '/kind: MutatingWebhookConfiguration/,$p' <<<"$rendered")" - test "$(grep -c '^ namespaceSelector:' <<<"$webhook_render")" -eq 14 - test "$(grep -c ' - "shadows"' <<<"$webhook_render")" -eq 14 + test "$(grep -c '^ namespaceSelector:' <<<"$webhook_render")" -eq 15 + test "$(grep -c ' - "shadows"' <<<"$webhook_render")" -eq 15 cosi_rbac="$(helm template scoped charts/${{ env.CHART_NAME }} \ --namespace operators \ diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 9e66daf0..5578acf3 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -74,7 +74,13 @@ jobs: kind_config: "" - name: layout runner: ubuntu-latest - filter: "!(gateway || manager || webhooks || external-gateway || dual-version || manual-mode || management-handle || node-local-pools || factor-migration || garage-versions || k8s-discovery)" + filter: "!(gateway || manager || webhooks || external-gateway || dual-version || manual-mode || management-handle || node-local-pools || factor-migration || garage-versions || k8s-discovery || replication)" + kind_config: "" + # replication runs two GarageClusters and real rclone Jobs + # (GarageReplication) with the operator's --enable-replication. + - name: replication + runner: ubuntu-latest + filter: "replication" kind_config: "" - name: node-local-pools runner: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index abf85f39..fd9bc1c4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,7 @@ UPSTREAM CODEBASE ../garage | `GarageNode` | `gn` | Node layout (zone, capacity, gateway) | | `GarageAdminToken` | `gat` | Admin API tokens | | `GarageReferenceGrant` | `grg` | Cross-namespace access grants (v1beta1+) | +| `GarageReplication` | `grepl` | Cross-cluster object replication/migration (rclone Jobs, `--enable-replication`) | ### Development Commands @@ -456,6 +457,60 @@ and a completed `LegacySTSMigrated` condition short-circuits future runs. --- +## Cross-cluster replication (GarageReplication) + +Copies objects between two **independent** Garage clusters through S3, for +replication and migration. It is **not** a backup (no snapshots, no Archive +mode; maintainer decision on #580) and not Garage federation. Design record: +`docs/design/2026-11-09-garage-replication-design.md`; user guide: +`docs/how-to/replication.md`. + +- Controller behind `--enable-replication` (`ENABLE_REPLICATION`, chart + `replication.enabled`, default off). It gates the controller, the Job + informer and the chart's `batch/jobs` + status RBAC; the webhook is always + registered and warns while the flag is off + (`GarageReplicationValidator.ControllerEnabled`). +- The API must be correct **without the webhook**: formats, enums, list keys + and the mode/mirror pairing are schema rules, pinned against a real API + server by `internal/controller/garagereplication_schema_test.go`. The webhook + only adds context checks (reference grant, destination exclusivity across + objects, implicit-namespace self-loop, cron). +- CEL on the shared `ClusterReference` (unbounded strings) blows the rule cost + budget; envtest's CRD install catches it. Keep such checks in Go. + Kubernetes 1.25 (the chart's smoke job) is stricter still: it costs CEL + string-extension functions such as `lowerAscii()` as unbounded, which the + 1.36 envtest does not catch. +- `buckets[]` and `status.buckets` are list-maps keyed by `destination`; Jobs + carry the destination in an annotation and the controller keys everything + on it. +- Credentials Secrets must carry the label + `garage.rajsingh.info/replication-credentials: "true"`, GarageKey-generated + ones included (maintainer, #580: otherwise any GarageKey in the namespace can + be copied out), checked by the controller on every reconcile + (`Ready=False/CredentialsNotAllowed`): otherwise `create garagereplications` + would amount to `get secrets`. CA bundles come from ConfigMaps only. +- Destination exclusivity is cluster-wide, first come first served; the + webhook compares spec keys, the controller also resolved URLs + (`Ready=False/DestinationConflict`). Errors must not name an object in + another namespace. +- Holds (`status.buckets[].hold`) are released only by `trigger-sync`, never by + a spec edit. `AwaitingTrigger` gates the first Mirror run of a bucket whose + `lastMode` is not Mirror (new, or switched from Copy) and is dropped when the + mode goes back to Copy. +- The Job script classifies failures by rclone exit code first (3 missing + source bucket, 5 retryable/unreachable, 7 in Mirror = delete cap; measured + against rclone 1.75.2), log text only for exit 1 (auth vs `NoSuchBucket`). + rclone reads a bare size as KiB, so the controller appends `B` + (`rcloneSize`). Re-measure with the contract test when bumping the image + (Renovate does not automerge it). +- Contract test: `test/contract/replication_contract_test.go` runs the Job's + container with docker/podman (non-root, read-only rootfs, tmpfs work dir) + and reads the 0600 termination log through a root container. CI sets + `REPLICATION_CONTRACT_REQUIRED=true`. Kind e2e: `test/e2e/replication_e2e_test.go`, + shard `replication`. + +--- + ## Configuration Reference ### GarageCluster Options diff --git a/PROJECT b/PROJECT index 1ac4f1cc..3f826a57 100644 --- a/PROJECT +++ b/PROJECT @@ -54,4 +54,13 @@ resources: kind: GarageAdminToken path: github.com/rajsinghtech/garage-operator/api/v1beta1 version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: rajsingh.info + group: garage + kind: GarageReplication + path: github.com/rajsinghtech/garage-operator/api/v1beta1 + version: v1beta1 version: "3" diff --git a/api/v1beta1/garagekey_webhook.go b/api/v1beta1/garagekey_webhook.go index e2897dee..0ea68c81 100644 --- a/api/v1beta1/garagekey_webhook.go +++ b/api/v1beta1/garagekey_webhook.go @@ -296,7 +296,7 @@ func validateImportKey(ik *ImportKeyConfig) error { return fmt.Errorf("importKey.secretRef.name is required") } for field, key := range map[string]string{ - "accessKeyIdKey": ik.AccessKeyIDKey, "secretAccessKeyKey": ik.SecretAccessKeyKey, + accessKeyIDKeyField: ik.AccessKeyIDKey, secretAccessKeyKeyField: ik.SecretAccessKeyKey, } { if key != "" { if problems := utilvalidation.IsConfigMapKey(key); len(problems) > 0 { @@ -429,8 +429,8 @@ func validateSecretTemplate(template *SecretTemplate) error { includeWebsiteURL := template.IncludeWebsiteURL != nil && *template.IncludeWebsiteURL includeCredentialsFile := template.IncludeCredentialsFile != nil && *template.IncludeCredentialsFile keys := []generatedKey{ - {field: "accessKeyIdKey", value: defaultString(template.AccessKeyIDKey, defaultAccessKeyIDSecretDataKey), enabled: true}, - {field: "secretAccessKeyKey", value: defaultString(template.SecretAccessKeyKey, defaultSecretAccessKeySecretDataKey), enabled: true}, + {field: accessKeyIDKeyField, value: defaultString(template.AccessKeyIDKey, defaultAccessKeyIDSecretDataKey), enabled: true}, + {field: secretAccessKeyKeyField, value: defaultString(template.SecretAccessKeyKey, defaultSecretAccessKeySecretDataKey), enabled: true}, {field: "endpointKey", value: defaultString(template.EndpointKey, "endpoint"), enabled: includeEndpoint}, {field: "hostKey", value: defaultString(template.HostKey, "host"), enabled: includeEndpoint}, {field: "schemeKey", value: defaultString(template.SchemeKey, "scheme"), enabled: includeEndpoint}, diff --git a/api/v1beta1/garagereferencegrant_types.go b/api/v1beta1/garagereferencegrant_types.go index 7ec1c7a1..8887b4e7 100644 --- a/api/v1beta1/garagereferencegrant_types.go +++ b/api/v1beta1/garagereferencegrant_types.go @@ -44,7 +44,8 @@ type ReferenceGrantFrom struct { // Kind is the resource kind allowed to make cross-namespace references. // GarageAdminToken remains in the schema for compatibility, but the static // credential path is namespace-local and does not accept cross-namespace grants. - // +kubebuilder:validation:Enum=GarageKey;GarageBucket;GarageAdminToken + // GarageReplication may reference a GarageCluster in another namespace. + // +kubebuilder:validation:Enum=GarageKey;GarageBucket;GarageAdminToken;GarageReplication // +required Kind string `json:"kind"` diff --git a/api/v1beta1/garagereplication_types.go b/api/v1beta1/garagereplication_types.go new file mode 100644 index 00000000..27ca4ae4 --- /dev/null +++ b/api/v1beta1/garagereplication_types.go @@ -0,0 +1,676 @@ +/* +Copyright 2026 Raj Singh. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GarageReplicationSpec defines the desired state of GarageReplication. +// +// GarageReplication copies objects between two independent Garage clusters +// through their S3 APIs, for replication and migration. It is not a backup: +// there are no point-in-time snapshots and no restore, and in mode Mirror a +// deletion or an overwrite on the source reaches the destination on the next +// run. Each run executes one Kubernetes Job per bucket in the +// GarageReplication's namespace, so the Jobs need network access to both S3 +// endpoints. Only objects (and their user metadata) are copied: buckets, keys, +// permissions, quotas, website and lifecycle configuration are not. Deleting a +// GarageReplication never deletes object data on either side. +// +kubebuilder:validation:XValidation:rule="has(self.mirror) == (has(self.mode) && self.mode == 'Mirror')",message="mirror must be set if and only if mode is Mirror" +// +kubebuilder:validation:XValidation:rule="!(has(self.source.endpoint) && has(self.destination.endpoint) && self.source.endpoint.url == self.destination.endpoint.url) || self.buckets.all(b, b.source != b.destination)",message="a bucket cannot be replicated onto itself: source and destination are the same endpoint" +type GarageReplicationSpec struct { + // Source is the cluster objects are read from. Its credentials only need + // read permission on the source buckets. + // +required + Source ReplicationEndpoint `json:"source"` + + // Destination is the cluster objects are written to. Its credentials need + // read and write permission on the destination buckets. Destination + // buckets must already exist: replication never creates buckets. A + // destination bucket can be written by only one GarageReplication. + // +required + Destination ReplicationEndpoint `json:"destination"` + + // Buckets lists the buckets to replicate, keyed by destination bucket. + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=50 + // +listType=map + // +listMapKey=destination + // +required + Buckets []ReplicationBucket `json:"buckets"` + + // Schedule is a cron expression (standard 5-field syntax, descriptors + // such as @hourly, optional CRON_TZ= prefix; UTC otherwise). Without a + // schedule, runs happen only through the garage.rajsingh.info/trigger-sync + // annotation (one-shot mode, e.g. for migrations). + // +kubebuilder:validation:MaxLength=128 + // +optional + Schedule string `json:"schedule,omitempty"` + + // Suspend stops starting new runs. A run already in progress finishes; + // delete its Jobs to stop it immediately. + // +optional + Suspend bool `json:"suspend,omitempty"` + + // Mode controls whether changes on the source can remove data on the + // destination. + // - Copy: copy new and changed objects. Source deletions are not + // propagated. Changed objects are overwritten on the destination. + // - Mirror: the destination reflects the source, including deletions, + // capped by mirror.maxDeletes per bucket and run. The first run waits + // for the trigger-sync annotation. + // +kubebuilder:validation:Enum=Copy;Mirror + // +kubebuilder:default=Copy + // +optional + Mode ReplicationMode `json:"mode,omitempty"` + + // Mirror configures mode Mirror. Required when mode is Mirror and + // rejected otherwise. + // +optional + Mirror *ReplicationMirrorConfig `json:"mirror,omitempty"` + + // Verification runs a comparison of source and destination after each + // successful copy. Check compares sizes and MD5 hashes (size only for + // multipart objects without a stored MD5) and fails the run on any + // difference. + // +kubebuilder:validation:Enum=None;Check + // +kubebuilder:default=None + // +optional + Verification ReplicationVerification `json:"verification,omitempty"` + + // Transfer tunes the copy. + // +optional + Transfer *ReplicationTransferConfig `json:"transfer,omitempty"` + + // Retry bounds retries and duration of each per-bucket Job. + // +optional + Retry *ReplicationRetryConfig `json:"retry,omitempty"` + + // Resources for the replication container. + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + + // PodTemplate customizes the replication Pods. + // +optional + PodTemplate *ReplicationPodTemplate `json:"podTemplate,omitempty"` + + // SuccessfulJobsHistoryLimit is the number of finished successful Jobs to + // keep. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=50 + // +kubebuilder:default=1 + // +optional + SuccessfulJobsHistoryLimit *int32 `json:"successfulJobsHistoryLimit,omitempty"` + + // FailedJobsHistoryLimit is the number of finished failed Jobs to keep. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=50 + // +kubebuilder:default=3 + // +optional + FailedJobsHistoryLimit *int32 `json:"failedJobsHistoryLimit,omitempty"` +} + +// ReplicationMode selects how deletions and overwrites propagate. +type ReplicationMode string + +const ( + // ReplicationModeCopy never deletes destination objects. + ReplicationModeCopy ReplicationMode = "Copy" + // ReplicationModeMirror propagates source deletions to the destination. + ReplicationModeMirror ReplicationMode = "Mirror" +) + +// ReplicationVerification selects the post-copy verification. +type ReplicationVerification string + +const ( + // ReplicationVerificationNone skips verification. + ReplicationVerificationNone ReplicationVerification = "None" + // ReplicationVerificationCheck compares source and destination after the copy. + ReplicationVerificationCheck ReplicationVerification = "Check" +) + +// LabelReplicationCredentials opts a Secret in to being used as +// GarageReplication credentials. It is required on every credentials Secret, +// including the ones generated for a GarageKey (set it through +// GarageKey.spec.secretTemplate.labels). +const LabelReplicationCredentials = AnnotationPrefix + "replication-credentials" + +// LabelReplicationCredentialsValue is the only value of +// LabelReplicationCredentials that opts a Secret in. +const LabelReplicationCredentialsValue = "true" + +// ReplicationEndpoint identifies one side of a replication. Exactly one of +// clusterRef or endpoint must be set. +// +kubebuilder:validation:XValidation:rule="has(self.clusterRef) != has(self.endpoint)",message="exactly one of clusterRef or endpoint must be set" +// +kubebuilder:validation:XValidation:rule="!has(self.clusterRef) || !has(self.clusterRef.kubeConfigSecretRef)",message="clusterRef.kubeConfigSecretRef is not supported" +type ReplicationEndpoint struct { + // ClusterRef references a GarageCluster (operator-managed or a + // management handle). The S3 endpoint and region are resolved from it. + // A cross-namespace reference requires a GarageReferenceGrant in the + // cluster's namespace that allows kind GarageReplication. + // +optional + ClusterRef *ClusterReference `json:"clusterRef,omitempty"` + + // Endpoint is an S3 endpoint of any Garage cluster, inside or outside + // Kubernetes. + // +optional + Endpoint *ReplicationS3Endpoint `json:"endpoint,omitempty"` + + // CredentialsSecretRef references a Secret in the GarageReplication's + // namespace holding an S3 access key. The Secret must carry the label + // garage.rajsingh.info/replication-credentials: "true" (for a GarageKey, + // set it in spec.secretTemplate.labels). The access key ID is sent to the + // endpoint, so without this opt-in the right to create a GarageReplication + // would amount to reading any Secret, or using any other application's + // key. The default data keys match the Secret generated for a GarageKey. + // +required + CredentialsSecretRef ReplicationCredentialsRef `json:"credentialsSecretRef"` +} + +// ReplicationS3Endpoint is an explicit S3 endpoint. +type ReplicationS3Endpoint struct { + // URL of the S3 API: scheme and host (with an optional port) only, e.g. + // https://s3.example.com or http://garage.ns.svc:3900. Buckets are + // addressed path-style below it, so it has no path, query, fragment or + // credentials. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=2048 + // +kubebuilder:validation:Pattern=`^https?://[^\s/?#@]+/?$` + // +required + URL string `json:"url"` + + // Region is the Garage s3_region used for request signing. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=128 + // +kubebuilder:validation:Pattern=`^[A-Za-z0-9_.-]+$` + // +kubebuilder:default=garage + // +optional + Region string `json:"region,omitempty"` + + // CABundleRef references a ConfigMap holding a PEM CA bundle used, in + // addition to the system trust store, to verify the endpoint's TLS + // certificate. + // +optional + CABundleRef *ReplicationCABundleRef `json:"caBundleRef,omitempty"` +} + +// ReplicationCABundleRef selects a key of a ConfigMap in the +// GarageReplication's namespace. A CA bundle is public, so it is read from a +// ConfigMap only. +type ReplicationCABundleRef struct { + // Name of the ConfigMap. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + // +required + Name string `json:"name"` + + // Key of the ConfigMap holding the PEM bundle. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[-._a-zA-Z0-9]+$` + // +kubebuilder:default="ca.crt" + // +optional + Key string `json:"key,omitempty"` +} + +// ReplicationCredentialsRef references a Secret holding an S3 access key. +// +kubebuilder:validation:XValidation:rule="(has(self.accessKeyIdKey) ? self.accessKeyIdKey : 'access-key-id') != (has(self.secretAccessKeyKey) ? self.secretAccessKeyKey : 'secret-access-key')",message="accessKeyIdKey and secretAccessKeyKey must be different" +type ReplicationCredentialsRef struct { + // Name of the Secret in the GarageReplication's namespace. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + // +required + Name string `json:"name"` + + // AccessKeyIDKey is the data key holding the access key ID. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[-._a-zA-Z0-9]+$` + // +kubebuilder:default="access-key-id" + // +optional + AccessKeyIDKey string `json:"accessKeyIdKey,omitempty"` + + // SecretAccessKeyKey is the data key holding the secret access key. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[-._a-zA-Z0-9]+$` + // +kubebuilder:default="secret-access-key" + // +optional + SecretAccessKeyKey string `json:"secretAccessKeyKey,omitempty"` +} + +// ReplicationBucket maps a source bucket to a destination bucket. +type ReplicationBucket struct { + // Source is the bucket name (global alias) on the source cluster. + // +kubebuilder:validation:MinLength=3 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$` + // +required + Source string `json:"source"` + + // Destination is the bucket name (global alias) on the destination + // cluster. It is the list key, so destinations are unique. + // +kubebuilder:validation:MinLength=3 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$` + // +required + Destination string `json:"destination"` +} + +// ReplicationMirrorConfig configures mode Mirror. +type ReplicationMirrorConfig struct { + // MaxDeletes caps the number of destination objects one run may delete + // per bucket. Once reached the run stops and fails with reason + // MaxDeletesExceeded; deletions done before that point are kept. + // +kubebuilder:validation:Minimum=1 + // +required + MaxDeletes int64 `json:"maxDeletes"` + + // MaxDeleteSize caps the total size of the destination objects one run + // may delete per bucket, in bytes with an optional K, M, G or T suffix + // (binary multiples: KiB, MiB, GiB, TiB), e.g. "10G". Once reached the run + // stops and fails with reason MaxDeletesExceeded, like maxDeletes. Empty + // means no size cap. + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:Pattern=`^[0-9]+(\.[0-9]+)?[KMGTkmgt]?$` + // +optional + MaxDeleteSize string `json:"maxDeleteSize,omitempty"` +} + +// ReplicationTransferConfig tunes the copy engine. +type ReplicationTransferConfig struct { + // Concurrency is the number of objects transferred in parallel per bucket. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=64 + // +kubebuilder:default=4 + // +optional + Concurrency int32 `json:"concurrency,omitempty"` + + // Checkers is the number of objects compared in parallel per bucket. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=128 + // +kubebuilder:default=8 + // +optional + Checkers int32 `json:"checkers,omitempty"` + + // BandwidthLimit caps the transfer rate of each per-bucket Job in bytes + // per second, with an optional K, M, G suffix (binary multiples: KiB/s, + // MiB/s, GiB/s), e.g. "50M". Empty means unlimited. + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:Pattern=`^[0-9]+(\.[0-9]+)?[KMGkmg]?$` + // +optional + BandwidthLimit string `json:"bandwidthLimit,omitempty"` + + // MaxConcurrentBuckets is the number of buckets replicated in parallel. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=50 + // +kubebuilder:default=1 + // +optional + MaxConcurrentBuckets int32 `json:"maxConcurrentBuckets,omitempty"` +} + +// ReplicationRetryConfig bounds retries and duration of each Job. +type ReplicationRetryConfig struct { + // BackoffLimit is the number of Pod retries of a per-bucket Job. Each + // retry restarts the bucket's comparison; already copied objects are + // skipped. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=10 + // +kubebuilder:default=2 + // +optional + BackoffLimit *int32 `json:"backoffLimit,omitempty"` + + // ActiveDeadlineSeconds bounds the duration of a per-bucket Job. + // +kubebuilder:validation:Minimum=60 + // +kubebuilder:default=21600 + // +optional + ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"` +} + +// ReplicationPodTemplate customizes replication Pods. +type ReplicationPodTemplate struct { + // Labels added to the Pods. + // +kubebuilder:validation:MaxProperties=64 + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // Annotations added to the Pods. + // +kubebuilder:validation:MaxProperties=64 + // +optional + Annotations map[string]string `json:"annotations,omitempty"` + + // NodeSelector for the Pods. + // +kubebuilder:validation:MaxProperties=64 + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Tolerations for the Pods. + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + + // Affinity for the Pods. + // +optional + Affinity *corev1.Affinity `json:"affinity,omitempty"` + + // PriorityClassName for the Pods. + // +kubebuilder:validation:MaxLength=253 + // +optional + PriorityClassName string `json:"priorityClassName,omitempty"` +} + +// ReplicationPhase summarizes the state of a GarageReplication. +// +kubebuilder:validation:Enum=Pending;Idle;Syncing;Suspended;Failed +type ReplicationPhase string + +const ( + // ReplicationPhasePending: the configuration is not usable yet. + ReplicationPhasePending ReplicationPhase = "Pending" + // ReplicationPhaseIdle: no run in progress and the last run succeeded. + ReplicationPhaseIdle ReplicationPhase = "Idle" + // ReplicationPhaseSyncing: a run is in progress. + ReplicationPhaseSyncing ReplicationPhase = "Syncing" + // ReplicationPhaseSuspended: spec.suspend is set. + ReplicationPhaseSuspended ReplicationPhase = "Suspended" + // ReplicationPhaseFailed: the last run failed. + ReplicationPhaseFailed ReplicationPhase = "Failed" +) + +// ReplicationRunReason is why a run started. +// +kubebuilder:validation:Enum=Initial;Scheduled;Triggered +type ReplicationRunReason string + +const ( + // ReplicationRunInitial is the run started when a Copy replication is created. + ReplicationRunInitial ReplicationRunReason = "Initial" + // ReplicationRunScheduled is a run started by spec.schedule. + ReplicationRunScheduled ReplicationRunReason = "Scheduled" + // ReplicationRunTriggered is a run started by the trigger-sync annotation. + ReplicationRunTriggered ReplicationRunReason = "Triggered" +) + +// ReplicationResult is the outcome of a run or of one bucket's Job. +// +kubebuilder:validation:Enum=Succeeded;Failed;Cancelled +type ReplicationResult string + +const ( + // ReplicationResultSucceeded: the copy (and verification) succeeded. + ReplicationResultSucceeded ReplicationResult = "Succeeded" + // ReplicationResultFailed: the copy or verification failed. + ReplicationResultFailed ReplicationResult = "Failed" + // ReplicationResultCancelled: the Job was deleted or cancelled before it finished. + ReplicationResultCancelled ReplicationResult = "Cancelled" +) + +// ReplicationVerificationResult is the outcome of verification. +// +kubebuilder:validation:Enum=Passed;Failed +type ReplicationVerificationResult string + +const ( + // ReplicationVerificationPassed: no difference found. + ReplicationVerificationPassed ReplicationVerificationResult = "Passed" + // ReplicationVerificationFailed: differences found. + ReplicationVerificationFailed ReplicationVerificationResult = "Failed" +) + +// ReplicationHold is why a bucket is held. Only the trigger-sync annotation +// releases a hold; editing the spec does not. +// +kubebuilder:validation:Enum=SourceEmpty;MaxDeletesExceeded;AwaitingTrigger +type ReplicationHold string + +const ( + // ReplicationHoldSourceEmpty: the source bucket listed empty while the + // destination was not, so Mirror refused to empty the destination. + ReplicationHoldSourceEmpty ReplicationHold = "SourceEmpty" + // ReplicationHoldMaxDeletesExceeded: a Mirror run reached mirror.maxDeletes + // or mirror.maxDeleteSize. + ReplicationHoldMaxDeletesExceeded ReplicationHold = "MaxDeletesExceeded" + // ReplicationHoldAwaitingTrigger: a Mirror bucket that has not run in + // Mirror mode yet (new, or switched from Copy) waits for a trigger-sync, + // so deletions never start without a person asking. It is dropped when + // the mode goes back to Copy, which never deletes. + ReplicationHoldAwaitingTrigger ReplicationHold = "AwaitingTrigger" +) + +// ReplicationFailureReason is a fixed reason for a failed Job. Engine output +// (which can contain object keys) stays in the Pod logs. +// +kubebuilder:validation:Enum=AuthFailed;BucketNotFound;EndpointUnreachable;SourceEmpty;MaxDeletesExceeded;VerificationFailed;DeadlineExceeded;TransferErrors;Unknown +type ReplicationFailureReason string + +const ( + ReplicationFailureAuthFailed ReplicationFailureReason = "AuthFailed" + ReplicationFailureBucketNotFound ReplicationFailureReason = "BucketNotFound" + ReplicationFailureEndpointUnreachable ReplicationFailureReason = "EndpointUnreachable" + ReplicationFailureSourceEmpty ReplicationFailureReason = "SourceEmpty" + ReplicationFailureMaxDeletesExceeded ReplicationFailureReason = "MaxDeletesExceeded" + ReplicationFailureVerificationFailed ReplicationFailureReason = "VerificationFailed" + ReplicationFailureDeadlineExceeded ReplicationFailureReason = "DeadlineExceeded" + ReplicationFailureTransferErrors ReplicationFailureReason = "TransferErrors" + ReplicationFailureUnknown ReplicationFailureReason = "Unknown" +) + +// GarageReplicationStatus defines the observed state of GarageReplication. +type GarageReplicationStatus struct { + // Phase summarizes the state. + // +optional + Phase ReplicationPhase `json:"phase,omitempty"` + + // ObservedGeneration is the last observed generation. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // CurrentRun describes the run in progress. + // +optional + CurrentRun *ReplicationRun `json:"currentRun,omitempty"` + + // LastRun describes the most recent finished run. + // +optional + LastRun *ReplicationRun `json:"lastRun,omitempty"` + + // LastScheduleTime is the start time of the most recent run. + // +optional + LastScheduleTime *metav1.Time `json:"lastScheduleTime,omitempty"` + + // LastSyncTime is when the most recent run finished, whatever its result. + // +optional + LastSyncTime *metav1.Time `json:"lastSyncTime,omitempty"` + + // LastSuccessfulSyncTime is when the most recent run in which every + // bucket succeeded finished. + // +optional + LastSuccessfulSyncTime *metav1.Time `json:"lastSuccessfulSyncTime,omitempty"` + + // NextSyncTime is when the next scheduled run starts. Unset without a + // schedule or while suspended. + // +optional + NextSyncTime *metav1.Time `json:"nextSyncTime,omitempty"` + + // Buckets reports per-bucket results, keyed by destination bucket. + // +listType=map + // +listMapKey=destination + // +optional + Buckets []ReplicationBucketStatus `json:"buckets,omitempty"` + + // LastOperation records the outcome of the last trigger-sync annotation. + // +optional + LastOperation *LastOperationStatus `json:"lastOperation,omitempty"` + + // Conditions: Ready (configuration resolved; reasons include + // CredentialsNotAllowed and DestinationConflict), Synced (every bucket + // succeeded in the last run on the current configuration), Suspended. + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// ReplicationRun describes a run. +type ReplicationRun struct { + // ID identifies the run; it is part of the Job names. + // +required + ID string `json:"id"` + + // Reason is why the run started. + // +required + Reason ReplicationRunReason `json:"reason"` + + // StartTime is when the run started. + // +required + StartTime metav1.Time `json:"startTime"` + + // Trigger is the trigger-sync annotation value that started the run. + // +optional + Trigger string `json:"trigger,omitempty"` + + // SpecHash identifies the configuration the run started with. + // +optional + SpecHash string `json:"specHash,omitempty"` + + // Buckets lists the destination buckets taking part in the run. + // +listType=set + // +optional + Buckets []string `json:"buckets,omitempty"` + + // CompletionTime is when the run finished. + // +optional + CompletionTime *metav1.Time `json:"completionTime,omitempty"` + + // Result is Succeeded when every bucket of the run succeeded, else Failed. + // +optional + Result ReplicationResult `json:"result,omitempty"` +} + +// ReplicationBucketStatus reports the last result for one bucket. +type ReplicationBucketStatus struct { + // Destination bucket; the list key. + // +required + Destination string `json:"destination"` + + // Source bucket. + // +required + Source string `json:"source"` + + // ActiveJob is the Job currently replicating this bucket. + // +optional + ActiveJob string `json:"activeJob,omitempty"` + + // LastJob is the last finished Job recorded for this bucket. + // +optional + LastJob string `json:"lastJob,omitempty"` + + // LastRunID is the run of LastJob. + // +optional + LastRunID string `json:"lastRunId,omitempty"` + + // LastResult is the outcome of LastJob. + // +optional + LastResult ReplicationResult `json:"lastResult,omitempty"` + + // LastMode is the mode LastJob ran in. A Mirror bucket whose last Job + // did not run in Mirror mode waits for a trigger-sync (hold + // AwaitingTrigger) before its first Mirror run. + // +optional + LastMode ReplicationMode `json:"lastMode,omitempty"` + + // LastCompletionTime is when LastJob finished. + // +optional + LastCompletionTime *metav1.Time `json:"lastCompletionTime,omitempty"` + + // LastSuccessfulTime is when the last successful Job for this bucket finished. + // +optional + LastSuccessfulTime *metav1.Time `json:"lastSuccessfulTime,omitempty"` + + // LastStatistics are the copy engine's counters for LastJob. Unset when + // the engine reported none (e.g. the Pod was killed). + // +optional + LastStatistics *ReplicationStatistics `json:"lastStatistics,omitempty"` + + // LastVerification is the verification outcome when verification ran. + // +optional + LastVerification ReplicationVerificationResult `json:"lastVerification,omitempty"` + + // Hold is set when this bucket must not run again without a person + // asking. Scheduled runs skip a held bucket. Only the trigger-sync + // annotation releases it; editing the spec does not. + // +optional + Hold ReplicationHold `json:"hold,omitempty"` + + // LastError is a fixed reason for a failed LastJob. + // +optional + LastError ReplicationFailureReason `json:"lastError,omitempty"` +} + +// ReplicationStatistics are counters reported by the copy engine for one Job. +type ReplicationStatistics struct { + // Transfers is the number of objects written to the destination. + Transfers int64 `json:"transfers"` + // Bytes is the number of bytes written to the destination. + Bytes int64 `json:"bytes"` + // Checks is the number of objects compared and found up to date. + Checks int64 `json:"checks"` + // Deletes is the number of destination objects deleted (Mirror). + Deletes int64 `json:"deletes"` + // Errors is the number of errors. + Errors int64 `json:"errors"` + // ElapsedSeconds is the duration of the copy. + ElapsedSeconds int64 `json:"elapsedSeconds"` + // Differences is the number of differences found by verification. + // +optional + Differences *int64 `json:"differences,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=grepl +// +kubebuilder:printcolumn:name="Mode",type="string",JSONPath=".spec.mode" +// +kubebuilder:printcolumn:name="Schedule",type="string",JSONPath=".spec.schedule" +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Last Success",type="date",JSONPath=".status.lastSuccessfulSyncTime" +// +kubebuilder:printcolumn:name="Next Sync",type="string",JSONPath=".status.nextSyncTime",priority=1 +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// GarageReplication is the Schema for the garagereplications API. +// It replicates objects between two independent Garage clusters. +type GarageReplication struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitzero"` + + // +required + Spec GarageReplicationSpec `json:"spec"` + + // +optional + Status GarageReplicationStatus `json:"status,omitzero"` +} + +// +kubebuilder:object:root=true + +// GarageReplicationList contains a list of GarageReplication +type GarageReplicationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitzero"` + Items []GarageReplication `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GarageReplication{}, &GarageReplicationList{}) +} diff --git a/api/v1beta1/garagereplication_webhook.go b/api/v1beta1/garagereplication_webhook.go new file mode 100644 index 00000000..8ad281be --- /dev/null +++ b/api/v1beta1/garagereplication_webhook.go @@ -0,0 +1,345 @@ +/* +Copyright 2026 Raj Singh. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "fmt" + "net" + "net/url" + "strings" + + "github.com/robfig/cron/v3" + utilvalidation "k8s.io/apimachinery/pkg/util/validation" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + garageReplicationKind = "GarageReplication" + + // ReplicationDefaultAccessKeyIDKey matches the GarageKey Secret default. + ReplicationDefaultAccessKeyIDKey = "access-key-id" + // ReplicationDefaultSecretAccessKeyKey matches the GarageKey Secret default. + ReplicationDefaultSecretAccessKeyKey = "secret-access-key" + // ReplicationDefaultCABundleKey is the default caBundleRef key. + ReplicationDefaultCABundleKey = "ca.crt" + // ReplicationDefaultRegion is Garage's default s3_region. + ReplicationDefaultRegion = "garage" + + // Field names of the credential key selectors, shared with GarageKey. + accessKeyIDKeyField = "accessKeyIdKey" + secretAccessKeyKeyField = "secretAccessKeyKey" +) + +var garagereplicationlog = logf.Log.WithName("garagereplication-resource") + +// SetupGarageReplicationWebhookWithManager sets up the webhook with the +// Manager. controllerEnabled reports whether this operator runs the +// GarageReplication controller (--enable-replication); without it admission +// warns that objects are not reconciled. +func SetupGarageReplicationWebhookWithManager(mgr ctrl.Manager, controllerEnabled bool) error { + return ctrl.NewWebhookManagedBy(mgr, &GarageReplication{}). + WithValidator(&GarageReplicationValidator{ + Client: mgr.GetClient(), + AuthorizationReader: mgr.GetAPIReader(), + ControllerEnabled: controllerEnabled, + }). + Complete() +} + +// +kubebuilder:webhook:path=/validate-garage-rajsingh-info-v1beta1-garagereplication,mutating=false,failurePolicy=fail,sideEffects=None,groups=garage.rajsingh.info,resources=garagereplications,verbs=create;update,versions=v1beta1,name=vgaragereplication.kb.io,admissionReviewVersions=v1 +// +kubebuilder:rbac:groups=garage.rajsingh.info,resources=garagereplications,verbs=get;list;watch + +var _ admission.Validator[*GarageReplication] = &GarageReplicationValidator{} + +// +kubebuilder:object:generate=false + +// GarageReplicationValidator handles validation for GarageReplication. +// +// Field formats, enums, list keys and the mode/mirror pairing are enforced by +// the CRD schema. The webhook only adds what needs context: the reference +// grant of a cross-namespace clusterRef, destination buckets claimed by +// another GarageReplication, and checks CEL cannot express. +type GarageReplicationValidator struct { + Client client.Client + AuthorizationReader client.Reader + // ControllerEnabled reports that this operator reconciles + // GarageReplications. While it is false, admission warns that the object + // will not be reconciled. + ControllerEnabled bool +} + +// ReplicationNotReconciledWarning is returned on admission while no +// replication controller runs. +const ReplicationNotReconciledWarning = "the GarageReplication controller is not enabled in this operator (--enable-replication, chart value replication.enabled): the object is stored but no objects will be copied" + +func (v *GarageReplicationValidator) authorizationReader() client.Reader { + if v.AuthorizationReader != nil { + return v.AuthorizationReader + } + return v.Client +} + +// ValidateCreate implements admission.Validator so a webhook will be registered for the type. +func (v *GarageReplicationValidator) ValidateCreate(ctx context.Context, obj *GarageReplication) (admission.Warnings, error) { + garagereplicationlog.Info("validate create", "name", obj.Name) + return v.validate(ctx, obj) +} + +// ValidateUpdate implements admission.Validator so a webhook will be registered for the type. +func (v *GarageReplicationValidator) ValidateUpdate(ctx context.Context, _, newObj *GarageReplication) (admission.Warnings, error) { + garagereplicationlog.Info("validate update", "name", newObj.Name) + return v.validate(ctx, newObj) +} + +// ValidateDelete implements admission.Validator so a webhook will be registered for the type. +func (v *GarageReplicationValidator) ValidateDelete(_ context.Context, obj *GarageReplication) (admission.Warnings, error) { + garagereplicationlog.Info("validate delete", "name", obj.Name) + return nil, nil +} + +func (v *GarageReplicationValidator) validate(ctx context.Context, obj *GarageReplication) (admission.Warnings, error) { + warnings, err := ValidateGarageReplicationSpec(obj) + if err != nil { + return warnings, err + } + for _, side := range []struct { + field string + endpoint ReplicationEndpoint + }{{"source", obj.Spec.Source}, {"destination", obj.Spec.Destination}} { + ref := side.endpoint.ClusterRef + if ref == nil { + continue + } + name, namespace := effectiveClusterReference(*ref, obj.Namespace) + if err := checkReferenceGrant(ctx, v.authorizationReader(), garageReplicationKind, obj.Namespace, garageClusterKind, namespace, name); err != nil { + return warnings, fmt.Errorf("spec.%s.clusterRef: %w", side.field, err) + } + } + if err := v.validateDestinationExclusivity(ctx, obj); err != nil { + return warnings, err + } + if !v.ControllerEnabled { + warnings = append(warnings, ReplicationNotReconciledWarning) + } + return warnings, nil +} + +// validateDestinationExclusivity rejects a destination bucket another +// GarageReplication already writes: two Mirrors on one bucket would delete +// each other's objects. The check is best-effort against the informer cache +// (it only sees the namespaces the operator watches); the controller applies +// the same rule before starting Jobs. +// +// The check is cluster-wide and first come, first served, so a namespace can +// claim a destination bucket name before another one does. The error names +// the other GarageReplication only when it is in the same namespace. +func (v *GarageReplicationValidator) validateDestinationExclusivity(ctx context.Context, obj *GarageReplication) error { + if v.Client == nil { + return nil + } + var existing GarageReplicationList + if err := v.Client.List(ctx, &existing); err != nil { + return fmt.Errorf("failed to check destination buckets of other GarageReplications: %w", err) + } + if other, bucket := FindReplicationDestinationConflict(obj, existing.Items); other != nil { + owner := "a GarageReplication in another namespace" + if other.Namespace == obj.Namespace { + owner = fmt.Sprintf("GarageReplication %q", other.Name) + } + return fmt.Errorf("destination bucket %q is already written by %s; a destination bucket can be written by only one GarageReplication", + bucket, owner) + } + return nil +} + +// EffectiveReplicationMode returns spec.mode with its default applied. +func EffectiveReplicationMode(spec *GarageReplicationSpec) ReplicationMode { + if spec.Mode == "" { + return ReplicationModeCopy + } + return spec.Mode +} + +// ParseReplicationSchedule parses spec.schedule. An empty schedule returns a +// nil schedule and no error. +func ParseReplicationSchedule(schedule string) (cron.Schedule, error) { + if strings.TrimSpace(schedule) == "" { + return nil, nil + } + return cron.ParseStandard(schedule) +} + +// ValidateGarageReplicationSpec validates what the CRD schema cannot and that +// does not need the API server. It is shared by admission and reconciliation +// so objects persisted without the webhook fail with the same error. +func ValidateGarageReplicationSpec(obj *GarageReplication) (admission.Warnings, error) { + var warnings admission.Warnings + spec := &obj.Spec + + for _, side := range []struct { + field string + endpoint ReplicationEndpoint + }{{"spec.source", spec.Source}, {"spec.destination", spec.Destination}} { + if (side.endpoint.ClusterRef == nil) == (side.endpoint.Endpoint == nil) { + return warnings, fmt.Errorf("%s: exactly one of clusterRef or endpoint must be set", side.field) + } + if ref := side.endpoint.ClusterRef; ref != nil { + if err := ValidateClusterReference(*ref, side.field+".clusterRef"); err != nil { + return warnings, err + } + } + } + + // The schema catches a self-loop between two endpoint URLs that differ at + // most by case. A clusterRef self-loop (whose namespace may be implicit) + // and URL spellings differing by a trailing slash or a default port are + // caught here. + sameCluster := ReplicationEndpointKey(spec.Source, obj.Namespace) == ReplicationEndpointKey(spec.Destination, obj.Namespace) + for i, b := range spec.Buckets { + if sameCluster && b.Source == b.Destination { + return warnings, fmt.Errorf("spec.buckets[%d] replicates bucket %q onto itself: source and destination are the same cluster", i, b.Source) + } + } + + if _, err := ParseReplicationSchedule(spec.Schedule); err != nil { + return warnings, fmt.Errorf("spec.schedule %q is not a valid cron expression: %w", spec.Schedule, err) + } + + if pt := spec.PodTemplate; pt != nil { + for key, value := range pt.Labels { + if problems := utilvalidation.IsQualifiedName(key); len(problems) > 0 { + return warnings, fmt.Errorf("spec.podTemplate.labels key %q is invalid: %s", key, strings.Join(problems, "; ")) + } + if problems := utilvalidation.IsValidLabelValue(value); len(problems) > 0 { + return warnings, fmt.Errorf("spec.podTemplate.labels[%q] value %q is invalid: %s", key, value, strings.Join(problems, "; ")) + } + } + for key := range pt.Annotations { + if problems := utilvalidation.IsQualifiedName(key); len(problems) > 0 { + return warnings, fmt.Errorf("spec.podTemplate.annotations key %q is invalid: %s", key, strings.Join(problems, "; ")) + } + } + } + + if EffectiveReplicationMode(spec) == ReplicationModeMirror { + warnings = append(warnings, "mode Mirror deletes destination objects that do not exist on the source (up to spec.mirror.maxDeletes per bucket and run); the first run waits for the garage.rajsingh.info/trigger-sync annotation") + } + if sameCluster { + warnings = append(warnings, "source and destination are the same cluster; this copies between buckets of one cluster and protects against neither cluster loss nor cluster migration") + } + return warnings, nil +} + +// ReplicationCredentialKeys returns the effective Secret data keys. +func ReplicationCredentialKeys(ref ReplicationCredentialsRef) (string, string) { + idKey, secretKey := ref.AccessKeyIDKey, ref.SecretAccessKeyKey + if idKey == "" { + idKey = ReplicationDefaultAccessKeyIDKey + } + if secretKey == "" { + secretKey = ReplicationDefaultSecretAccessKeyKey + } + return idKey, secretKey +} + +// ReplicationEndpointKey identifies the cluster an endpoint points at as +// written in the spec: "cluster:/" for a clusterRef (the +// namespace defaulted), or the normalized URL for an explicit endpoint. +// A clusterRef and a URL that resolve to the same cluster get different +// keys; the controller compares resolved URLs as well. +func ReplicationEndpointKey(e ReplicationEndpoint, namespace string) string { + if e.ClusterRef != nil { + name, ns := effectiveClusterReference(*e.ClusterRef, namespace) + return "cluster:" + ns + "/" + name + } + if e.Endpoint != nil { + return NormalizeReplicationURL(e.Endpoint.URL) + } + return "" +} + +// NormalizeReplicationURL lowercases scheme and host and drops a trailing +// slash and the scheme's default port, so spellings of one endpoint compare +// equal. +func NormalizeReplicationURL(raw string) string { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Host == "" { + return strings.TrimSuffix(strings.ToLower(raw), "/") + } + scheme := strings.ToLower(u.Scheme) + host := strings.ToLower(u.Hostname()) + port := u.Port() + if (scheme == "http" && port == "80") || (scheme == "https" && port == "443") { + port = "" + } + if port != "" { + host = net.JoinHostPort(host, port) + } else if strings.Contains(host, ":") { + host = "[" + host + "]" + } + return scheme + "://" + host +} + +// ReplicationPrecedes orders GarageReplications claiming the same destination +// bucket: the older one keeps it (creation time, then namespace/name). An +// object being created has no creation time yet and loses to every +// existing one. +func ReplicationPrecedes(a, b *GarageReplication) bool { + at, bt := a.CreationTimestamp, b.CreationTimestamp + switch { + case at.IsZero() != bt.IsZero(): + return bt.IsZero() + case !at.Equal(&bt): + return at.Before(&bt) + } + if a.Namespace != b.Namespace { + return a.Namespace < b.Namespace + } + return a.Name < b.Name +} + +// FindReplicationDestinationConflict returns a GarageReplication of items that +// precedes self and writes one of self's destination buckets on the same +// destination cluster, with that bucket. It returns nil when self owns all of +// its destinations. +func FindReplicationDestinationConflict(self *GarageReplication, items []GarageReplication) (*GarageReplication, string) { + if self == nil { + return nil, "" + } + selfKey := ReplicationEndpointKey(self.Spec.Destination, self.Namespace) + for i := range items { + other := &items[i] + if other.Namespace == self.Namespace && other.Name == self.Name { + continue + } + if ReplicationEndpointKey(other.Spec.Destination, other.Namespace) != selfKey || !ReplicationPrecedes(other, self) { + continue + } + for _, b := range self.Spec.Buckets { + for _, ob := range other.Spec.Buckets { + if b.Destination == ob.Destination { + return other, b.Destination + } + } + } + } + return nil, "" +} diff --git a/api/v1beta1/garagereplication_webhook_test.go b/api/v1beta1/garagereplication_webhook_test.go new file mode 100644 index 00000000..ac85fe5e --- /dev/null +++ b/api/v1beta1/garagereplication_webhook_test.go @@ -0,0 +1,294 @@ +/* +Copyright 2026 Raj Singh. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func validReplication() *GarageReplication { + return &GarageReplication{ + ObjectMeta: metav1.ObjectMeta{Name: "repl", Namespace: testSourceNS}, + Spec: GarageReplicationSpec{ + Source: ReplicationEndpoint{ + ClusterRef: &ClusterReference{Name: testCluster}, + CredentialsSecretRef: ReplicationCredentialsRef{Name: "src"}, + }, + Destination: ReplicationEndpoint{ + Endpoint: &ReplicationS3Endpoint{URL: "https://s3.dr.example.com"}, + CredentialsSecretRef: ReplicationCredentialsRef{Name: "dst"}, + }, + Buckets: []ReplicationBucket{ + {Source: "medias", Destination: "medias"}, + {Source: "pg-dumps", Destination: "pg-dumps-dr"}, + }, + Schedule: "*/15 * * * *", + }, + } +} + +// Formats, enums, list keys and the mode/mirror pairing are schema rules, +// covered against a real API server in +// internal/controller/garagereplication_schema_test.go. These cases cover +// what only the webhook and the controller check. +func TestValidateGarageReplicationSpec(t *testing.T) { + cases := []struct { + name string + mutate func(*GarageReplication) + wantErr string + warning string + }{ + {name: "valid copy", mutate: func(*GarageReplication) {}}, + {name: "descriptor schedule", mutate: func(r *GarageReplication) { r.Spec.Schedule = "@hourly" }}, + {name: "timezone schedule", mutate: func(r *GarageReplication) { r.Spec.Schedule = "CRON_TZ=Europe/Paris 0 2 * * *" }}, + {name: "one-shot without schedule", mutate: func(r *GarageReplication) { r.Spec.Schedule = "" }}, + {name: "bad schedule", mutate: func(r *GarageReplication) { r.Spec.Schedule = "every 5 minutes" }, wantErr: "not a valid cron expression"}, + {name: "both clusterRef and endpoint", mutate: func(r *GarageReplication) { + r.Spec.Source.Endpoint = &ReplicationS3Endpoint{URL: "https://x.example.com"} + }, wantErr: "exactly one of clusterRef or endpoint"}, + {name: "kubeConfigSecretRef", mutate: func(r *GarageReplication) { + r.Spec.Source.ClusterRef.KubeConfigSecretRef = &corev1.SecretKeySelector{} + }, wantErr: "kubeConfigSecretRef is not supported"}, + {name: "self loop on the same clusterRef", mutate: func(r *GarageReplication) { + r.Spec.Destination.Endpoint = nil + r.Spec.Destination.ClusterRef = &ClusterReference{Name: testCluster} + }, wantErr: "onto itself"}, + {name: "self loop with an implicit namespace", mutate: func(r *GarageReplication) { + r.Spec.Destination.Endpoint = nil + r.Spec.Destination.ClusterRef = &ClusterReference{Name: testCluster, Namespace: testSourceNS} + }, wantErr: "onto itself"}, + {name: "self loop with another URL spelling", mutate: func(r *GarageReplication) { + r.Spec.Source.ClusterRef = nil + r.Spec.Source.Endpoint = &ReplicationS3Endpoint{URL: "HTTPS://s3.dr.example.com:443/"} + }, wantErr: "onto itself"}, + {name: "same cluster with renamed buckets warns", mutate: func(r *GarageReplication) { + r.Spec.Destination.Endpoint = nil + r.Spec.Destination.ClusterRef = &ClusterReference{Name: testCluster} + r.Spec.Buckets = []ReplicationBucket{{Source: "medias", Destination: "medias-copy"}} + }, warning: "same cluster"}, + {name: "mirror warns about deletions and the first trigger", mutate: func(r *GarageReplication) { + r.Spec.Mode = ReplicationModeMirror + r.Spec.Mirror = &ReplicationMirrorConfig{MaxDeletes: 100} + }, warning: "first run waits for the garage.rajsingh.info/trigger-sync annotation"}, + {name: "bad pod label", mutate: func(r *GarageReplication) { + r.Spec.PodTemplate = &ReplicationPodTemplate{Labels: map[string]string{"bad key": "v"}} + }, wantErr: "podTemplate.labels"}, + {name: "bad pod annotation", mutate: func(r *GarageReplication) { + r.Spec.PodTemplate = &ReplicationPodTemplate{Annotations: map[string]string{"bad key": "v"}} + }, wantErr: "podTemplate.annotations"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := validReplication() + tc.mutate(r) + warnings, err := ValidateGarageReplicationSpec(r) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("want error containing %q, got %v", tc.wantErr, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tc.warning != "" && !strings.Contains(strings.Join(warnings, "\n"), tc.warning) { + t.Fatalf("want warning containing %q, got %v", tc.warning, warnings) + } + }) + } +} + +func TestNormalizeReplicationURL(t *testing.T) { + for in, want := range map[string]string{ + "https://s3.example.com": "https://s3.example.com", + "HTTPS://S3.Example.com/": "https://s3.example.com", + "https://s3.example.com:443": "https://s3.example.com", + "http://garage.ns.svc:80/": "http://garage.ns.svc", + "http://garage.ns.svc:3900": "http://garage.ns.svc:3900", + "https://s3.example.com:80": "https://s3.example.com:80", + "http://[fd00::1]:3900": "http://[fd00::1]:3900", + "http://[FD00::1]/": "http://[fd00::1]", + } { + if got := NormalizeReplicationURL(in); got != want { + t.Errorf("NormalizeReplicationURL(%q) = %q, want %q", in, got, want) + } + } +} + +func TestFindReplicationDestinationConflict(t *testing.T) { + older := validReplication() + older.Name = "older" + older.CreationTimestamp = metav1.NewTime(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + newer := validReplication() + newer.Name = "newer" + newer.CreationTimestamp = metav1.NewTime(older.CreationTimestamp.Add(time.Hour)) + + if other, bucket := FindReplicationDestinationConflict(newer, []GarageReplication{*older, *newer}); other == nil || other.Name != "older" || bucket != "medias" { + t.Fatalf("newer must lose medias to older, got %v %q", other, bucket) + } + if other, _ := FindReplicationDestinationConflict(older, []GarageReplication{*older, *newer}); other != nil { + t.Fatalf("older keeps its destinations, got conflict with %s", other.Name) + } + + // An object being created has no creation time and loses to existing ones. + created := validReplication() + created.Name = "created" + if other, _ := FindReplicationDestinationConflict(created, []GarageReplication{*newer}); other == nil { + t.Fatal("a new object must not take a destination bucket already written") + } + + // Same bucket names on another destination cluster do not conflict. + elsewhere := validReplication() + elsewhere.Name = "elsewhere" + elsewhere.Spec.Destination.Endpoint.URL = "https://s3.other.example.com" + if other, _ := FindReplicationDestinationConflict(elsewhere, []GarageReplication{*older}); other != nil { + t.Fatalf("another destination cluster must not conflict, got %s", other.Name) + } + + // Another URL spelling of the same destination does conflict. + spelled := validReplication() + spelled.Name = "spelled" + spelled.Spec.Destination.Endpoint.URL = "https://S3.dr.example.com:443/" + spelled.Spec.Buckets = []ReplicationBucket{{Source: "other", Destination: "pg-dumps-dr"}} + if other, bucket := FindReplicationDestinationConflict(spelled, []GarageReplication{*older}); other == nil || bucket != "pg-dumps-dr" { + t.Fatalf("same destination with another spelling must conflict, got %v %q", other, bucket) + } + + // Disjoint buckets on the same destination are fine. + disjoint := validReplication() + disjoint.Name = "disjoint" + disjoint.Spec.Buckets = []ReplicationBucket{{Source: "logs", Destination: "logs"}} + if other, _ := FindReplicationDestinationConflict(disjoint, []GarageReplication{*older}); other != nil { + t.Fatalf("disjoint buckets must not conflict, got %s", other.Name) + } + + // A clusterRef destination: an omitted namespace equals the object's own. + byRef := validReplication() + byRef.Name = "by-ref" + byRef.CreationTimestamp = older.CreationTimestamp + byRef.Spec.Destination = ReplicationEndpoint{ClusterRef: &ClusterReference{Name: "dr"}, CredentialsSecretRef: ReplicationCredentialsRef{Name: "dst"}} + byRefExplicit := byRef.DeepCopy() + byRefExplicit.Name = "by-ref-explicit" + byRefExplicit.CreationTimestamp = newer.CreationTimestamp + byRefExplicit.Spec.Destination.ClusterRef.Namespace = testSourceNS + if other, _ := FindReplicationDestinationConflict(byRefExplicit, []GarageReplication{*byRef}); other == nil { + t.Fatal("an implicit and an explicit namespace of one cluster must conflict") + } +} + +func TestGarageReplicationWebhookRejectsTakenDestination(t *testing.T) { + scheme := runtime.NewScheme() + if err := AddToScheme(scheme); err != nil { + t.Fatal(err) + } + existing := validReplication() + existing.Name = "existing" + existing.Namespace = testTargetNS + existing.CreationTimestamp = metav1.NewTime(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(existing).Build() + v := &GarageReplicationValidator{Client: c} + + // The holder is in another namespace: the error must not name it. + r := validReplication() + _, err := v.ValidateCreate(context.Background(), r) + if err == nil || !strings.Contains(err.Error(), "already written by a GarageReplication in another namespace") { + t.Fatalf("want a destination conflict error, got %v", err) + } + if strings.Contains(err.Error(), testTargetNS) || strings.Contains(err.Error(), "existing") { + t.Fatalf("the error must not reveal another namespace's object: %v", err) + } + r.Spec.Buckets = []ReplicationBucket{{Source: "logs", Destination: "logs"}} + if _, err := v.ValidateCreate(context.Background(), r); err != nil { + t.Fatalf("a free destination bucket was rejected: %v", err) + } + + // In the same namespace, the holder is named. + sameNS := existing.DeepCopy() + sameNS.Namespace = testSourceNS + sameNS.ResourceVersion = "" + c = fake.NewClientBuilder().WithScheme(scheme).WithObjects(sameNS).Build() + v = &GarageReplicationValidator{Client: c} + if _, err := v.ValidateCreate(context.Background(), validReplication()); err == nil || !strings.Contains(err.Error(), `already written by GarageReplication "existing"`) { + t.Fatalf("want a conflict naming the same-namespace object, got %v", err) + } +} + +func TestGarageReplicationWebhookWarnsWhenNotReconciled(t *testing.T) { + scheme := runtime.NewScheme() + if err := AddToScheme(scheme); err != nil { + t.Fatal(err) + } + c := fake.NewClientBuilder().WithScheme(scheme).Build() + for _, enabled := range []bool{false, true} { + v := &GarageReplicationValidator{Client: c, ControllerEnabled: enabled} + warnings, err := v.ValidateCreate(context.Background(), validReplication()) + if err != nil { + t.Fatalf("valid object rejected: %v", err) + } + if got := slices.Contains(warnings, ReplicationNotReconciledWarning); got == enabled { + t.Fatalf("ControllerEnabled=%v: not-reconciled warning present=%v, warnings %q", enabled, got, warnings) + } + } +} + +func TestGarageReplicationWebhookRequiresGrantForCrossNamespaceCluster(t *testing.T) { + scheme := runtime.NewScheme() + if err := AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + r := validReplication() + r.Spec.Source.ClusterRef.Namespace = testTargetNS + + c := fake.NewClientBuilder().WithScheme(scheme).Build() + v := &GarageReplicationValidator{Client: c} + if _, err := v.ValidateCreate(context.Background(), r); err == nil || !strings.Contains(err.Error(), "GarageReferenceGrant") { + t.Fatalf("want a missing-grant error, got %v", err) + } + + grant := &GarageReferenceGrant{ + ObjectMeta: metav1.ObjectMeta{Name: "allow", Namespace: testTargetNS}, + Spec: GarageReferenceGrantSpec{ + From: []ReferenceGrantFrom{{Kind: garageReplicationKind, Namespace: testSourceNS}}, + To: []ReferenceGrantTo{{Kind: garageClusterKind, Name: testCluster}}, + }, + } + c = fake.NewClientBuilder().WithScheme(scheme).WithObjects(grant).Build() + v = &GarageReplicationValidator{Client: c} + if _, err := v.ValidateCreate(context.Background(), r); err != nil { + t.Fatalf("granted reference rejected: %v", err) + } + + // A grant for another kind does not authorize replication. + grant.Spec.From[0].Kind = garageBucketKind + c = fake.NewClientBuilder().WithScheme(scheme).WithObjects(grant).Build() + v = &GarageReplicationValidator{Client: c} + if _, err := v.ValidateCreate(context.Background(), r); err == nil { + t.Fatal("a GarageBucket grant must not authorize a GarageReplication") + } +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 5f117d1e..5cb3dfac 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -2085,6 +2085,182 @@ func (in *GarageReferenceGrantValidator) DeepCopy() *GarageReferenceGrantValidat return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GarageReplication) DeepCopyInto(out *GarageReplication) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GarageReplication. +func (in *GarageReplication) DeepCopy() *GarageReplication { + if in == nil { + return nil + } + out := new(GarageReplication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GarageReplication) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GarageReplicationList) DeepCopyInto(out *GarageReplicationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GarageReplication, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GarageReplicationList. +func (in *GarageReplicationList) DeepCopy() *GarageReplicationList { + if in == nil { + return nil + } + out := new(GarageReplicationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GarageReplicationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GarageReplicationSpec) DeepCopyInto(out *GarageReplicationSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + in.Destination.DeepCopyInto(&out.Destination) + if in.Buckets != nil { + in, out := &in.Buckets, &out.Buckets + *out = make([]ReplicationBucket, len(*in)) + copy(*out, *in) + } + if in.Mirror != nil { + in, out := &in.Mirror, &out.Mirror + *out = new(ReplicationMirrorConfig) + **out = **in + } + if in.Transfer != nil { + in, out := &in.Transfer, &out.Transfer + *out = new(ReplicationTransferConfig) + **out = **in + } + if in.Retry != nil { + in, out := &in.Retry, &out.Retry + *out = new(ReplicationRetryConfig) + (*in).DeepCopyInto(*out) + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + if in.PodTemplate != nil { + in, out := &in.PodTemplate, &out.PodTemplate + *out = new(ReplicationPodTemplate) + (*in).DeepCopyInto(*out) + } + if in.SuccessfulJobsHistoryLimit != nil { + in, out := &in.SuccessfulJobsHistoryLimit, &out.SuccessfulJobsHistoryLimit + *out = new(int32) + **out = **in + } + if in.FailedJobsHistoryLimit != nil { + in, out := &in.FailedJobsHistoryLimit, &out.FailedJobsHistoryLimit + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GarageReplicationSpec. +func (in *GarageReplicationSpec) DeepCopy() *GarageReplicationSpec { + if in == nil { + return nil + } + out := new(GarageReplicationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GarageReplicationStatus) DeepCopyInto(out *GarageReplicationStatus) { + *out = *in + if in.CurrentRun != nil { + in, out := &in.CurrentRun, &out.CurrentRun + *out = new(ReplicationRun) + (*in).DeepCopyInto(*out) + } + if in.LastRun != nil { + in, out := &in.LastRun, &out.LastRun + *out = new(ReplicationRun) + (*in).DeepCopyInto(*out) + } + if in.LastScheduleTime != nil { + in, out := &in.LastScheduleTime, &out.LastScheduleTime + *out = (*in).DeepCopy() + } + if in.LastSyncTime != nil { + in, out := &in.LastSyncTime, &out.LastSyncTime + *out = (*in).DeepCopy() + } + if in.LastSuccessfulSyncTime != nil { + in, out := &in.LastSuccessfulSyncTime, &out.LastSuccessfulSyncTime + *out = (*in).DeepCopy() + } + if in.NextSyncTime != nil { + in, out := &in.NextSyncTime, &out.NextSyncTime + *out = (*in).DeepCopy() + } + if in.Buckets != nil { + in, out := &in.Buckets, &out.Buckets + *out = make([]ReplicationBucketStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.LastOperation != nil { + in, out := &in.LastOperation, &out.LastOperation + *out = new(LastOperationStatus) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GarageReplicationStatus. +func (in *GarageReplicationStatus) DeepCopy() *GarageReplicationStatus { + if in == nil { + return nil + } + out := new(GarageReplicationStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImportKeyConfig) DeepCopyInto(out *ImportKeyConfig) { *out = *in @@ -3298,6 +3474,64 @@ func (in *RepairStatus) DeepCopy() *RepairStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationBucket) DeepCopyInto(out *ReplicationBucket) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationBucket. +func (in *ReplicationBucket) DeepCopy() *ReplicationBucket { + if in == nil { + return nil + } + out := new(ReplicationBucket) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationBucketStatus) DeepCopyInto(out *ReplicationBucketStatus) { + *out = *in + if in.LastCompletionTime != nil { + in, out := &in.LastCompletionTime, &out.LastCompletionTime + *out = (*in).DeepCopy() + } + if in.LastSuccessfulTime != nil { + in, out := &in.LastSuccessfulTime, &out.LastSuccessfulTime + *out = (*in).DeepCopy() + } + if in.LastStatistics != nil { + in, out := &in.LastStatistics, &out.LastStatistics + *out = new(ReplicationStatistics) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationBucketStatus. +func (in *ReplicationBucketStatus) DeepCopy() *ReplicationBucketStatus { + if in == nil { + return nil + } + out := new(ReplicationBucketStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationCABundleRef) DeepCopyInto(out *ReplicationCABundleRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationCABundleRef. +func (in *ReplicationCABundleRef) DeepCopy() *ReplicationCABundleRef { + if in == nil { + return nil + } + out := new(ReplicationCABundleRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplicationConfig) DeepCopyInto(out *ReplicationConfig) { *out = *in @@ -3318,6 +3552,215 @@ func (in *ReplicationConfig) DeepCopy() *ReplicationConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationCredentialsRef) DeepCopyInto(out *ReplicationCredentialsRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationCredentialsRef. +func (in *ReplicationCredentialsRef) DeepCopy() *ReplicationCredentialsRef { + if in == nil { + return nil + } + out := new(ReplicationCredentialsRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationEndpoint) DeepCopyInto(out *ReplicationEndpoint) { + *out = *in + if in.ClusterRef != nil { + in, out := &in.ClusterRef, &out.ClusterRef + *out = new(ClusterReference) + (*in).DeepCopyInto(*out) + } + if in.Endpoint != nil { + in, out := &in.Endpoint, &out.Endpoint + *out = new(ReplicationS3Endpoint) + (*in).DeepCopyInto(*out) + } + out.CredentialsSecretRef = in.CredentialsSecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationEndpoint. +func (in *ReplicationEndpoint) DeepCopy() *ReplicationEndpoint { + if in == nil { + return nil + } + out := new(ReplicationEndpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationMirrorConfig) DeepCopyInto(out *ReplicationMirrorConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationMirrorConfig. +func (in *ReplicationMirrorConfig) DeepCopy() *ReplicationMirrorConfig { + if in == nil { + return nil + } + out := new(ReplicationMirrorConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationPodTemplate) DeepCopyInto(out *ReplicationPodTemplate) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationPodTemplate. +func (in *ReplicationPodTemplate) DeepCopy() *ReplicationPodTemplate { + if in == nil { + return nil + } + out := new(ReplicationPodTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationRetryConfig) DeepCopyInto(out *ReplicationRetryConfig) { + *out = *in + if in.BackoffLimit != nil { + in, out := &in.BackoffLimit, &out.BackoffLimit + *out = new(int32) + **out = **in + } + if in.ActiveDeadlineSeconds != nil { + in, out := &in.ActiveDeadlineSeconds, &out.ActiveDeadlineSeconds + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationRetryConfig. +func (in *ReplicationRetryConfig) DeepCopy() *ReplicationRetryConfig { + if in == nil { + return nil + } + out := new(ReplicationRetryConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationRun) DeepCopyInto(out *ReplicationRun) { + *out = *in + in.StartTime.DeepCopyInto(&out.StartTime) + if in.Buckets != nil { + in, out := &in.Buckets, &out.Buckets + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.CompletionTime != nil { + in, out := &in.CompletionTime, &out.CompletionTime + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationRun. +func (in *ReplicationRun) DeepCopy() *ReplicationRun { + if in == nil { + return nil + } + out := new(ReplicationRun) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationS3Endpoint) DeepCopyInto(out *ReplicationS3Endpoint) { + *out = *in + if in.CABundleRef != nil { + in, out := &in.CABundleRef, &out.CABundleRef + *out = new(ReplicationCABundleRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationS3Endpoint. +func (in *ReplicationS3Endpoint) DeepCopy() *ReplicationS3Endpoint { + if in == nil { + return nil + } + out := new(ReplicationS3Endpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationStatistics) DeepCopyInto(out *ReplicationStatistics) { + *out = *in + if in.Differences != nil { + in, out := &in.Differences, &out.Differences + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationStatistics. +func (in *ReplicationStatistics) DeepCopy() *ReplicationStatistics { + if in == nil { + return nil + } + out := new(ReplicationStatistics) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicationTransferConfig) DeepCopyInto(out *ReplicationTransferConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationTransferConfig. +func (in *ReplicationTransferConfig) DeepCopy() *ReplicationTransferConfig { + if in == nil { + return nil + } + out := new(ReplicationTransferConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *S3APIConfig) DeepCopyInto(out *S3APIConfig) { *out = *in diff --git a/charts/garage-operator/crd-bases/garage.rajsingh.info_garagereferencegrants.yaml b/charts/garage-operator/crd-bases/garage.rajsingh.info_garagereferencegrants.yaml index 8f1ba94b..c6813c4a 100644 --- a/charts/garage-operator/crd-bases/garage.rajsingh.info_garagereferencegrants.yaml +++ b/charts/garage-operator/crd-bases/garage.rajsingh.info_garagereferencegrants.yaml @@ -77,10 +77,12 @@ spec: Kind is the resource kind allowed to make cross-namespace references. GarageAdminToken remains in the schema for compatibility, but the static credential path is namespace-local and does not accept cross-namespace grants. + GarageReplication may reference a GarageCluster in another namespace. enum: - GarageKey - GarageBucket - GarageAdminToken + - GarageReplication type: string namespace: description: |- diff --git a/charts/garage-operator/crd-bases/garage.rajsingh.info_garagereplications.yaml b/charts/garage-operator/crd-bases/garage.rajsingh.info_garagereplications.yaml new file mode 100644 index 00000000..9fe85359 --- /dev/null +++ b/charts/garage-operator/crd-bases/garage.rajsingh.info_garagereplications.yaml @@ -0,0 +1,1958 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.22.0 + name: garagereplications.garage.rajsingh.info +spec: + group: garage.rajsingh.info + names: + kind: GarageReplication + listKind: GarageReplicationList + plural: garagereplications + shortNames: + - grepl + singular: garagereplication + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.mode + name: Mode + type: string + - jsonPath: .spec.schedule + name: Schedule + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.lastSuccessfulSyncTime + name: Last Success + type: date + - jsonPath: .status.nextSyncTime + name: Next Sync + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + GarageReplication is the Schema for the garagereplications API. + It replicates objects between two independent Garage clusters. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + GarageReplicationSpec defines the desired state of GarageReplication. + + GarageReplication copies objects between two independent Garage clusters + through their S3 APIs, for replication and migration. It is not a backup: + there are no point-in-time snapshots and no restore, and in mode Mirror a + deletion or an overwrite on the source reaches the destination on the next + run. Each run executes one Kubernetes Job per bucket in the + GarageReplication's namespace, so the Jobs need network access to both S3 + endpoints. Only objects (and their user metadata) are copied: buckets, keys, + permissions, quotas, website and lifecycle configuration are not. Deleting a + GarageReplication never deletes object data on either side. + properties: + buckets: + description: Buckets lists the buckets to replicate, keyed by destination + bucket. + items: + description: ReplicationBucket maps a source bucket to a destination + bucket. + properties: + destination: + description: |- + Destination is the bucket name (global alias) on the destination + cluster. It is the list key, so destinations are unique. + maxLength: 63 + minLength: 3 + pattern: ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ + type: string + source: + description: Source is the bucket name (global alias) on the + source cluster. + maxLength: 63 + minLength: 3 + pattern: ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ + type: string + required: + - destination + - source + type: object + maxItems: 50 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - destination + x-kubernetes-list-type: map + destination: + description: |- + Destination is the cluster objects are written to. Its credentials need + read and write permission on the destination buckets. Destination + buckets must already exist: replication never creates buckets. A + destination bucket can be written by only one GarageReplication. + properties: + clusterRef: + description: |- + ClusterRef references a GarageCluster (operator-managed or a + management handle). The S3 endpoint and region are resolved from it. + A cross-namespace reference requires a GarageReferenceGrant in the + cluster's namespace that allows kind GarageReplication. + properties: + kubeConfigSecretRef: + description: |- + KubeConfigSecretRef is reserved for a future remote Kubernetes client integration. + It is currently rejected by admission because the operator does not use it. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + name: + description: Name of the GarageCluster resource. + type: string + namespace: + description: |- + Namespace of the GarageCluster. Defaults to the referencing resource's namespace. + Cross-namespace references require a GarageReferenceGrant where supported by + the owning resource. GarageNode and GarageAdminToken reject them. + type: string + required: + - name + type: object + credentialsSecretRef: + description: |- + CredentialsSecretRef references a Secret in the GarageReplication's + namespace holding an S3 access key. The Secret must carry the label + garage.rajsingh.info/replication-credentials: "true" (for a GarageKey, + set it in spec.secretTemplate.labels). The access key ID is sent to the + endpoint, so without this opt-in the right to create a GarageReplication + would amount to reading any Secret, or using any other application's + key. The default data keys match the Secret generated for a GarageKey. + properties: + accessKeyIdKey: + default: access-key-id + description: AccessKeyIDKey is the data key holding the access + key ID. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + name: + description: Name of the Secret in the GarageReplication's + namespace. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + secretAccessKeyKey: + default: secret-access-key + description: SecretAccessKeyKey is the data key holding the + secret access key. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: accessKeyIdKey and secretAccessKeyKey must be different + rule: '(has(self.accessKeyIdKey) ? self.accessKeyIdKey : ''access-key-id'') + != (has(self.secretAccessKeyKey) ? self.secretAccessKeyKey + : ''secret-access-key'')' + endpoint: + description: |- + Endpoint is an S3 endpoint of any Garage cluster, inside or outside + Kubernetes. + properties: + caBundleRef: + description: |- + CABundleRef references a ConfigMap holding a PEM CA bundle used, in + addition to the system trust store, to verify the endpoint's TLS + certificate. + properties: + key: + default: ca.crt + description: Key of the ConfigMap holding the PEM bundle. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + name: + description: Name of the ConfigMap. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + region: + default: garage + description: Region is the Garage s3_region used for request + signing. + maxLength: 128 + minLength: 1 + pattern: ^[A-Za-z0-9_.-]+$ + type: string + url: + description: |- + URL of the S3 API: scheme and host (with an optional port) only, e.g. + https://s3.example.com or http://garage.ns.svc:3900. Buckets are + addressed path-style below it, so it has no path, query, fragment or + credentials. + maxLength: 2048 + minLength: 1 + pattern: ^https?://[^\s/?#@]+/?$ + type: string + required: + - url + type: object + required: + - credentialsSecretRef + type: object + x-kubernetes-validations: + - message: exactly one of clusterRef or endpoint must be set + rule: has(self.clusterRef) != has(self.endpoint) + - message: clusterRef.kubeConfigSecretRef is not supported + rule: '!has(self.clusterRef) || !has(self.clusterRef.kubeConfigSecretRef)' + failedJobsHistoryLimit: + default: 3 + description: FailedJobsHistoryLimit is the number of finished failed + Jobs to keep. + format: int32 + maximum: 50 + minimum: 0 + type: integer + mirror: + description: |- + Mirror configures mode Mirror. Required when mode is Mirror and + rejected otherwise. + properties: + maxDeleteSize: + description: |- + MaxDeleteSize caps the total size of the destination objects one run + may delete per bucket, in bytes with an optional K, M, G or T suffix + (binary multiples: KiB, MiB, GiB, TiB), e.g. "10G". Once reached the run + stops and fails with reason MaxDeletesExceeded, like maxDeletes. Empty + means no size cap. + maxLength: 32 + pattern: ^[0-9]+(\.[0-9]+)?[KMGTkmgt]?$ + type: string + maxDeletes: + description: |- + MaxDeletes caps the number of destination objects one run may delete + per bucket. Once reached the run stops and fails with reason + MaxDeletesExceeded; deletions done before that point are kept. + format: int64 + minimum: 1 + type: integer + required: + - maxDeletes + type: object + mode: + default: Copy + description: |- + Mode controls whether changes on the source can remove data on the + destination. + - Copy: copy new and changed objects. Source deletions are not + propagated. Changed objects are overwritten on the destination. + - Mirror: the destination reflects the source, including deletions, + capped by mirror.maxDeletes per bucket and run. The first run waits + for the trigger-sync annotation. + enum: + - Copy + - Mirror + type: string + podTemplate: + description: PodTemplate customizes the replication Pods. + properties: + affinity: + description: Affinity for the Pods. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + annotations: + additionalProperties: + type: string + description: Annotations added to the Pods. + maxProperties: 64 + type: object + labels: + additionalProperties: + type: string + description: Labels added to the Pods. + maxProperties: 64 + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector for the Pods. + maxProperties: 64 + type: object + priorityClassName: + description: PriorityClassName for the Pods. + maxLength: 253 + type: string + tolerations: + description: Tolerations for the Pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + resources: + description: Resources for the replication container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + retry: + description: Retry bounds retries and duration of each per-bucket + Job. + properties: + activeDeadlineSeconds: + default: 21600 + description: ActiveDeadlineSeconds bounds the duration of a per-bucket + Job. + format: int64 + minimum: 60 + type: integer + backoffLimit: + default: 2 + description: |- + BackoffLimit is the number of Pod retries of a per-bucket Job. Each + retry restarts the bucket's comparison; already copied objects are + skipped. + format: int32 + maximum: 10 + minimum: 0 + type: integer + type: object + schedule: + description: |- + Schedule is a cron expression (standard 5-field syntax, descriptors + such as @hourly, optional CRON_TZ= prefix; UTC otherwise). Without a + schedule, runs happen only through the garage.rajsingh.info/trigger-sync + annotation (one-shot mode, e.g. for migrations). + maxLength: 128 + type: string + source: + description: |- + Source is the cluster objects are read from. Its credentials only need + read permission on the source buckets. + properties: + clusterRef: + description: |- + ClusterRef references a GarageCluster (operator-managed or a + management handle). The S3 endpoint and region are resolved from it. + A cross-namespace reference requires a GarageReferenceGrant in the + cluster's namespace that allows kind GarageReplication. + properties: + kubeConfigSecretRef: + description: |- + KubeConfigSecretRef is reserved for a future remote Kubernetes client integration. + It is currently rejected by admission because the operator does not use it. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + name: + description: Name of the GarageCluster resource. + type: string + namespace: + description: |- + Namespace of the GarageCluster. Defaults to the referencing resource's namespace. + Cross-namespace references require a GarageReferenceGrant where supported by + the owning resource. GarageNode and GarageAdminToken reject them. + type: string + required: + - name + type: object + credentialsSecretRef: + description: |- + CredentialsSecretRef references a Secret in the GarageReplication's + namespace holding an S3 access key. The Secret must carry the label + garage.rajsingh.info/replication-credentials: "true" (for a GarageKey, + set it in spec.secretTemplate.labels). The access key ID is sent to the + endpoint, so without this opt-in the right to create a GarageReplication + would amount to reading any Secret, or using any other application's + key. The default data keys match the Secret generated for a GarageKey. + properties: + accessKeyIdKey: + default: access-key-id + description: AccessKeyIDKey is the data key holding the access + key ID. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + name: + description: Name of the Secret in the GarageReplication's + namespace. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + secretAccessKeyKey: + default: secret-access-key + description: SecretAccessKeyKey is the data key holding the + secret access key. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: accessKeyIdKey and secretAccessKeyKey must be different + rule: '(has(self.accessKeyIdKey) ? self.accessKeyIdKey : ''access-key-id'') + != (has(self.secretAccessKeyKey) ? self.secretAccessKeyKey + : ''secret-access-key'')' + endpoint: + description: |- + Endpoint is an S3 endpoint of any Garage cluster, inside or outside + Kubernetes. + properties: + caBundleRef: + description: |- + CABundleRef references a ConfigMap holding a PEM CA bundle used, in + addition to the system trust store, to verify the endpoint's TLS + certificate. + properties: + key: + default: ca.crt + description: Key of the ConfigMap holding the PEM bundle. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + name: + description: Name of the ConfigMap. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + region: + default: garage + description: Region is the Garage s3_region used for request + signing. + maxLength: 128 + minLength: 1 + pattern: ^[A-Za-z0-9_.-]+$ + type: string + url: + description: |- + URL of the S3 API: scheme and host (with an optional port) only, e.g. + https://s3.example.com or http://garage.ns.svc:3900. Buckets are + addressed path-style below it, so it has no path, query, fragment or + credentials. + maxLength: 2048 + minLength: 1 + pattern: ^https?://[^\s/?#@]+/?$ + type: string + required: + - url + type: object + required: + - credentialsSecretRef + type: object + x-kubernetes-validations: + - message: exactly one of clusterRef or endpoint must be set + rule: has(self.clusterRef) != has(self.endpoint) + - message: clusterRef.kubeConfigSecretRef is not supported + rule: '!has(self.clusterRef) || !has(self.clusterRef.kubeConfigSecretRef)' + successfulJobsHistoryLimit: + default: 1 + description: |- + SuccessfulJobsHistoryLimit is the number of finished successful Jobs to + keep. + format: int32 + maximum: 50 + minimum: 0 + type: integer + suspend: + description: |- + Suspend stops starting new runs. A run already in progress finishes; + delete its Jobs to stop it immediately. + type: boolean + transfer: + description: Transfer tunes the copy. + properties: + bandwidthLimit: + description: |- + BandwidthLimit caps the transfer rate of each per-bucket Job in bytes + per second, with an optional K, M, G suffix (binary multiples: KiB/s, + MiB/s, GiB/s), e.g. "50M". Empty means unlimited. + maxLength: 32 + pattern: ^[0-9]+(\.[0-9]+)?[KMGkmg]?$ + type: string + checkers: + default: 8 + description: Checkers is the number of objects compared in parallel + per bucket. + format: int32 + maximum: 128 + minimum: 1 + type: integer + concurrency: + default: 4 + description: Concurrency is the number of objects transferred + in parallel per bucket. + format: int32 + maximum: 64 + minimum: 1 + type: integer + maxConcurrentBuckets: + default: 1 + description: MaxConcurrentBuckets is the number of buckets replicated + in parallel. + format: int32 + maximum: 50 + minimum: 1 + type: integer + type: object + verification: + default: None + description: |- + Verification runs a comparison of source and destination after each + successful copy. Check compares sizes and MD5 hashes (size only for + multipart objects without a stored MD5) and fails the run on any + difference. + enum: + - None + - Check + type: string + required: + - buckets + - destination + - source + type: object + x-kubernetes-validations: + - message: mirror must be set if and only if mode is Mirror + rule: has(self.mirror) == (has(self.mode) && self.mode == 'Mirror') + - message: 'a bucket cannot be replicated onto itself: source and destination + are the same endpoint' + rule: '!(has(self.source.endpoint) && has(self.destination.endpoint) + && self.source.endpoint.url == self.destination.endpoint.url) || self.buckets.all(b, + b.source != b.destination)' + status: + description: GarageReplicationStatus defines the observed state of GarageReplication. + properties: + buckets: + description: Buckets reports per-bucket results, keyed by destination + bucket. + items: + description: ReplicationBucketStatus reports the last result for + one bucket. + properties: + activeJob: + description: ActiveJob is the Job currently replicating this + bucket. + type: string + destination: + description: Destination bucket; the list key. + type: string + hold: + description: |- + Hold is set when this bucket must not run again without a person + asking. Scheduled runs skip a held bucket. Only the trigger-sync + annotation releases it; editing the spec does not. + enum: + - SourceEmpty + - MaxDeletesExceeded + - AwaitingTrigger + type: string + lastCompletionTime: + description: LastCompletionTime is when LastJob finished. + format: date-time + type: string + lastError: + description: LastError is a fixed reason for a failed LastJob. + enum: + - AuthFailed + - BucketNotFound + - EndpointUnreachable + - SourceEmpty + - MaxDeletesExceeded + - VerificationFailed + - DeadlineExceeded + - TransferErrors + - Unknown + type: string + lastJob: + description: LastJob is the last finished Job recorded for this + bucket. + type: string + lastMode: + description: |- + LastMode is the mode LastJob ran in. A Mirror bucket whose last Job + did not run in Mirror mode waits for a trigger-sync (hold + AwaitingTrigger) before its first Mirror run. + type: string + lastResult: + description: LastResult is the outcome of LastJob. + enum: + - Succeeded + - Failed + - Cancelled + type: string + lastRunId: + description: LastRunID is the run of LastJob. + type: string + lastStatistics: + description: |- + LastStatistics are the copy engine's counters for LastJob. Unset when + the engine reported none (e.g. the Pod was killed). + properties: + bytes: + description: Bytes is the number of bytes written to the + destination. + format: int64 + type: integer + checks: + description: Checks is the number of objects compared and + found up to date. + format: int64 + type: integer + deletes: + description: Deletes is the number of destination objects + deleted (Mirror). + format: int64 + type: integer + differences: + description: Differences is the number of differences found + by verification. + format: int64 + type: integer + elapsedSeconds: + description: ElapsedSeconds is the duration of the copy. + format: int64 + type: integer + errors: + description: Errors is the number of errors. + format: int64 + type: integer + transfers: + description: Transfers is the number of objects written + to the destination. + format: int64 + type: integer + required: + - bytes + - checks + - deletes + - elapsedSeconds + - errors + - transfers + type: object + lastSuccessfulTime: + description: LastSuccessfulTime is when the last successful + Job for this bucket finished. + format: date-time + type: string + lastVerification: + description: LastVerification is the verification outcome when + verification ran. + enum: + - Passed + - Failed + type: string + source: + description: Source bucket. + type: string + required: + - destination + - source + type: object + type: array + x-kubernetes-list-map-keys: + - destination + x-kubernetes-list-type: map + conditions: + description: |- + Conditions: Ready (configuration resolved; reasons include + CredentialsNotAllowed and DestinationConflict), Synced (every bucket + succeeded in the last run on the current configuration), Suspended. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + currentRun: + description: CurrentRun describes the run in progress. + properties: + buckets: + description: Buckets lists the destination buckets taking part + in the run. + items: + type: string + type: array + x-kubernetes-list-type: set + completionTime: + description: CompletionTime is when the run finished. + format: date-time + type: string + id: + description: ID identifies the run; it is part of the Job names. + type: string + reason: + description: Reason is why the run started. + enum: + - Initial + - Scheduled + - Triggered + type: string + result: + description: Result is Succeeded when every bucket of the run + succeeded, else Failed. + enum: + - Succeeded + - Failed + - Cancelled + type: string + specHash: + description: SpecHash identifies the configuration the run started + with. + type: string + startTime: + description: StartTime is when the run started. + format: date-time + type: string + trigger: + description: Trigger is the trigger-sync annotation value that + started the run. + type: string + required: + - id + - reason + - startTime + type: object + lastOperation: + description: LastOperation records the outcome of the last trigger-sync + annotation. + properties: + error: + description: Error contains the error message when Succeeded is + false + type: string + succeeded: + description: Succeeded indicates the operation completed without + error + type: boolean + triggeredAt: + description: TriggeredAt is when the operation was triggered + format: date-time + type: string + type: + description: Type is the operation type (e.g. "Snapshot", "Repair:Blocks", + "Scrub:start") + type: string + type: object + lastRun: + description: LastRun describes the most recent finished run. + properties: + buckets: + description: Buckets lists the destination buckets taking part + in the run. + items: + type: string + type: array + x-kubernetes-list-type: set + completionTime: + description: CompletionTime is when the run finished. + format: date-time + type: string + id: + description: ID identifies the run; it is part of the Job names. + type: string + reason: + description: Reason is why the run started. + enum: + - Initial + - Scheduled + - Triggered + type: string + result: + description: Result is Succeeded when every bucket of the run + succeeded, else Failed. + enum: + - Succeeded + - Failed + - Cancelled + type: string + specHash: + description: SpecHash identifies the configuration the run started + with. + type: string + startTime: + description: StartTime is when the run started. + format: date-time + type: string + trigger: + description: Trigger is the trigger-sync annotation value that + started the run. + type: string + required: + - id + - reason + - startTime + type: object + lastScheduleTime: + description: LastScheduleTime is the start time of the most recent + run. + format: date-time + type: string + lastSuccessfulSyncTime: + description: |- + LastSuccessfulSyncTime is when the most recent run in which every + bucket succeeded finished. + format: date-time + type: string + lastSyncTime: + description: LastSyncTime is when the most recent run finished, whatever + its result. + format: date-time + type: string + nextSyncTime: + description: |- + NextSyncTime is when the next scheduled run starts. Unset without a + schedule or while suspended. + format: date-time + type: string + observedGeneration: + description: ObservedGeneration is the last observed generation. + format: int64 + type: integer + phase: + description: Phase summarizes the state. + enum: + - Pending + - Idle + - Syncing + - Suspended + - Failed + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/garage-operator/templates/_helpers.tpl b/charts/garage-operator/templates/_helpers.tpl index 52b28150..74d1eae8 100644 --- a/charts/garage-operator/templates/_helpers.tpl +++ b/charts/garage-operator/templates/_helpers.tpl @@ -216,3 +216,13 @@ predates the ingress value renders the default (disabled) instead of failing. {{- define "garage-operator.ingressEnabled" -}} {{- if (.Values.ingress | default dict).enabled -}}true{{- end -}} {{- end }} + +{{/* +Whether the GarageReplication controller is enabled (replication.enabled). +Renders "true" or nothing. Nil-safe like garage-operator.ingressEnabled so +`helm upgrade --reuse-values` from a release that predates the value renders +it disabled. +*/}} +{{- define "garage-operator.replicationEnabled" -}} +{{- if (.Values.replication | default dict).enabled -}}true{{- end -}} +{{- end }} diff --git a/charts/garage-operator/templates/clusterrole.yaml b/charts/garage-operator/templates/clusterrole.yaml index 1c490880..d4bc0caf 100644 --- a/charts/garage-operator/templates/clusterrole.yaml +++ b/charts/garage-operator/templates/clusterrole.yaml @@ -119,6 +119,44 @@ rules: - patch - update - watch +# GarageReplication: read-only for the admission webhook's destination +# conflict check +- apiGroups: + - garage.rajsingh.info + resources: + - garagereplications + verbs: + - get + - list + - watch +{{- if include "garage-operator.replicationEnabled" . }} +# GarageReplication controller (replication.enabled): consumes the +# trigger-sync annotation, writes status and runs one Job per bucket and run +- apiGroups: + - garage.rajsingh.info + resources: + - garagereplications + verbs: + - patch +- apiGroups: + - garage.rajsingh.info + resources: + - garagereplications/status + verbs: + - get + - patch + - update +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - watch +{{- end }} # Finalizers for Garage resources - apiGroups: - garage.rajsingh.info diff --git a/charts/garage-operator/templates/deployment.yaml b/charts/garage-operator/templates/deployment.yaml index 3f16ec06..deadbe9a 100644 --- a/charts/garage-operator/templates/deployment.yaml +++ b/charts/garage-operator/templates/deployment.yaml @@ -74,6 +74,12 @@ spec: {{- if include "garage-operator.ingressEnabled" . }} - --enable-ingress {{- end }} + {{- if include "garage-operator.replicationEnabled" . }} + - --enable-replication + {{- with (.Values.replication | default dict).image }} + - --replication-image={{ . }} + {{- end }} + {{- end }} {{- if ne .Values.clusterDomain "cluster.local" }} - --cluster-domain={{ .Values.clusterDomain }} {{- end }} diff --git a/charts/garage-operator/templates/namespace-rbac.yaml b/charts/garage-operator/templates/namespace-rbac.yaml index c6958cf3..fa4e2cd7 100644 --- a/charts/garage-operator/templates/namespace-rbac.yaml +++ b/charts/garage-operator/templates/namespace-rbac.yaml @@ -5,6 +5,7 @@ {{- $releaseNamespace := .Release.Namespace -}} {{- $gatewayAPIEnabled := .Values.gatewayAPI.enabled -}} {{- $ingressEnabled := include "garage-operator.ingressEnabled" . -}} +{{- $replicationEnabled := include "garage-operator.replicationEnabled" . -}} {{- $watchedNamespaces := splitList "," (include "garage-operator.watchedNamespaces" .) -}} {{- range $ns := $watchedNamespaces }} --- @@ -106,6 +107,44 @@ rules: - patch - update - watch +# GarageReplication: read-only for the admission webhook's destination +# conflict check +- apiGroups: + - garage.rajsingh.info + resources: + - garagereplications + verbs: + - get + - list + - watch +{{- if $replicationEnabled }} +# GarageReplication controller (replication.enabled): consumes the +# trigger-sync annotation, writes status and runs one Job per bucket and run +- apiGroups: + - garage.rajsingh.info + resources: + - garagereplications + verbs: + - patch +- apiGroups: + - garage.rajsingh.info + resources: + - garagereplications/status + verbs: + - get + - patch + - update +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - watch +{{- end }} # Finalizers for Garage resources - apiGroups: - garage.rajsingh.info diff --git a/charts/garage-operator/templates/webhooks.yaml b/charts/garage-operator/templates/webhooks.yaml index 662d8ac7..32de1485 100644 --- a/charts/garage-operator/templates/webhooks.yaml +++ b/charts/garage-operator/templates/webhooks.yaml @@ -385,4 +385,25 @@ webhooks: - garagereferencegrants sideEffects: None {{- include "garage-operator.webhookNamespaceSelector" . | nindent 2 }} +- name: vgaragereplication.kb.io + admissionReviewVersions: + - v1 + clientConfig: + service: + name: {{ include "garage-operator.webhookServiceName" . }} + namespace: {{ .Release.Namespace }} + path: /validate-garage-rajsingh-info-v1beta1-garagereplication + failurePolicy: {{ .Values.webhooks.failurePolicy }} + rules: + - apiGroups: + - garage.rajsingh.info + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - garagereplications + sideEffects: None + {{- include "garage-operator.webhookNamespaceSelector" . | nindent 2 }} {{- end }} diff --git a/charts/garage-operator/values.yaml b/charts/garage-operator/values.yaml index eb8486f2..ced157ce 100644 --- a/charts/garage-operator/values.yaml +++ b/charts/garage-operator/values.yaml @@ -342,6 +342,21 @@ ingress: # -- Enable Ingress support (Ingress website exposure) enabled: false +# Cross-cluster object replication (GarageReplication). When enabled, the +# operator renders --enable-replication and RBAC on batch/jobs and on +# GarageReplication status, and runs one rclone Job per bucket and run in the +# GarageReplication's namespace. It is off by default so the operator holds no +# Job write access, and starts no Job informer, unless replication is in use. +# With it disabled, GarageReplication objects are validated but not +# reconciled, and admission warns about it. +replication: + # -- Enable the GarageReplication controller + enabled: false + # -- rclone image for replication Jobs. Empty uses the operator's built-in + # digest-pinned docker.io/rclone/rclone image. Override it to use a mirror + # (air-gapped installs); the image must provide /bin/sh and rclone. + image: "" + # COSI (Container Object Storage Interface) driver configuration cosi: # -- Enable COSI driver for Kubernetes-native object storage provisioning diff --git a/cmd/main.go b/cmd/main.go index 1d119fca..5e6168f9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -189,6 +189,22 @@ func main() { } } + // Replication flag (mirrors --enable-ingress): gates the GarageReplication + // controller, so the operator starts no Job informer and needs no + // batch/jobs RBAC unless replication is in use. + var enableReplication bool + flag.BoolVar(&enableReplication, "enable-replication", false, + "Enable the GarageReplication controller (cross-cluster object replication with rclone Jobs). "+ + "Requires RBAC on batch/jobs (chart value replication.enabled).") + if v := os.Getenv("ENABLE_REPLICATION"); v != "" { + if parsed, err := strconv.ParseBool(v); err == nil { + enableReplication = parsed + } + } + var replicationImage string + flag.StringVar(&replicationImage, "replication-image", os.Getenv("REPLICATION_IMAGE"), + "rclone image used by GarageReplication Jobs. If empty, uses the built-in digest-pinned rclone image.") + opts := zap.Options{ Development: false, } @@ -409,6 +425,19 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "GarageReferenceGrant") os.Exit(1) } + if enableReplication { + if err := (&controller.GarageReplicationReconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Scheme: mgr.GetScheme(), + EventRecorder: mgr.GetEventRecorderFor("garagereplication-controller"), //nolint:staticcheck // core/v1 Events match the chart's existing RBAC + ClusterDomain: clusterDomain, + Image: replicationImage, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "GarageReplication") + os.Exit(1) + } + } // Setup webhooks if webhook server is configured if len(webhookCertPath) > 0 { @@ -457,6 +486,13 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "GarageReferenceGrant") os.Exit(1) } + // Registered even when --enable-replication is off: the chart always + // installs the ValidatingWebhookConfiguration entry, and admission + // then warns that the object is not reconciled. + if err := garagev1beta1.SetupGarageReplicationWebhookWithManager(mgr, enableReplication); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "GarageReplication") + os.Exit(1) + } } // +kubebuilder:scaffold:builder diff --git a/config/crd/bases/garage.rajsingh.info_garagereferencegrants.yaml b/config/crd/bases/garage.rajsingh.info_garagereferencegrants.yaml index 8f1ba94b..c6813c4a 100644 --- a/config/crd/bases/garage.rajsingh.info_garagereferencegrants.yaml +++ b/config/crd/bases/garage.rajsingh.info_garagereferencegrants.yaml @@ -77,10 +77,12 @@ spec: Kind is the resource kind allowed to make cross-namespace references. GarageAdminToken remains in the schema for compatibility, but the static credential path is namespace-local and does not accept cross-namespace grants. + GarageReplication may reference a GarageCluster in another namespace. enum: - GarageKey - GarageBucket - GarageAdminToken + - GarageReplication type: string namespace: description: |- diff --git a/config/crd/bases/garage.rajsingh.info_garagereplications.yaml b/config/crd/bases/garage.rajsingh.info_garagereplications.yaml new file mode 100644 index 00000000..9fe85359 --- /dev/null +++ b/config/crd/bases/garage.rajsingh.info_garagereplications.yaml @@ -0,0 +1,1958 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.22.0 + name: garagereplications.garage.rajsingh.info +spec: + group: garage.rajsingh.info + names: + kind: GarageReplication + listKind: GarageReplicationList + plural: garagereplications + shortNames: + - grepl + singular: garagereplication + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.mode + name: Mode + type: string + - jsonPath: .spec.schedule + name: Schedule + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.lastSuccessfulSyncTime + name: Last Success + type: date + - jsonPath: .status.nextSyncTime + name: Next Sync + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + GarageReplication is the Schema for the garagereplications API. + It replicates objects between two independent Garage clusters. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + GarageReplicationSpec defines the desired state of GarageReplication. + + GarageReplication copies objects between two independent Garage clusters + through their S3 APIs, for replication and migration. It is not a backup: + there are no point-in-time snapshots and no restore, and in mode Mirror a + deletion or an overwrite on the source reaches the destination on the next + run. Each run executes one Kubernetes Job per bucket in the + GarageReplication's namespace, so the Jobs need network access to both S3 + endpoints. Only objects (and their user metadata) are copied: buckets, keys, + permissions, quotas, website and lifecycle configuration are not. Deleting a + GarageReplication never deletes object data on either side. + properties: + buckets: + description: Buckets lists the buckets to replicate, keyed by destination + bucket. + items: + description: ReplicationBucket maps a source bucket to a destination + bucket. + properties: + destination: + description: |- + Destination is the bucket name (global alias) on the destination + cluster. It is the list key, so destinations are unique. + maxLength: 63 + minLength: 3 + pattern: ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ + type: string + source: + description: Source is the bucket name (global alias) on the + source cluster. + maxLength: 63 + minLength: 3 + pattern: ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ + type: string + required: + - destination + - source + type: object + maxItems: 50 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - destination + x-kubernetes-list-type: map + destination: + description: |- + Destination is the cluster objects are written to. Its credentials need + read and write permission on the destination buckets. Destination + buckets must already exist: replication never creates buckets. A + destination bucket can be written by only one GarageReplication. + properties: + clusterRef: + description: |- + ClusterRef references a GarageCluster (operator-managed or a + management handle). The S3 endpoint and region are resolved from it. + A cross-namespace reference requires a GarageReferenceGrant in the + cluster's namespace that allows kind GarageReplication. + properties: + kubeConfigSecretRef: + description: |- + KubeConfigSecretRef is reserved for a future remote Kubernetes client integration. + It is currently rejected by admission because the operator does not use it. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + name: + description: Name of the GarageCluster resource. + type: string + namespace: + description: |- + Namespace of the GarageCluster. Defaults to the referencing resource's namespace. + Cross-namespace references require a GarageReferenceGrant where supported by + the owning resource. GarageNode and GarageAdminToken reject them. + type: string + required: + - name + type: object + credentialsSecretRef: + description: |- + CredentialsSecretRef references a Secret in the GarageReplication's + namespace holding an S3 access key. The Secret must carry the label + garage.rajsingh.info/replication-credentials: "true" (for a GarageKey, + set it in spec.secretTemplate.labels). The access key ID is sent to the + endpoint, so without this opt-in the right to create a GarageReplication + would amount to reading any Secret, or using any other application's + key. The default data keys match the Secret generated for a GarageKey. + properties: + accessKeyIdKey: + default: access-key-id + description: AccessKeyIDKey is the data key holding the access + key ID. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + name: + description: Name of the Secret in the GarageReplication's + namespace. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + secretAccessKeyKey: + default: secret-access-key + description: SecretAccessKeyKey is the data key holding the + secret access key. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: accessKeyIdKey and secretAccessKeyKey must be different + rule: '(has(self.accessKeyIdKey) ? self.accessKeyIdKey : ''access-key-id'') + != (has(self.secretAccessKeyKey) ? self.secretAccessKeyKey + : ''secret-access-key'')' + endpoint: + description: |- + Endpoint is an S3 endpoint of any Garage cluster, inside or outside + Kubernetes. + properties: + caBundleRef: + description: |- + CABundleRef references a ConfigMap holding a PEM CA bundle used, in + addition to the system trust store, to verify the endpoint's TLS + certificate. + properties: + key: + default: ca.crt + description: Key of the ConfigMap holding the PEM bundle. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + name: + description: Name of the ConfigMap. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + region: + default: garage + description: Region is the Garage s3_region used for request + signing. + maxLength: 128 + minLength: 1 + pattern: ^[A-Za-z0-9_.-]+$ + type: string + url: + description: |- + URL of the S3 API: scheme and host (with an optional port) only, e.g. + https://s3.example.com or http://garage.ns.svc:3900. Buckets are + addressed path-style below it, so it has no path, query, fragment or + credentials. + maxLength: 2048 + minLength: 1 + pattern: ^https?://[^\s/?#@]+/?$ + type: string + required: + - url + type: object + required: + - credentialsSecretRef + type: object + x-kubernetes-validations: + - message: exactly one of clusterRef or endpoint must be set + rule: has(self.clusterRef) != has(self.endpoint) + - message: clusterRef.kubeConfigSecretRef is not supported + rule: '!has(self.clusterRef) || !has(self.clusterRef.kubeConfigSecretRef)' + failedJobsHistoryLimit: + default: 3 + description: FailedJobsHistoryLimit is the number of finished failed + Jobs to keep. + format: int32 + maximum: 50 + minimum: 0 + type: integer + mirror: + description: |- + Mirror configures mode Mirror. Required when mode is Mirror and + rejected otherwise. + properties: + maxDeleteSize: + description: |- + MaxDeleteSize caps the total size of the destination objects one run + may delete per bucket, in bytes with an optional K, M, G or T suffix + (binary multiples: KiB, MiB, GiB, TiB), e.g. "10G". Once reached the run + stops and fails with reason MaxDeletesExceeded, like maxDeletes. Empty + means no size cap. + maxLength: 32 + pattern: ^[0-9]+(\.[0-9]+)?[KMGTkmgt]?$ + type: string + maxDeletes: + description: |- + MaxDeletes caps the number of destination objects one run may delete + per bucket. Once reached the run stops and fails with reason + MaxDeletesExceeded; deletions done before that point are kept. + format: int64 + minimum: 1 + type: integer + required: + - maxDeletes + type: object + mode: + default: Copy + description: |- + Mode controls whether changes on the source can remove data on the + destination. + - Copy: copy new and changed objects. Source deletions are not + propagated. Changed objects are overwritten on the destination. + - Mirror: the destination reflects the source, including deletions, + capped by mirror.maxDeletes per bucket and run. The first run waits + for the trigger-sync annotation. + enum: + - Copy + - Mirror + type: string + podTemplate: + description: PodTemplate customizes the replication Pods. + properties: + affinity: + description: Affinity for the Pods. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + annotations: + additionalProperties: + type: string + description: Annotations added to the Pods. + maxProperties: 64 + type: object + labels: + additionalProperties: + type: string + description: Labels added to the Pods. + maxProperties: 64 + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector for the Pods. + maxProperties: 64 + type: object + priorityClassName: + description: PriorityClassName for the Pods. + maxLength: 253 + type: string + tolerations: + description: Tolerations for the Pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + resources: + description: Resources for the replication container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + retry: + description: Retry bounds retries and duration of each per-bucket + Job. + properties: + activeDeadlineSeconds: + default: 21600 + description: ActiveDeadlineSeconds bounds the duration of a per-bucket + Job. + format: int64 + minimum: 60 + type: integer + backoffLimit: + default: 2 + description: |- + BackoffLimit is the number of Pod retries of a per-bucket Job. Each + retry restarts the bucket's comparison; already copied objects are + skipped. + format: int32 + maximum: 10 + minimum: 0 + type: integer + type: object + schedule: + description: |- + Schedule is a cron expression (standard 5-field syntax, descriptors + such as @hourly, optional CRON_TZ= prefix; UTC otherwise). Without a + schedule, runs happen only through the garage.rajsingh.info/trigger-sync + annotation (one-shot mode, e.g. for migrations). + maxLength: 128 + type: string + source: + description: |- + Source is the cluster objects are read from. Its credentials only need + read permission on the source buckets. + properties: + clusterRef: + description: |- + ClusterRef references a GarageCluster (operator-managed or a + management handle). The S3 endpoint and region are resolved from it. + A cross-namespace reference requires a GarageReferenceGrant in the + cluster's namespace that allows kind GarageReplication. + properties: + kubeConfigSecretRef: + description: |- + KubeConfigSecretRef is reserved for a future remote Kubernetes client integration. + It is currently rejected by admission because the operator does not use it. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + name: + description: Name of the GarageCluster resource. + type: string + namespace: + description: |- + Namespace of the GarageCluster. Defaults to the referencing resource's namespace. + Cross-namespace references require a GarageReferenceGrant where supported by + the owning resource. GarageNode and GarageAdminToken reject them. + type: string + required: + - name + type: object + credentialsSecretRef: + description: |- + CredentialsSecretRef references a Secret in the GarageReplication's + namespace holding an S3 access key. The Secret must carry the label + garage.rajsingh.info/replication-credentials: "true" (for a GarageKey, + set it in spec.secretTemplate.labels). The access key ID is sent to the + endpoint, so without this opt-in the right to create a GarageReplication + would amount to reading any Secret, or using any other application's + key. The default data keys match the Secret generated for a GarageKey. + properties: + accessKeyIdKey: + default: access-key-id + description: AccessKeyIDKey is the data key holding the access + key ID. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + name: + description: Name of the Secret in the GarageReplication's + namespace. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + secretAccessKeyKey: + default: secret-access-key + description: SecretAccessKeyKey is the data key holding the + secret access key. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: accessKeyIdKey and secretAccessKeyKey must be different + rule: '(has(self.accessKeyIdKey) ? self.accessKeyIdKey : ''access-key-id'') + != (has(self.secretAccessKeyKey) ? self.secretAccessKeyKey + : ''secret-access-key'')' + endpoint: + description: |- + Endpoint is an S3 endpoint of any Garage cluster, inside or outside + Kubernetes. + properties: + caBundleRef: + description: |- + CABundleRef references a ConfigMap holding a PEM CA bundle used, in + addition to the system trust store, to verify the endpoint's TLS + certificate. + properties: + key: + default: ca.crt + description: Key of the ConfigMap holding the PEM bundle. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + name: + description: Name of the ConfigMap. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + region: + default: garage + description: Region is the Garage s3_region used for request + signing. + maxLength: 128 + minLength: 1 + pattern: ^[A-Za-z0-9_.-]+$ + type: string + url: + description: |- + URL of the S3 API: scheme and host (with an optional port) only, e.g. + https://s3.example.com or http://garage.ns.svc:3900. Buckets are + addressed path-style below it, so it has no path, query, fragment or + credentials. + maxLength: 2048 + minLength: 1 + pattern: ^https?://[^\s/?#@]+/?$ + type: string + required: + - url + type: object + required: + - credentialsSecretRef + type: object + x-kubernetes-validations: + - message: exactly one of clusterRef or endpoint must be set + rule: has(self.clusterRef) != has(self.endpoint) + - message: clusterRef.kubeConfigSecretRef is not supported + rule: '!has(self.clusterRef) || !has(self.clusterRef.kubeConfigSecretRef)' + successfulJobsHistoryLimit: + default: 1 + description: |- + SuccessfulJobsHistoryLimit is the number of finished successful Jobs to + keep. + format: int32 + maximum: 50 + minimum: 0 + type: integer + suspend: + description: |- + Suspend stops starting new runs. A run already in progress finishes; + delete its Jobs to stop it immediately. + type: boolean + transfer: + description: Transfer tunes the copy. + properties: + bandwidthLimit: + description: |- + BandwidthLimit caps the transfer rate of each per-bucket Job in bytes + per second, with an optional K, M, G suffix (binary multiples: KiB/s, + MiB/s, GiB/s), e.g. "50M". Empty means unlimited. + maxLength: 32 + pattern: ^[0-9]+(\.[0-9]+)?[KMGkmg]?$ + type: string + checkers: + default: 8 + description: Checkers is the number of objects compared in parallel + per bucket. + format: int32 + maximum: 128 + minimum: 1 + type: integer + concurrency: + default: 4 + description: Concurrency is the number of objects transferred + in parallel per bucket. + format: int32 + maximum: 64 + minimum: 1 + type: integer + maxConcurrentBuckets: + default: 1 + description: MaxConcurrentBuckets is the number of buckets replicated + in parallel. + format: int32 + maximum: 50 + minimum: 1 + type: integer + type: object + verification: + default: None + description: |- + Verification runs a comparison of source and destination after each + successful copy. Check compares sizes and MD5 hashes (size only for + multipart objects without a stored MD5) and fails the run on any + difference. + enum: + - None + - Check + type: string + required: + - buckets + - destination + - source + type: object + x-kubernetes-validations: + - message: mirror must be set if and only if mode is Mirror + rule: has(self.mirror) == (has(self.mode) && self.mode == 'Mirror') + - message: 'a bucket cannot be replicated onto itself: source and destination + are the same endpoint' + rule: '!(has(self.source.endpoint) && has(self.destination.endpoint) + && self.source.endpoint.url == self.destination.endpoint.url) || self.buckets.all(b, + b.source != b.destination)' + status: + description: GarageReplicationStatus defines the observed state of GarageReplication. + properties: + buckets: + description: Buckets reports per-bucket results, keyed by destination + bucket. + items: + description: ReplicationBucketStatus reports the last result for + one bucket. + properties: + activeJob: + description: ActiveJob is the Job currently replicating this + bucket. + type: string + destination: + description: Destination bucket; the list key. + type: string + hold: + description: |- + Hold is set when this bucket must not run again without a person + asking. Scheduled runs skip a held bucket. Only the trigger-sync + annotation releases it; editing the spec does not. + enum: + - SourceEmpty + - MaxDeletesExceeded + - AwaitingTrigger + type: string + lastCompletionTime: + description: LastCompletionTime is when LastJob finished. + format: date-time + type: string + lastError: + description: LastError is a fixed reason for a failed LastJob. + enum: + - AuthFailed + - BucketNotFound + - EndpointUnreachable + - SourceEmpty + - MaxDeletesExceeded + - VerificationFailed + - DeadlineExceeded + - TransferErrors + - Unknown + type: string + lastJob: + description: LastJob is the last finished Job recorded for this + bucket. + type: string + lastMode: + description: |- + LastMode is the mode LastJob ran in. A Mirror bucket whose last Job + did not run in Mirror mode waits for a trigger-sync (hold + AwaitingTrigger) before its first Mirror run. + type: string + lastResult: + description: LastResult is the outcome of LastJob. + enum: + - Succeeded + - Failed + - Cancelled + type: string + lastRunId: + description: LastRunID is the run of LastJob. + type: string + lastStatistics: + description: |- + LastStatistics are the copy engine's counters for LastJob. Unset when + the engine reported none (e.g. the Pod was killed). + properties: + bytes: + description: Bytes is the number of bytes written to the + destination. + format: int64 + type: integer + checks: + description: Checks is the number of objects compared and + found up to date. + format: int64 + type: integer + deletes: + description: Deletes is the number of destination objects + deleted (Mirror). + format: int64 + type: integer + differences: + description: Differences is the number of differences found + by verification. + format: int64 + type: integer + elapsedSeconds: + description: ElapsedSeconds is the duration of the copy. + format: int64 + type: integer + errors: + description: Errors is the number of errors. + format: int64 + type: integer + transfers: + description: Transfers is the number of objects written + to the destination. + format: int64 + type: integer + required: + - bytes + - checks + - deletes + - elapsedSeconds + - errors + - transfers + type: object + lastSuccessfulTime: + description: LastSuccessfulTime is when the last successful + Job for this bucket finished. + format: date-time + type: string + lastVerification: + description: LastVerification is the verification outcome when + verification ran. + enum: + - Passed + - Failed + type: string + source: + description: Source bucket. + type: string + required: + - destination + - source + type: object + type: array + x-kubernetes-list-map-keys: + - destination + x-kubernetes-list-type: map + conditions: + description: |- + Conditions: Ready (configuration resolved; reasons include + CredentialsNotAllowed and DestinationConflict), Synced (every bucket + succeeded in the last run on the current configuration), Suspended. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + currentRun: + description: CurrentRun describes the run in progress. + properties: + buckets: + description: Buckets lists the destination buckets taking part + in the run. + items: + type: string + type: array + x-kubernetes-list-type: set + completionTime: + description: CompletionTime is when the run finished. + format: date-time + type: string + id: + description: ID identifies the run; it is part of the Job names. + type: string + reason: + description: Reason is why the run started. + enum: + - Initial + - Scheduled + - Triggered + type: string + result: + description: Result is Succeeded when every bucket of the run + succeeded, else Failed. + enum: + - Succeeded + - Failed + - Cancelled + type: string + specHash: + description: SpecHash identifies the configuration the run started + with. + type: string + startTime: + description: StartTime is when the run started. + format: date-time + type: string + trigger: + description: Trigger is the trigger-sync annotation value that + started the run. + type: string + required: + - id + - reason + - startTime + type: object + lastOperation: + description: LastOperation records the outcome of the last trigger-sync + annotation. + properties: + error: + description: Error contains the error message when Succeeded is + false + type: string + succeeded: + description: Succeeded indicates the operation completed without + error + type: boolean + triggeredAt: + description: TriggeredAt is when the operation was triggered + format: date-time + type: string + type: + description: Type is the operation type (e.g. "Snapshot", "Repair:Blocks", + "Scrub:start") + type: string + type: object + lastRun: + description: LastRun describes the most recent finished run. + properties: + buckets: + description: Buckets lists the destination buckets taking part + in the run. + items: + type: string + type: array + x-kubernetes-list-type: set + completionTime: + description: CompletionTime is when the run finished. + format: date-time + type: string + id: + description: ID identifies the run; it is part of the Job names. + type: string + reason: + description: Reason is why the run started. + enum: + - Initial + - Scheduled + - Triggered + type: string + result: + description: Result is Succeeded when every bucket of the run + succeeded, else Failed. + enum: + - Succeeded + - Failed + - Cancelled + type: string + specHash: + description: SpecHash identifies the configuration the run started + with. + type: string + startTime: + description: StartTime is when the run started. + format: date-time + type: string + trigger: + description: Trigger is the trigger-sync annotation value that + started the run. + type: string + required: + - id + - reason + - startTime + type: object + lastScheduleTime: + description: LastScheduleTime is the start time of the most recent + run. + format: date-time + type: string + lastSuccessfulSyncTime: + description: |- + LastSuccessfulSyncTime is when the most recent run in which every + bucket succeeded finished. + format: date-time + type: string + lastSyncTime: + description: LastSyncTime is when the most recent run finished, whatever + its result. + format: date-time + type: string + nextSyncTime: + description: |- + NextSyncTime is when the next scheduled run starts. Unset without a + schedule or while suspended. + format: date-time + type: string + observedGeneration: + description: ObservedGeneration is the last observed generation. + format: int64 + type: integer + phase: + description: Phase summarizes the state. + enum: + - Pending + - Idle + - Syncing + - Suspended + - Failed + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 755097ef..18720034 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -8,6 +8,7 @@ resources: - bases/garage.rajsingh.info_garagenodes.yaml - bases/garage.rajsingh.info_garageadmintokens.yaml - bases/garage.rajsingh.info_garagereferencegrants.yaml +- bases/garage.rajsingh.info_garagereplications.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 16b175d6..df6b2346 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -77,6 +77,16 @@ rules: - get - list - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - watch - apiGroups: - garage.rajsingh.info resources: @@ -111,6 +121,7 @@ rules: - garageclusters/status - garagekeys/status - garagenodes/status + - garagereplications/status verbs: - get - patch @@ -130,6 +141,15 @@ rules: verbs: - patch - update +- apiGroups: + - garage.rajsingh.info + resources: + - garagereplications + verbs: + - get + - list + - patch + - watch - apiGroups: - gateway.networking.k8s.io resources: diff --git a/config/samples/garage_v1beta1_garagereplication.yaml b/config/samples/garage_v1beta1_garagereplication.yaml new file mode 100644 index 00000000..df83ddb6 --- /dev/null +++ b/config/samples/garage_v1beta1_garagereplication.yaml @@ -0,0 +1,43 @@ +# Replicates two buckets from an operator-managed GarageCluster to a Garage +# cluster reachable at an external S3 endpoint (another Kubernetes cluster, +# a Helm install, bare metal...). This is replication, not a backup: there are +# no point-in-time snapshots. +# +# Credentials are read from Secrets in this namespace. Each Secret must carry +# the label garage.rajsingh.info/replication-credentials: "true"; for a +# GarageKey, set it in spec.secretTemplate.labels. The default data keys +# (access-key-id / secret-access-key) match the GarageKey Secret. +apiVersion: garage.rajsingh.info/v1beta1 +kind: GarageReplication +metadata: + name: production-to-dr +spec: + source: + clusterRef: + name: garage + credentialsSecretRef: + name: replication-reader # read on medias and pg-dumps + destination: + endpoint: + url: https://s3.dr.example.com + region: garage + credentialsSecretRef: + name: dr-replication-writer # read + write on the destination buckets + buckets: + - source: medias + destination: medias + - source: pg-dumps + destination: pg-dumps-dr + schedule: "*/15 * * * *" + # Copy never deletes destination objects. Mirror (with mirror.maxDeletes) + # propagates deletions; its first run waits for the trigger-sync annotation. + mode: Copy + transfer: + concurrency: 4 + bandwidthLimit: "50M" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 512Mi diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 3e10cc29..5f174d9e 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -9,6 +9,7 @@ resources: - garage_v1beta1_garagenode_maintenance.yaml - garage_v1beta1_garageadmintoken.yaml - garage_v1beta1_garagereferencegrant.yaml +- garage_v1beta1_garagereplication.yaml - garage_v1beta2_garagecluster_auto_per_node.yaml - garage_v1beta2_garagecluster_node_local_pools.yaml - garage_v1beta2_garagecluster_eject_to_manual.yaml diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index e9530be8..5abd5b55 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -274,6 +274,26 @@ webhooks: resources: - garagereferencegrants sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-garage-rajsingh-info-v1beta1-garagereplication + failurePolicy: Fail + name: vgaragereplication.kb.io + rules: + - apiGroups: + - garage.rajsingh.info + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - garagereplications + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/docs/design/2026-11-09-garage-replication-design.md b/docs/design/2026-11-09-garage-replication-design.md new file mode 100644 index 00000000..1db207e3 --- /dev/null +++ b/docs/design/2026-11-09-garage-replication-design.md @@ -0,0 +1,478 @@ +# GarageReplication: cross-cluster object replication + +Status: **implemented, behind `--enable-replication`.** The work lands in two +stacked changes, merged together: + +1. **API**: the `GarageReplication` CRD, its schema validation, the admission + webhook and the `GarageReferenceGrant` kind. +2. **Controller**: the rclone Job engine, the `--enable-replication` gate, CI + coverage of the engine (a contract job running the Job script in the + pinned rclone image, and a Kind e2e scenario) and the user guide. + +"Review decisions" at the end records what changed after the first review of +the combined proposal (#580). + +## Problem + +Garage replicates data between the nodes of **one** cluster (the layout and +`replication.factor`). It has nothing that replicates between two +**independent** clusters: no bucket replication, no change feed, no bucket +notifications. Users want two things from the operator: + +1. **Replication**: keep a warm copy of a production cluster's buckets on + another cluster. +2. **Migration**: move objects to a new cluster with a short write freeze. + +Today they write their own CronJobs around `rclone` or `aws s3 sync`. Each team +works out credentials, path-style addressing, regions, deletion safety and +monitoring again. + +### Not a backup + +rclone `copy`/`sync` is replication. There are no point-in-time snapshots and +no restore, and in `Mirror` mode an accidental deletion or overwrite on the +source reaches the destination on the next run. Bucket backups need a snapshot +tool (kopia, restic) and a separate design: the existing Kubernetes backup +operators (k8up, VolSync, Kopiur) only take PVCs as sources. The API and the +docs do not use backup wording, and there is no archive mode. + +## Constraints and upstream semantics + +- **S3 only.** The two clusters share no RPC secret, layout or metadata. + Everything goes through the public S3 API. Garage's internal tables (keys, + bucket IDs, aliases, permissions, quotas, website config, lifecycle, + `node_key`) are **not** copied, and copying them between clusters is out of + scope. +- **No change feed.** Garage has no event notifications or replication log. + Every run lists both sides. A run is incremental in what it **transfers**, + not in what it **lists**: listing a bucket costs O(objects) on each run. +- **No object versioning.** Garage does not implement S3 versioning or object + lock. +- **Checksums.** For a single-part upload Garage's ETag is the hex MD5 of the + body, so size + MD5 is a reliable comparison. A multipart ETag is not an MD5 + of the object (`md5-of-part-md5s-N`). For multipart objects that rclone did + not upload itself, the comparison falls back to size only. +- **Signing.** SigV4 needs the cluster's `s3_api.s3_region` (default `garage`) + and path-style addressing. +- **Not tied to the operator's cluster.** Either side may be a GarageCluster in + this Kubernetes cluster, a Garage in another Kubernetes cluster, or a Garage + outside Kubernetes (Helm, bare metal). The copy runs where the + `GarageReplication` lives and only needs network reach to both S3 endpoints. +- **Repository conventions.** The new CRD is `v1beta1`, like every CRD except + GarageCluster. Cross-namespace `clusterRef` goes through a + `GarageReferenceGrant`. Optional features are off behind a flag. One-shot + actions use annotations that are removed once handled, with the result in + `status.lastOperation`. + +## Replication engine + +| Option | For | Against | +|---|---|---| +| **rclone in a Job** (selected) | Mature S3 → S3 `copy` / `sync` / `check`: size+MD5 comparison (`--checksum`), concurrency, bandwidth limit, retries, a deletion cap (`--max-delete`), metadata copy (`-M`), JSON logs with a final stats block. Configured through env vars, so credentials never appear in args. | A third-party image the operator does not build: pinned by digest, overridable for air-gapped installs. Stats come from parsing its output. | +| `aws s3 sync` / `mc mirror` | Common tools. | `aws s3 sync` compares size and mtime only and cannot cap deletes. `mc mirror` is AGPL with weaker deletion guards. | +| Native Go (aws-sdk-go-v2) | One image, typed stats. | Re-implements listing/diff, multipart streaming, retries and bandwidth limits: a lot of data-path code to own. | + +**Decision: rclone, run as a Kubernetes Job**, upstream `rclone/rclone` image +pinned by digest (`--replication-image` to override). The CRD exposes no +rclone flags, so the engine can change later without an API change. + +How the Job is configured: + +- Remotes come from env vars (`RCLONE_CONFIG_SRC_*`, `RCLONE_CONFIG_DST_*`, + provider `Other`, path style). The access key ID and secret use + `valueFrom.secretKeyRef`; the operator never reads secret **values**. +- Base flags: `--checksum`, `-M`, `--s3-no-check-bucket` (never create + buckets), JSON logs at NOTICE level. +- A small POSIX `sh` script (busybox in the rclone image) runs rclone and + writes one bounded `key=value` line to `/dev/termination-log`: exit code, + fixed failure reason, verification result and rclone counters. No object + name or rclone error text reaches status. +- Failures are classified by rclone's exit code first, as measured against + rclone 1.75.2 by the contract tests: 3 (directory not found, a missing source + bucket) is `BucketNotFound`; 5 (retryable errors left after the retries: + connection refused, timeouts, 5xx) is `EndpointUnreachable`; in `Mirror`, 7 + (fatal error, which only `--max-delete` and `--max-delete-size` raise with + these flags) is `MaxDeletesExceeded`. Exit code 1 covers both an + authentication failure and a missing destination bucket (a PUT answered + 404 `NoSuchBucket`), so the log decides those two; anything else is + `TransferErrors`. +- API sizes (`bandwidthLimit`, `mirror.maxDeleteSize`) are bytes; a bare number + gets a `B` suffix because rclone reads it as KiB. + +## API + +```yaml +apiVersion: garage.rajsingh.info/v1beta1 +kind: GarageReplication +metadata: + name: production-to-dr + namespace: storage +spec: + source: + # Exactly one of clusterRef | endpoint. + clusterRef: # operator-managed GarageCluster or management handle + name: garage-prod + namespace: storage # cross-namespace needs a GarageReferenceGrant + credentialsSecretRef: # in the GarageReplication namespace; must opt in + name: prod-replication-reader + # accessKeyIdKey: access-key-id (default; same as GarageKey Secrets) + # secretAccessKeyKey: secret-access-key (default) + destination: + endpoint: # any Garage, anywhere + url: https://s3.dr.example.com # scheme + host[:port] only + region: garage # default "garage" + caBundleRef: # optional: ConfigMap key with a PEM CA + name: dr-ca + key: ca.crt # default + credentialsSecretRef: + name: dr-replication-writer + + buckets: # list-map keyed by destination; 1..50 + - source: medias + destination: medias + - source: pg-dumps + destination: pg-dumps-dr + + schedule: "*/15 * * * *" # optional; empty = runs only on trigger-sync + suspend: false + + mode: Copy # Copy (default) | Mirror + mirror: # required with mode Mirror, rejected otherwise + maxDeletes: 1000 # per bucket per run; the run stops once reached + maxDeleteSize: "10G" # optional cap on deleted bytes per bucket per run + + verification: None # None | Check (rclone check after the copy) + + transfer: + concurrency: 4 + checkers: 8 + bandwidthLimit: "50M" # bytes/s with K, M, G suffix; empty = unlimited + maxConcurrentBuckets: 1 + + retry: + backoffLimit: 2 + activeDeadlineSeconds: 21600 + + resources: {} + podTemplate: {} # labels, annotations, nodeSelector, tolerations, + # affinity, priorityClassName + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 +``` + +API decisions: + +- **`mode`, not `deletionPolicy`.** `GarageBucket.spec.deletionPolicy` already + means "what happens to the Garage bucket when the CR is deleted". Deleting a + `GarageReplication` never touches object data, in either mode. +- **Buckets are a list-map keyed by `destination`.** `destination` is required + (a schema default cannot copy another field), so the API server itself + rejects two entries writing the same bucket, and server-side apply merges + entries by destination. The same source may feed several destinations. + `status.buckets` uses the same key. +- **Credentials must opt in.** The Job sends the access key ID to the + endpoint, which the creator of the `GarageReplication` chooses. If any Secret + could be named, `create garagereplications` would amount to `get secrets`. + The controller therefore only uses a Secret labelled + `garage.rajsingh.info/replication-credentials: "true"`. Only someone who can + write the Secret can add that label. Secrets generated for a `GarageKey` are + not exempt: otherwise anyone allowed to create a `GarageReplication` could + use another application's key in the namespace and copy its buckets to an + endpoint of their choice. A `GarageKey` opts in through + `spec.secretTemplate.labels`. The check runs in the controller, before + any Job is created and on every run, not in admission: the Secret may not + exist yet when the `GarageReplication` is applied, and the label can be + removed later. A Secret that does not opt in sets `Ready=False` with reason + `CredentialsNotAllowed`. +- **CA bundles come from ConfigMaps only.** A CA certificate is public, and a + Secret reference would reopen the exfiltration path above. +- **`clusterRef` gets the endpoint and region** from the referenced + GarageCluster through `ResolveS3Endpoint` and its `s3Api.region`, which + covers managed clusters and management handles. `GarageReferenceGrant`'s + `from.kind` enum gains `GarageReplication`. +- **Engine-neutral, no escape hatch.** No raw rclone flags and no `extraArgs`: + they would allow destructive or credential-leaking behaviour (`--delete-*`, + `--dump headers`) outside the validated model. No TLS skip-verify option. +- **Typed status.** `phase`, run reasons, results, verification results, holds + and failure reasons are enums. + +### Validation + +The API is correct without the webhook. The CRD schema enforces: + +- S3 bucket names (pattern, 3–63 characters) on both `source` and + `destination`, and unique destinations (list-map key). +- Exactly one of `clusterRef` / `endpoint` per side, and no + `clusterRef.kubeConfigSecretRef`. +- `endpoint.url`: `http(s)://host[:port]` with no path, query, fragment or + credentials; `region` format. +- Secret and ConfigMap names (DNS-1123 subdomain) and data keys, and different + `accessKeyIdKey` / `secretAccessKeyKey`. +- `mirror` set if and only if `mode: Mirror`; `maxDeletes ≥ 1`, so Mirror + cannot be uncapped by omission. +- A bucket replicated onto itself between two identical endpoint URLs. The + comparison is exact: Kubernetes 1.25 costs `lowerAscii()` as unbounded and + rejects the CRD, so normalization is left to the webhook. +- Enums, bounds and `maxLength` on every string. + +The webhook adds what needs context or cannot be written in CEL: + +- the `GarageReferenceGrant` of a cross-namespace `clusterRef`; +- **destination exclusivity**: a destination bucket on a given destination + cluster can be written by only one `GarageReplication`, whatever the + namespace or mode (two Mirrors would delete each other's objects). The + destination cluster is identified by its `clusterRef` (namespace defaulted) + or by the normalized URL (lowercase, no trailing slash, no default port). + The older object keeps the bucket. The check uses the informer cache, so it + sees only the namespaces the operator watches. It is cluster-wide and first + come, first served: a namespace can claim a destination bucket name before + another one, and the error names the other object only when it is in the + same namespace; +- the self-loop when a `clusterRef` namespace is implicit or URLs differ by + case, a trailing slash or a default port (the CEL rule on unbounded `ClusterReference` + strings exceeds the cost budget); +- the cron expression (`robfig/cron/v3`, the CronJob parser) and Pod label and + annotation keys; +- warnings for `Mirror`, for same-cluster replication, and that the object is + not reconciled while `--enable-replication` is off. + +The controller re-runs the same checks before starting Jobs, including +destination exclusivity (two objects admitted concurrently, or created while +the webhook was unavailable), and also compares resolved URLs, so a +`clusterRef` and an `endpoint` that reach the same cluster are detected. The +loser gets `Ready=False` with reason `DestinationConflict` and runs nothing. + +## Deletion semantics + +| Mode | rclone | Source delete | Source overwrite | Destination-only objects | +|---|---|---|---|---| +| `Copy` (default) | `copy` | kept on destination | destination overwritten | kept | +| `Mirror` | `sync --max-delete N` | deleted on destination | destination overwritten | **deleted** | + +- Nothing destructive is on by default. `Copy` still **overwrites** changed + objects; the docs say this plainly. +- **First Mirror run waits for a person.** A Mirror bucket whose last Job did + not run in Mirror mode (a new replication, or one switched from Copy) gets + `hold: AwaitingTrigger`; switching back to Copy drops it. The first deletions happen only after a + `trigger-sync` annotation, once someone has checked source and destination. +- **Empty-source guard.** Before `sync`, the script checks whether the source + lists empty while the destination does not. If so it refuses (exit 64, + reason `SourceEmpty`): a wrong bucket, a key with an empty view or a wiped + source would otherwise empty the destination. +- **`maxDeletes`, `maxDeleteSize`.** rclone deletes up to N objects (and, when + set, up to that many bytes), then aborts (exit 65, reason + `MaxDeletesExceeded`). +- **Safety hold.** `podFailurePolicy` fails the Job on exit 64 and 65 without + retrying, because a retry would delete another N objects. The bucket is then + held (`status.buckets[].hold`) and scheduled runs skip it. **Only the + `trigger-sync` annotation releases a hold.** Editing the spec does not: an + unrelated edit (bandwidth, resources) must not re-arm deletions. +- Switching from `Mirror` to `Copy`, or removing a bucket from the spec, + cancels a running Job of that bucket instead of letting it finish. + +## Reconciliation + +A standalone `GarageReplicationReconciler`. It only **reads** GarageClusters +(for endpoints) and `Owns()` the `batch/v1` Jobs it creates. It never calls +the Admin API. + +Operator-created Jobs, not a CronJob: scheduled runs, `trigger-sync` runs, +per-bucket Jobs with a parallelism cap and the cancel-on-downgrade rule would +otherwise be split between the CronJob controller and the operator. One code +path that owns every Job keeps "at most one run per bucket" a single +invariant. + +Loop: + +1. Resolve references: endpoints, Secrets and their opt-in, CA ConfigMaps, + reference grants, destination exclusivity. On failure set `Ready=False` + with a fixed reason and create no Job. +2. Record finished Jobs into `status.buckets` from their termination message; + a Job deleted by a person is recorded as `Cancelled`. Prune history. +3. Cancel Jobs per the downgrade rule. +4. Decide whether a run is due: `trigger-sync` (also while suspended, and the + only way to release holds), or the cron time. Missed runs collapse into one. +5. Persist the run in `status.currentRun` **before** creating Jobs, then + remove the trigger annotation. Job names + `--` are deterministic, so a retried + reconcile never creates a second Job. +6. Close the run once every participating bucket has a result. + +Job shape: `restartPolicy: Never`, `backoffLimit`, `activeDeadlineSeconds`, +owner reference to the `GarageReplication`. Pod security: non-root, +read-only root filesystem, all capabilities dropped, RuntimeDefault seccomp, +no service account token. rclone stops on SIGTERM; single-part PUTs are +atomic, and an interrupted multipart upload is cleaned up by an +`abortIncompleteMultipartUpload` lifecycle rule. + +No finalizer: owner references garbage-collect the Jobs, and object data on +either side is never touched. + +**Feature gate.** `--enable-replication` (chart `replication.enabled`, default +`false`) controls the controller, the Job informer and the `batch/jobs` RBAC. +While it is off, the webhook warns that new objects will not be reconciled. + +## Status and observability + +```yaml +status: + observedGeneration: 4 + phase: Syncing # Pending | Idle | Syncing | Suspended | Failed + lastScheduleTime: ... + lastSyncTime: ... # last run that finished, any result + lastSuccessfulSyncTime: ... # last run where every bucket succeeded + nextSyncTime: ... + currentRun: { id: t5bz2c, reason: Scheduled, startTime: ..., buckets: [medias] } + lastRun: { id: t5bz0g, reason: Scheduled, result: Succeeded, completionTime: ... } + buckets: # list-map keyed by destination + - destination: medias + source: medias + activeJob: production-to-dr-t5bz2c-1f2e3d4c + lastResult: Succeeded # Succeeded | Failed | Cancelled + lastMode: Copy # mode of the last Job; Mirror without a Mirror Job waits for a trigger + lastStatistics: { transfers: 1250, bytes: 1073741824, checks: 98012, deletes: 0, errors: 0, elapsedSeconds: 41 } + lastVerification: Passed + hold: "" # SourceEmpty | MaxDeletesExceeded | AwaitingTrigger + lastError: "" # fixed reason, never raw rclone output + lastOperation: { type: TriggerSync, triggeredAt: ..., succeeded: true } + conditions: + - type: Ready # e.g. CredentialsNotAllowed, DestinationConflict + - type: Synced # NeverSynced | SyncFailed | SafetyHold | ConfigChanged | Synced + - type: Suspended +``` + +- Statistics are rclone's counters for the **last run** of each bucket. There + is no replication lag (no change feed). A run whose termination message + cannot be parsed gets `lastStatistics: null`, never zeros. +- `lastError` is a fixed reason: `AuthFailed`, `BucketNotFound`, + `EndpointUnreachable`, `SourceEmpty`, `MaxDeletesExceeded`, + `VerificationFailed`, `DeadlineExceeded`, `TransferErrors` or `Unknown`. +- Metrics (gauges derived from status): last success timestamp, last failed, + last transferred bytes, held; labelled by namespace, replication, source and + destination bucket. + +## Migration workflow + +| Step | Action | +|---|---| +| 1 | Recreate buckets and keys on the new cluster declaratively (see next section). | +| 2 | Create a `GarageReplication` (`mode: Copy`, a schedule). | +| 3 | Scheduled runs keep the destination close to the source while apps keep writing to the source. | +| 4 | Freeze writes: scale the writers down, or drop `write` from their GarageKey permissions on the source. | +| 5 | Final sync: set `mode: Mirror` (if source deletions must reach the destination) and `verification: Check`, then annotate `garage.rajsingh.info/trigger-sync`. | +| 6 | Wait until `lastSuccessfulSyncTime` is after the freeze and `Synced=True` with 0 differences. | +| 7 | Point apps at the destination. | +| 8 | `suspend: true`, or delete the `GarageReplication` (data is not touched). | + +Limitations: + +- **Not point-in-time.** A run lists while writes continue; only a write + freeze makes the final sync complete. +- **Deletes in `Copy` mode are never propagated.** Anything written only to + the destination before cutover is deleted by the final `Mirror` run + (subject to `maxDeletes`). +- **No bidirectional or active-active replication.** +- **Metadata fidelity.** Content-Type and user metadata are copied. + `Last-Modified` becomes the destination upload time. Multipart objects get + different ETags. In-progress multipart uploads are not copied. +- **Listing cost.** Each run lists every object on both sides. + +## Configuration and identity: what is *not* replicated + +| Layer | Replicated? | How to carry it over | +|---|---|---| +| Objects and user metadata | **Yes** | this CRD | +| Buckets, aliases, quotas, website, lifecycle | No | `GarageBucket` CRs on the destination cluster | +| Access keys | No | `GarageKey` with `importKey.secretRef` reusing the same key ID and secret | +| Bucket ↔ key permissions | No | `GarageKey.bucketPermissions` / `GarageBucket.keyPermissions` | +| Cluster config, layout, node IDs, admin tokens | No, never | `GarageCluster` spec | + +### Least privilege + +- Source key: `read` on the listed buckets only. +- Destination key: `read` + `write` on the destination buckets (Garage's + `write` includes delete). No `owner`, no bucket creation. + +## Alternatives considered + +- **Garage-level replication (shared RPC, federation).** That is + `remoteClusters` and gives one cluster, not two independent ones. +- **A native Garage feature.** Garage has no bucket replication API, change + feed or notifications. A change feed upstream would make runs incremental; + the engine-neutral API leaves room for it. +- **Fields on `GarageBucket` (`spec.replicateTo`).** Ties replication to + buckets the operator manages; external-to-external replication would be + impossible. + +## Compatibility and data safety + +- Additive: a new CRD and a controller behind a disabled-by-default flag. + While the flag is off, objects are accepted, not reconciled, and admission + warns about it. +- Destructive behaviour needs an explicit `mode: Mirror`, a manual first + trigger, and is capped by `maxDeletes` and the empty-source guard. + +## Test plan + +- **Schema (envtest):** valid objects and defaults; every schema + rule above is rejected by the API server without the webhook + (`internal/controller/garagereplication_schema_test.go`). +- **Webhook:** reference grants, destination exclusivity + (ordering, URL normalization, implicit namespaces), self-loops, cron, Pod + metadata keys, warnings. +- **Controller unit tests** (`garagereplication_controller_test.go`): Job + rendering (no secret value in the Job), credential opt-in, CA from a + ConfigMap, hold release only by `trigger-sync`, first Mirror run held (new + or switched from Copy), destination conflicts through resolved URLs, run + bookkeeping. +- **Contract tests in CI** (`test/contract/replication_contract_test.go`, the + Garage Contract workflow): the Job's container run with Docker in the pinned + image, as UID 65532 on a read-only root filesystem, against two Garage + processes: Copy, Mirror, both delete caps and the empty-source guard with + their exit codes, metadata, verification, failure reasons, SIGTERM. The + workflow sets `REPLICATION_CONTRACT_REQUIRED=true` so the test fails instead + of skipping without a container runtime. +- **Kind e2e** (`Label("replication")`, two GarageClusters): Copy, Mirror with + the first trigger, both guards with hold and release, and the migration + sequence. + +## Review decisions + +From the review of the combined proposal (#580): + +1. Scope narrowed to replication and migration: the `Archive` mode and all + backup wording are removed. +2. Credentials need an opt-in label on the Secret, GarageKey-generated + Secrets included; CA bundles come from ConfigMaps only. +3. A hold is released only by `trigger-sync`, never by a spec edit; the first + Mirror run waits for a trigger. +4. Destination buckets are exclusive across `GarageReplication`s (webhook and + controller). +5. Validation moved into the schema; `buckets[]` and `status.buckets` are + list-maps keyed by `destination`; status fields are typed enums. +6. The engine is tested in CI: the contract job runs the Job script in the + pinned rclone image, and the Kind `replication` e2e runs two + GarageClusters. + +From the review of the API change: + +7. The opt-in label is required on every credentials Secret, GarageKey + Secrets included (a GarageKey sets it through `spec.secretTemplate.labels`). +8. Admission warns that objects are not reconciled while + `--enable-replication` is off. +9. The destination conflict error does not name an object of another + namespace; destination exclusivity is documented as cluster-wide, first + come, first served. + +Non-blocking items taken in the controller change: a size-based delete cap +(`mirror.maxDeleteSize`), classification by rclone exit code first, the first +Mirror run waiting for a trigger (also after a switch from Copy, tracked by +`status.buckets[].lastMode`), typed status enums with list-maps, and comparing +resolved URLs for same-cluster and destination-conflict detection. + +## Deferred + +`--max-age` top-up runs between full listings; per-bucket prefix filters; +server-side checksums once the supported floor is confirmed; bidirectional +sync (not planned); a bucket/key configuration export helper for migrations; +bucket backups with snapshots (separate design). diff --git a/docs/how-to/replication.md b/docs/how-to/replication.md new file mode 100644 index 00000000..d49b251c --- /dev/null +++ b/docs/how-to/replication.md @@ -0,0 +1,406 @@ +# Replicate between clusters + +`GarageReplication` copies objects between two **independent** Garage clusters +through their S3 APIs. Use it to: + +- **Replicate**: keep a warm copy of production buckets on another cluster. +- **Migrate**: move buckets to a new cluster with a short write freeze. + +It is **not a backup**. There are no point-in-time snapshots and no restore, +and in `Mirror` mode an accidental deletion or overwrite on the source reaches +the destination on the next run. Bucket backups need a snapshot tool. + +It is also different from Garage's own replication (`replication.factor`), +which spreads data across the nodes of one cluster, and from +[federation](federation.md), which joins several Kubernetes clusters into one +Garage cluster. Two clusters linked by `GarageReplication` share nothing: no +RPC secret, no layout, no keys. + +## How it works + +Each run starts one Kubernetes Job per bucket, in the `GarageReplication`'s +namespace. The Job runs [rclone](https://rclone.org) (image +`docker.io/rclone/rclone`, pinned by digest) to compare the source and +destination buckets and copy what differs. The operator: + +- schedules runs (cron), never runs two at once, and runs at most + `transfer.maxConcurrentBuckets` buckets in parallel; +- passes credentials to rclone only through `secretKeyRef` environment + variables. They never appear in the Job's arguments, in status or in the + operator's logs, and the operator never reads their values; +- reads each Job's result from its termination message and records rclone's + counters per bucket in `status.buckets`; +- deletes old Jobs (`successfulJobsHistoryLimit: 1`, `failedJobsHistoryLimit: 3`). + +The Jobs need network access to **both** S3 endpoints. Nothing else talks to +Garage: the operator does not use either cluster's Admin API for replication. +The Pods run as a non-root user with a read-only root filesystem, no +capabilities and no service account token, so they pass the `restricted` Pod +Security Standard. + +## Enable the controller + +Replication is off by default. Enable it in the chart: + +```yaml +replication: + enabled: true + # image: registry.example.com/mirror/rclone:1.75.2 # optional mirror +``` + +This adds `--enable-replication` to the operator and grants it +`create/delete/get/list/watch` on `batch/jobs` and write access to +`GarageReplication` status. Without it, `GarageReplication` objects are +validated but not reconciled, and `kubectl apply` prints a warning saying so. + +## Credentials (least privilege) + +Each side reads an S3 access key from a Secret **in the replication's +namespace**. The Job sends that key to the endpoint named in the spec, so the +Secret must **opt in** with the label +`garage.rajsingh.info/replication-credentials: "true"`. Without the label the +replication reports `Ready=False` with reason `CredentialsNotAllowed` and runs +nothing. Only someone who can write the Secret can add the label, so being +allowed to create a `GarageReplication` does not let anyone copy another +application's buckets with its key. + +The default data keys, `access-key-id` and `secret-access-key`, are the ones a +`GarageKey` writes. Set the label through the key's Secret template: + +```yaml +# Read-only key on the source cluster +apiVersion: garage.rajsingh.info/v1beta1 +kind: GarageKey +metadata: + name: replication-reader + namespace: replication +spec: + clusterRef: + name: garage + namespace: storage # needs a GarageReferenceGrant for kind GarageKey + secretTemplate: + labels: + garage.rajsingh.info/replication-credentials: "true" + bucketPermissions: + - globalAlias: medias + read: true +``` + +| Side | Permissions | +|---|---| +| Source | `read` on every source bucket | +| Destination | `read` + `write` on every destination bucket (Garage's `write` includes deletes, which `Mirror` needs) | + +No `owner` permission and no bucket creation are needed. Replication never +creates buckets: a missing bucket fails with `BucketNotFound`. Create +destination buckets beforehand, for example with `GarageBucket`. + +For a destination outside the operator's reach, create and label the Secret +yourself: + +```bash +kubectl -n replication create secret generic dr-replication-writer \ + --from-literal=access-key-id=GK... --from-literal=secret-access-key=... +kubectl -n replication label secret dr-replication-writer \ + garage.rajsingh.info/replication-credentials=true +``` + +A private CA for an `endpoint` comes from a **ConfigMap** (`caBundleRef`, +key `ca.crt` by default). A CA certificate is public; a Secret reference would +reopen the path the opt-in closes. + +## Examples + +### Replicate to another site + +```yaml +apiVersion: garage.rajsingh.info/v1beta1 +kind: GarageReplication +metadata: + name: production-to-dr + namespace: replication +spec: + source: + clusterRef: {name: garage, namespace: storage} # needs a GarageReferenceGrant + credentialsSecretRef: {name: replication-reader} + destination: + endpoint: + url: https://s3.dr.example.com + region: garage + caBundleRef: {name: dr-ca} # only for a private CA + credentialsSecretRef: {name: dr-replication-writer} + buckets: + - source: medias + destination: medias + - source: pg-dumps + destination: pg-dumps-dr + schedule: "*/15 * * * *" +``` + +`buckets` is keyed by `destination`: a destination bucket appears once, and the +same source may feed several destinations. + +A cross-namespace `clusterRef` needs a grant in the cluster's namespace: + +```yaml +apiVersion: garage.rajsingh.info/v1beta1 +kind: GarageReferenceGrant +metadata: + name: allow-replication + namespace: storage +spec: + from: + - kind: GarageReplication + namespace: replication + to: + - kind: GarageCluster + name: garage +``` + +### One-shot copy + +Without `schedule`, a `Copy` replication runs once when it is created, then +only when triggered: + +```bash +kubectl -n replication annotate garagereplication production-to-dr \ + garage.rajsingh.info/trigger-sync="$(date +%s)" --overwrite +``` + +## Deletion semantics + +Nothing destructive is enabled by default. + +| `mode` | Source object deleted | Source object changed | Object only on the destination | +|---|---|---|---| +| `Copy` (default) | kept on the destination | **overwritten** on the destination | kept | +| `Mirror` | deleted on the destination | overwritten | **deleted** | + +`Copy` does not make the destination immutable: changed objects are +overwritten. + +Safety guards for `Mirror`: + +- **First run on a trigger.** A bucket that has not run in `Mirror` mode yet + (a new replication, or one switched from `Copy`) is held with + `hold: AwaitingTrigger` and `Synced=False/AwaitingTrigger`. Check both sides, + then set the `trigger-sync` annotation to start it. Later scheduled runs + need no trigger. +- **Delete caps.** `mirror.maxDeletes` is required; `mirror.maxDeleteSize` + (for example `10G`) optionally caps the deleted bytes too. A run that + reaches either cap stops with `MaxDeletesExceeded`. Deletions made up to + that point are kept. +- **Empty source.** If the source bucket is empty while the destination is + not, the run refuses to start (`SourceEmpty`). This protects against a wrong + bucket name, a key that sees nothing, or a wiped source. +- **Hold.** After a cap or the empty-source guard trips, the bucket is + **held**: `status.buckets[].hold` is set, `Synced=False/SafetyHold`, scheduled + runs skip it, and the Job is not retried, so the next run does not delete + another batch. **Only the `trigger-sync` annotation releases a hold.** + Editing the spec (for example raising `maxDeletes`) does not: investigate, + adjust the spec if needed, then trigger. + +Changing `mode` from `Mirror` to `Copy` cancels running `Mirror` Jobs and drops +`AwaitingTrigger` holds (`Copy` never deletes). Removing a bucket from +`spec.buckets`, or giving its destination another source, cancels its running +Job. Other changes apply from the next run; the running Job finishes with the +configuration it started with. + +A destination bucket can be written by only one `GarageReplication`, across +all namespaces: two `Mirror`s on one bucket would delete each other's objects. +The first one keeps it; admission rejects the second, and if one slips through +(created while the webhook was down, or naming the same cluster through a +`clusterRef` and a URL) it reports `Ready=False/DestinationConflict` and runs +nothing. A conflicting object in another namespace is not named. + +Deleting a `GarageReplication` deletes its Jobs (owner references) and never +touches objects on either side. + +## Operate + +| Action | How | +|---|---| +| Pause | `spec.suspend: true`. No scheduled run starts; a running run finishes. | +| Stop now | `kubectl delete job -l garage.rajsingh.info/replication-uid=`. The bucket is recorded as `Cancelled`. | +| Resume | `spec.suspend: false`. | +| Run now, release holds | `garage.rajsingh.info/trigger-sync=`. Works while suspended. If a run is in progress, the triggered run starts right after it. The annotation is removed once the run starts, and `status.lastOperation` records it. | + +```console +$ kubectl get garagereplications -n replication +NAME MODE SCHEDULE PHASE LAST SUCCESS AGE +production-to-dr Copy */15 * * * * Syncing 14m 3d +``` + +`status` reports: + +- `phase`: `Pending` (configuration not usable, see `Ready`), `Idle`, + `Syncing`, `Suspended` or `Failed` (the last run had a failed bucket); +- `currentRun` / `lastRun`: ID, reason (`Initial`, `Scheduled`, + `Triggered`), start and completion time, result; +- `lastSyncTime`, `lastSuccessfulSyncTime`, `nextSyncTime`; +- `buckets[]`, keyed by `destination`: `activeJob`, `lastResult`, `lastMode`, + `lastSuccessfulTime`, `hold`, `lastError`, `lastVerification`, and + `lastStatistics` (`transfers`, `bytes`, `checks`, `deletes`, `errors`, + `elapsedSeconds`, `differences`). The statistics are rclone's own counters + for the last Job, and they are absent when the Job reported none (for + example a Pod that was killed). + +Conditions: `Ready` (endpoints, Secrets, opt-in, grants and destination +exclusivity resolve; reasons include `CredentialsNotAllowed`, +`DestinationConflict`, `SecretNotFound`, `ReferenceNotGranted`), `Synced` +(every bucket succeeded in the last run on the current configuration; reasons +`NeverSynced`, `SyncFailed`, `SafetyHold`, `AwaitingTrigger`, `ConfigChanged`), +`Suspended`. Events: `SyncStarted`, `SyncSucceeded`, `SyncFailed`, +`SyncCancelled`, `SyncSkipped`. + +`lastError` is one of these fixed reasons, classified from rclone's exit code +first and from its log only where the exit code is ambiguous. rclone's own +output, which can name objects, stays in the Job's Pod logs. + +| Reason | Meaning | +|---|---| +| `AuthFailed` | the S3 key was rejected (403) | +| `BucketNotFound` | a source or destination bucket does not exist | +| `EndpointUnreachable` | connection, timeout, server or TLS errors left after rclone's retries | +| `SourceEmpty` | empty-source guard (see above) | +| `MaxDeletesExceeded` | `maxDeletes` or `maxDeleteSize` reached (see above) | +| `VerificationFailed` | `verification: Check` found differences | +| `DeadlineExceeded` | the Job ran longer than `retry.activeDeadlineSeconds` | +| `TransferErrors` | other rclone errors; see the Pod log | +| `Unknown` | the Pod ended without a result (for example it was OOM-killed) | + +Prometheus metrics (operator registry), with labels `namespace`, +`replication`, `source_bucket` and `destination_bucket`: + +- `garage_operator_replication_last_success_timestamp_seconds` +- `garage_operator_replication_last_failed` +- `garage_operator_replication_last_transferred_bytes` +- `garage_operator_replication_held` + +Example alert for a 15-minute schedule: + +```yaml +- alert: GarageReplicationStale + expr: time() - garage_operator_replication_last_success_timestamp_seconds > 3600 + for: 10m +``` + +### Tuning + +| Field | Default | Effect | +|---|---|---| +| `transfer.concurrency` | 4 | objects copied in parallel per bucket (`rclone --transfers`) | +| `transfer.checkers` | 8 | objects compared in parallel per bucket (`--checkers`) | +| `transfer.bandwidthLimit` | unlimited | bytes/s per Job; `K`, `M`, `G` are KiB, MiB, GiB, e.g. `50M` (`--bwlimit`) | +| `transfer.maxConcurrentBuckets` | 1 | buckets replicated at the same time | +| `retry.backoffLimit` | 2 | Pod retries per bucket and run; already-copied objects are skipped | +| `retry.activeDeadlineSeconds` | 21600 | maximum duration of a bucket's Job | +| `verification` | `None` | `Check` re-compares source and destination after the copy | +| `resources`, `podTemplate` | | resources, node selector, tolerations, affinity, priority class, Pod labels/annotations | + +Inside a run, rclone retries failed operations itself (3 attempts of the +whole bucket, 10 low-level retries). Kubernetes then retries the Pod +`backoffLimit` times, and the next scheduled run tries again. + +Objects are compared by size and MD5. For objects uploaded in one part, +Garage's ETag is the MD5 of the content, so the comparison is exact. For +multipart objects that rclone did not upload itself there is no MD5, so the +comparison falls back to size only. A changed object of the same size, uploaded +in several parts by another client, is then not detected. + +## Migrate to a new cluster + +1. Recreate the configuration on the new cluster declaratively (see + [What is not replicated](#what-is-not-replicated)): `GarageBucket`s, and + `GarageKey`s with `importKey` reusing the **same** access key IDs and secrets, + so applications only need a new endpoint. +2. Create a `GarageReplication` from the old cluster to the new one with + `mode: Copy` and a schedule. The first run copies everything. +3. Let scheduled runs keep the new cluster close while applications write to + the old one. Watch `lastStatistics.transfers` drop to the write rate. +4. Freeze writes: scale the writers down, or remove `write` from their keys on + the old cluster. +5. Final sync: remove the schedule, set `mode: Mirror` (with + `mirror.maxDeletes`) if deletions made since step 2 must reach the new + cluster, and `verification: Check`. Then trigger a run; the trigger is also + what lets the first `Mirror` run start: + + ```bash + kubectl annotate garagereplication migrate garage.rajsingh.info/trigger-sync="final-$(date +%s)" --overwrite + ``` + +6. Wait for `lastRun.reason: Triggered`, `lastRun.result: Succeeded`, + `Synced=True`, and `lastVerification: Passed` with `differences: 0` on + every bucket. +7. Point applications at the new cluster. +8. Set `suspend: true` or delete the `GarageReplication` (no data is deleted). + +The Kind end-to-end test (`Label("replication")`) runs this sequence. + +Limitations of this procedure: + +- **A run is not a point-in-time snapshot.** It lists while writes continue; + objects written after the listing are picked up by the next run. Only the + write freeze in step 4 makes the final sync complete. +- **`Copy` never propagates deletions.** That is why step 5 switches to + `Mirror`. Anything written directly to the new cluster before cutover is + deleted by that run, up to `maxDeletes`. +- **No two-way replication.** Two `Mirror` replications in opposite directions + delete each other's writes. +- **Object metadata.** `Content-Type`, `Cache-Control` and user metadata are + copied. `Last-Modified` becomes the time of the copy (rclone keeps the + original in `x-amz-meta-mtime`). Multipart objects get new ETags. + In-progress multipart uploads are not copied. + +## Switch to the replica after losing a site + +1. Set `suspend: true` on the replication. If the source is unreachable, runs + fail with `EndpointUnreachable` and change nothing on the destination. +2. Check `status.buckets[].lastSuccessfulTime`: objects written to the source + after that time are not on the replica. +3. Recreate keys and permissions on the replica (see below) and point + applications at it. +4. Once the original site is back, replicate in the other direction with + `mode: Copy` first (never `Mirror` until you have checked which side holds + which data), then cut back with the migration procedure. + +## What is not replicated + +Replication copies **objects and their metadata only**. Garage's internal +metadata is cluster-specific and must never be copied between clusters. + +| Layer | Replicated? | How to carry it over | +|---|---|---| +| Objects and user metadata | yes | `GarageReplication` | +| Buckets, aliases, quotas, website, lifecycle rules | no | `GarageBucket` on the destination cluster (copy the manifests if the source is declarative) | +| Access keys | no | `GarageKey` with `importKey` (same ID and secret; IDs must be `GK` + 24 hex characters before Garage v2.3) | +| Bucket ↔ key permissions | no | `GarageKey.spec.bucketPermissions` / `GarageBucket.spec.keyPermissions` | +| CORS rules, website redirects and routing rules | no | S3 API | +| Cluster configuration, layout, node IDs, admin tokens | no, never | the destination's own `GarageCluster` | + +## Known limitations + +- Each run lists every object on both sides. Transfers are incremental, but + the listing is not, because Garage has no change feed. For buckets with tens + of millions of objects, use a schedule interval longer than one full listing. +- The copy runs in the operator's Kubernetes cluster. It streams through the + Job Pod (no server-side copy between clusters), so plan network bandwidth + and use `bandwidthLimit`. +- A `SIGTERM` (Job deleted, deadline, node drain) stops rclone cleanly. Objects + are uploaded atomically, but an interrupted multipart upload leaves an + incomplete upload on the destination. Add an + `abortIncompleteMultipartUploadDays` lifecycle rule to destination buckets: + + ```yaml + apiVersion: garage.rajsingh.info/v1beta1 + kind: GarageBucket + metadata: + name: medias + spec: + clusterRef: {name: garage-dr} + lifecycle: + rules: + - id: abort-stale-uploads + status: Enabled + abortIncompleteMultipartUploadDays: 2 + ``` diff --git a/docs/reference/api/garagereferencegrant-v1beta1.md b/docs/reference/api/garagereferencegrant-v1beta1.md index e9277117..cc44670a 100644 --- a/docs/reference/api/garagereferencegrant-v1beta1.md +++ b/docs/reference/api/garagereferencegrant-v1beta1.md @@ -13,7 +13,7 @@ GarageReferenceGrantSpec defines which namespaces and resource kinds are permitt | Field path | Type | Constraints | Description | | --- | --- | --- | --- | | `from` | array of object | required; min items: 1 | From lists the permitted sources of cross-namespace references. | -| `from[].kind` | string | required; enum: \["GarageKey", "GarageBucket", "GarageAdminToken"\] | Kind is the resource kind allowed to make cross-namespace references. GarageAdminToken remains in the schema for compatibility, but the static credential path is namespace-local and does not accept cross-namespace grants. | +| `from[].kind` | string | required; enum: \["GarageKey", "GarageBucket", "GarageAdminToken", "GarageReplication"\] | Kind is the resource kind allowed to make cross-namespace references. GarageAdminToken remains in the schema for compatibility, but the static credential path is namespace-local and does not accept cross-namespace grants. GarageReplication may reference a GarageCluster in another namespace. | | `from[].namespace` | string | min length: 1 | Namespace is the exact namespace from which cross-namespace references are allowed. Exactly one of Namespace or NamespaceSelector must be set. | | `from[].namespaceSelector` | object | | NamespaceSelector selects source namespaces by their Kubernetes labels. Exactly one of Namespace or NamespaceSelector must be set. An empty selector matches every namespace. | | `from[].namespaceSelector.matchExpressions` | array of object | | matchExpressions is a list of label selector requirements. The requirements are ANDed. | diff --git a/docs/reference/api/garagereplication-v1beta1.md b/docs/reference/api/garagereplication-v1beta1.md new file mode 100644 index 00000000..e40ef960 --- /dev/null +++ b/docs/reference/api/garagereplication-v1beta1.md @@ -0,0 +1,254 @@ +# GarageReplication v1beta1 + + + +API version: `garage.rajsingh.info/v1beta1`. Scope: Namespaced. + +The [custom-resource guide](../custom-resources.md) explains operational behavior. The [JSON schemas](../samples.md) are the validation source of truth. + +## Spec + +GarageReplicationSpec defines the desired state of GarageReplication. GarageReplication copies objects between two independent Garage clusters through their S3 APIs, for replication and migration. It is not a backup: there are no point-in-time snapshots and no restore, and in mode Mirror a deletion or an overwrite on the source reaches the destination on the next run. Each run executes one Kubernetes Job per bucket in the GarageReplication's namespace, so the Jobs need network access to both S3 endpoints. Only objects (and their user metadata) are copied: buckets, keys, permissions, quotas, website and lifecycle configuration are not. Deleting a GarageReplication never deletes object data on either side. + +| Field path | Type | Constraints | Description | +| --- | --- | --- | --- | +| `buckets` | array of object | required; min items: 1; max items: 50 | Buckets lists the buckets to replicate, keyed by destination bucket. | +| `buckets[].destination` | string | required; min length: 3; max length: 63; pattern: "^\[a-z0-9\]\[a-z0-9.-\]{1,61}\[a-z0-9\]$" | Destination is the bucket name (global alias) on the destination cluster. It is the list key, so destinations are unique. | +| `buckets[].source` | string | required; min length: 3; max length: 63; pattern: "^\[a-z0-9\]\[a-z0-9.-\]{1,61}\[a-z0-9\]$" | Source is the bucket name (global alias) on the source cluster. | +| `destination` | object | required; validation: exactly one of clusterRef or endpoint must be set; validation: clusterRef.kubeConfigSecretRef is not supported | Destination is the cluster objects are written to. Its credentials need read and write permission on the destination buckets. Destination buckets must already exist: replication never creates buckets. A destination bucket can be written by only one GarageReplication. | +| `destination.clusterRef` | object | | ClusterRef references a GarageCluster (operator-managed or a management handle). The S3 endpoint and region are resolved from it. A cross-namespace reference requires a GarageReferenceGrant in the cluster's namespace that allows kind GarageReplication. | +| `destination.clusterRef.kubeConfigSecretRef` | object | | KubeConfigSecretRef is reserved for a future remote Kubernetes client integration. It is currently rejected by admission because the operator does not use it. | +| `destination.clusterRef.kubeConfigSecretRef.key` | string | required | The key of the secret to select from. Must be a valid secret key. | +| `destination.clusterRef.kubeConfigSecretRef.name` | string | default: "" | Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names | +| `destination.clusterRef.kubeConfigSecretRef.optional` | boolean | | Specify whether the Secret or its key must be defined | +| `destination.clusterRef.name` | string | required | Name of the GarageCluster resource. | +| `destination.clusterRef.namespace` | string | | Namespace of the GarageCluster. Defaults to the referencing resource's namespace. Cross-namespace references require a GarageReferenceGrant where supported by the owning resource. GarageNode and GarageAdminToken reject them. | +| `destination.credentialsSecretRef` | object | required; validation: accessKeyIdKey and secretAccessKeyKey must be different | CredentialsSecretRef references a Secret in the GarageReplication's namespace holding an S3 access key. The Secret must carry the label garage.rajsingh.info/replication-credentials: "true" (for a GarageKey, set it in spec.secretTemplate.labels). The access key ID is sent to the endpoint, so without this opt-in the right to create a GarageReplication would amount to reading any Secret, or using any other application's key. The default data keys match the Secret generated for a GarageKey. | +| `destination.credentialsSecretRef.accessKeyIdKey` | string | default: "access-key-id"; min length: 1; max length: 253; pattern: "^\[-._a-zA-Z0-9\]+$" | AccessKeyIDKey is the data key holding the access key ID. | +| `destination.credentialsSecretRef.name` | string | required; min length: 1; max length: 253; pattern: "^\[a-z0-9\](\[-a-z0-9\]*\[a-z0-9\])?(\\.\[a-z0-9\](\[-a-z0-9\]*\[a-z0-9\])?)*$" | Name of the Secret in the GarageReplication's namespace. | +| `destination.credentialsSecretRef.secretAccessKeyKey` | string | default: "secret-access-key"; min length: 1; max length: 253; pattern: "^\[-._a-zA-Z0-9\]+$" | SecretAccessKeyKey is the data key holding the secret access key. | +| `destination.endpoint` | object | | Endpoint is an S3 endpoint of any Garage cluster, inside or outside Kubernetes. | +| `destination.endpoint.caBundleRef` | object | | CABundleRef references a ConfigMap holding a PEM CA bundle used, in addition to the system trust store, to verify the endpoint's TLS certificate. | +| `destination.endpoint.caBundleRef.key` | string | default: "ca.crt"; min length: 1; max length: 253; pattern: "^\[-._a-zA-Z0-9\]+$" | Key of the ConfigMap holding the PEM bundle. | +| `destination.endpoint.caBundleRef.name` | string | required; min length: 1; max length: 253; pattern: "^\[a-z0-9\](\[-a-z0-9\]*\[a-z0-9\])?(\\.\[a-z0-9\](\[-a-z0-9\]*\[a-z0-9\])?)*$" | Name of the ConfigMap. | +| `destination.endpoint.region` | string | default: "garage"; min length: 1; max length: 128; pattern: "^\[A-Za-z0-9_.-\]+$" | Region is the Garage s3_region used for request signing. | +| `destination.endpoint.url` | string | required; min length: 1; max length: 2048; pattern: "^https?://\[^\\s/?#@\]+/?$" | URL of the S3 API: scheme and host (with an optional port) only, e.g. https://s3.example.com or http://garage.ns.svc:3900. Buckets are addressed path-style below it, so it has no path, query, fragment or credentials. | +| `failedJobsHistoryLimit` | integer | default: 3; format: "int32"; min: 0; max: 50 | FailedJobsHistoryLimit is the number of finished failed Jobs to keep. | +| `mirror` | object | | Mirror configures mode Mirror. Required when mode is Mirror and rejected otherwise. | +| `mirror.maxDeleteSize` | string | max length: 32; pattern: "^\[0-9\]+(\\.\[0-9\]+)?\[KMGTkmgt\]?$" | MaxDeleteSize caps the total size of the destination objects one run may delete per bucket, in bytes with an optional K, M, G or T suffix (binary multiples: KiB, MiB, GiB, TiB), e.g. "10G". Once reached the run stops and fails with reason MaxDeletesExceeded, like maxDeletes. Empty means no size cap. | +| `mirror.maxDeletes` | integer | required; format: "int64"; min: 1 | MaxDeletes caps the number of destination objects one run may delete per bucket. Once reached the run stops and fails with reason MaxDeletesExceeded; deletions done before that point are kept. | +| `mode` | string | default: "Copy"; enum: \["Copy", "Mirror"\] | Mode controls whether changes on the source can remove data on the destination. - Copy: copy new and changed objects. Source deletions are not propagated. Changed objects are overwritten on the destination. - Mirror: the destination reflects the source, including deletions, capped by mirror.maxDeletes per bucket and run. The first run waits for the trigger-sync annotation. | +| `podTemplate` | object | | PodTemplate customizes the replication Pods. | +| `podTemplate.affinity` | object | | Affinity for the Pods. | +| `podTemplate.affinity.nodeAffinity` | object | | Describes node affinity scheduling rules for the pod. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution` | array of object | | The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[].preference` | object | required | A node selector term, associated with the corresponding weight. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[].preference.matchExpressions` | array of object | | A list of node selector requirements by node's labels. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[].preference.matchExpressions[].key` | string | required | The label key that the selector applies to. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[].preference.matchExpressions[].operator` | string | required | Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[].preference.matchExpressions[].values` | array of string | | An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[].preference.matchFields` | array of object | | A list of node selector requirements by node's fields. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[].preference.matchFields[].key` | string | required | The label key that the selector applies to. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[].preference.matchFields[].operator` | string | required | Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[].preference.matchFields[].values` | array of string | | An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[].weight` | integer | required; format: "int32" | Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. | +| `podTemplate.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution` | object | | If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. | +| `podTemplate.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms` | array of object | required | Required. A list of node selector terms. The terms are ORed. | +| `podTemplate.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions` | array of object | | A list of node selector requirements by node's labels. | +| `podTemplate.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].key` | string | required | The label key that the selector applies to. | +| `podTemplate.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].operator` | string | required | Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. | +| `podTemplate.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].values` | array of string | | An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchFields` | array of object | | A list of node selector requirements by node's fields. | +| `podTemplate.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchFields[].key` | string | required | The label key that the selector applies to. | +| `podTemplate.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchFields[].operator` | string | required | Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. | +| `podTemplate.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchFields[].values` | array of string | | An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.podAffinity` | object | | Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution` | array of object | | The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm` | object | required | Required. A pod affinity term, associated with the corresponding weight. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector` | object | | A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector.matchExpressions` | array of object | | matchExpressions is a list of label selector requirements. The requirements are ANDed. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector.matchExpressions[].key` | string | required | key is the label key that the selector applies to. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector.matchExpressions[].operator` | string | required | operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector.matchExpressions[].values` | array of string | | values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector.matchLabels` | map of string | | matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.matchLabelKeys` | array of string | | MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.mismatchLabelKeys` | array of string | | MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector` | object | | A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector.matchExpressions` | array of object | | matchExpressions is a list of label selector requirements. The requirements are ANDed. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector.matchExpressions[].key` | string | required | key is the label key that the selector applies to. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector.matchExpressions[].operator` | string | required | operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector.matchExpressions[].values` | array of string | | values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector.matchLabels` | map of string | | matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaces` | array of string | | namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.topologyKey` | string | required | This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. | +| `podTemplate.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[].weight` | integer | required; format: "int32" | weight associated with matching the corresponding podAffinityTerm, in the range 1-100. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution` | array of object | | If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector` | object | | A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector.matchExpressions` | array of object | | matchExpressions is a list of label selector requirements. The requirements are ANDed. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector.matchExpressions[].key` | string | required | key is the label key that the selector applies to. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector.matchExpressions[].operator` | string | required | operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector.matchExpressions[].values` | array of string | | values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector.matchLabels` | map of string | | matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].matchLabelKeys` | array of string | | MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].mismatchLabelKeys` | array of string | | MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector` | object | | A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector.matchExpressions` | array of object | | matchExpressions is a list of label selector requirements. The requirements are ANDed. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector.matchExpressions[].key` | string | required | key is the label key that the selector applies to. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector.matchExpressions[].operator` | string | required | operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector.matchExpressions[].values` | array of string | | values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector.matchLabels` | map of string | | matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaces` | array of string | | namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". | +| `podTemplate.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[].topologyKey` | string | required | This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. | +| `podTemplate.affinity.podAntiAffinity` | object | | Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution` | array of object | | The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and subtracting "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm` | object | required | Required. A pod affinity term, associated with the corresponding weight. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector` | object | | A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector.matchExpressions` | array of object | | matchExpressions is a list of label selector requirements. The requirements are ANDed. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector.matchExpressions[].key` | string | required | key is the label key that the selector applies to. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector.matchExpressions[].operator` | string | required | operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector.matchExpressions[].values` | array of string | | values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.labelSelector.matchLabels` | map of string | | matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.matchLabelKeys` | array of string | | MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.mismatchLabelKeys` | array of string | | MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector` | object | | A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector.matchExpressions` | array of object | | matchExpressions is a list of label selector requirements. The requirements are ANDed. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector.matchExpressions[].key` | string | required | key is the label key that the selector applies to. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector.matchExpressions[].operator` | string | required | operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector.matchExpressions[].values` | array of string | | values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaceSelector.matchLabels` | map of string | | matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.namespaces` | array of string | | namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].podAffinityTerm.topologyKey` | string | required | This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. | +| `podTemplate.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[].weight` | integer | required; format: "int32" | weight associated with matching the corresponding podAffinityTerm, in the range 1-100. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution` | array of object | | If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector` | object | | A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector.matchExpressions` | array of object | | matchExpressions is a list of label selector requirements. The requirements are ANDed. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector.matchExpressions[].key` | string | required | key is the label key that the selector applies to. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector.matchExpressions[].operator` | string | required | operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector.matchExpressions[].values` | array of string | | values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].labelSelector.matchLabels` | map of string | | matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].matchLabelKeys` | array of string | | MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].mismatchLabelKeys` | array of string | | MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector` | object | | A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector.matchExpressions` | array of object | | matchExpressions is a list of label selector requirements. The requirements are ANDed. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector.matchExpressions[].key` | string | required | key is the label key that the selector applies to. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector.matchExpressions[].operator` | string | required | operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector.matchExpressions[].values` | array of string | | values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaceSelector.matchLabels` | map of string | | matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].namespaces` | array of string | | namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". | +| `podTemplate.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[].topologyKey` | string | required | This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. | +| `podTemplate.annotations` | map of string | | Annotations added to the Pods. | +| `podTemplate.labels` | map of string | | Labels added to the Pods. | +| `podTemplate.nodeSelector` | map of string | | NodeSelector for the Pods. | +| `podTemplate.priorityClassName` | string | max length: 253 | PriorityClassName for the Pods. | +| `podTemplate.tolerations` | array of object | | Tolerations for the Pods. | +| `podTemplate.tolerations[].effect` | string | | Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. | +| `podTemplate.tolerations[].key` | string | | Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. | +| `podTemplate.tolerations[].operator` | string | | Operator represents a key's relationship to the value. Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). | +| `podTemplate.tolerations[].tolerationSeconds` | integer | format: "int64" | TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. | +| `podTemplate.tolerations[].value` | string | | Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. | +| `resources` | object | | Resources for the replication container. | +| `resources.claims` | array of object | | Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. | +| `resources.claims[].name` | string | required | Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. | +| `resources.claims[].request` | string | | Request is the name chosen for a request in the referenced claim. If empty, everything from the claim is made available, otherwise only the result of this request. | +| `resources.limits` | map of integer or string | | Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| `resources.requests` | map of integer or string | | Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| `retry` | object | | Retry bounds retries and duration of each per-bucket Job. | +| `retry.activeDeadlineSeconds` | integer | default: 21600; format: "int64"; min: 60 | ActiveDeadlineSeconds bounds the duration of a per-bucket Job. | +| `retry.backoffLimit` | integer | default: 2; format: "int32"; min: 0; max: 10 | BackoffLimit is the number of Pod retries of a per-bucket Job. Each retry restarts the bucket's comparison; already copied objects are skipped. | +| `schedule` | string | max length: 128 | Schedule is a cron expression (standard 5-field syntax, descriptors such as @hourly, optional CRON_TZ= prefix; UTC otherwise). Without a schedule, runs happen only through the garage.rajsingh.info/trigger-sync annotation (one-shot mode, e.g. for migrations). | +| `source` | object | required; validation: exactly one of clusterRef or endpoint must be set; validation: clusterRef.kubeConfigSecretRef is not supported | Source is the cluster objects are read from. Its credentials only need read permission on the source buckets. | +| `source.clusterRef` | object | | ClusterRef references a GarageCluster (operator-managed or a management handle). The S3 endpoint and region are resolved from it. A cross-namespace reference requires a GarageReferenceGrant in the cluster's namespace that allows kind GarageReplication. | +| `source.clusterRef.kubeConfigSecretRef` | object | | KubeConfigSecretRef is reserved for a future remote Kubernetes client integration. It is currently rejected by admission because the operator does not use it. | +| `source.clusterRef.kubeConfigSecretRef.key` | string | required | The key of the secret to select from. Must be a valid secret key. | +| `source.clusterRef.kubeConfigSecretRef.name` | string | default: "" | Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names | +| `source.clusterRef.kubeConfigSecretRef.optional` | boolean | | Specify whether the Secret or its key must be defined | +| `source.clusterRef.name` | string | required | Name of the GarageCluster resource. | +| `source.clusterRef.namespace` | string | | Namespace of the GarageCluster. Defaults to the referencing resource's namespace. Cross-namespace references require a GarageReferenceGrant where supported by the owning resource. GarageNode and GarageAdminToken reject them. | +| `source.credentialsSecretRef` | object | required; validation: accessKeyIdKey and secretAccessKeyKey must be different | CredentialsSecretRef references a Secret in the GarageReplication's namespace holding an S3 access key. The Secret must carry the label garage.rajsingh.info/replication-credentials: "true" (for a GarageKey, set it in spec.secretTemplate.labels). The access key ID is sent to the endpoint, so without this opt-in the right to create a GarageReplication would amount to reading any Secret, or using any other application's key. The default data keys match the Secret generated for a GarageKey. | +| `source.credentialsSecretRef.accessKeyIdKey` | string | default: "access-key-id"; min length: 1; max length: 253; pattern: "^\[-._a-zA-Z0-9\]+$" | AccessKeyIDKey is the data key holding the access key ID. | +| `source.credentialsSecretRef.name` | string | required; min length: 1; max length: 253; pattern: "^\[a-z0-9\](\[-a-z0-9\]*\[a-z0-9\])?(\\.\[a-z0-9\](\[-a-z0-9\]*\[a-z0-9\])?)*$" | Name of the Secret in the GarageReplication's namespace. | +| `source.credentialsSecretRef.secretAccessKeyKey` | string | default: "secret-access-key"; min length: 1; max length: 253; pattern: "^\[-._a-zA-Z0-9\]+$" | SecretAccessKeyKey is the data key holding the secret access key. | +| `source.endpoint` | object | | Endpoint is an S3 endpoint of any Garage cluster, inside or outside Kubernetes. | +| `source.endpoint.caBundleRef` | object | | CABundleRef references a ConfigMap holding a PEM CA bundle used, in addition to the system trust store, to verify the endpoint's TLS certificate. | +| `source.endpoint.caBundleRef.key` | string | default: "ca.crt"; min length: 1; max length: 253; pattern: "^\[-._a-zA-Z0-9\]+$" | Key of the ConfigMap holding the PEM bundle. | +| `source.endpoint.caBundleRef.name` | string | required; min length: 1; max length: 253; pattern: "^\[a-z0-9\](\[-a-z0-9\]*\[a-z0-9\])?(\\.\[a-z0-9\](\[-a-z0-9\]*\[a-z0-9\])?)*$" | Name of the ConfigMap. | +| `source.endpoint.region` | string | default: "garage"; min length: 1; max length: 128; pattern: "^\[A-Za-z0-9_.-\]+$" | Region is the Garage s3_region used for request signing. | +| `source.endpoint.url` | string | required; min length: 1; max length: 2048; pattern: "^https?://\[^\\s/?#@\]+/?$" | URL of the S3 API: scheme and host (with an optional port) only, e.g. https://s3.example.com or http://garage.ns.svc:3900. Buckets are addressed path-style below it, so it has no path, query, fragment or credentials. | +| `successfulJobsHistoryLimit` | integer | default: 1; format: "int32"; min: 0; max: 50 | SuccessfulJobsHistoryLimit is the number of finished successful Jobs to keep. | +| `suspend` | boolean | | Suspend stops starting new runs. A run already in progress finishes; delete its Jobs to stop it immediately. | +| `transfer` | object | | Transfer tunes the copy. | +| `transfer.bandwidthLimit` | string | max length: 32; pattern: "^\[0-9\]+(\\.\[0-9\]+)?\[KMGkmg\]?$" | BandwidthLimit caps the transfer rate of each per-bucket Job in bytes per second, with an optional K, M, G suffix (binary multiples: KiB/s, MiB/s, GiB/s), e.g. "50M". Empty means unlimited. | +| `transfer.checkers` | integer | default: 8; format: "int32"; min: 1; max: 128 | Checkers is the number of objects compared in parallel per bucket. | +| `transfer.concurrency` | integer | default: 4; format: "int32"; min: 1; max: 64 | Concurrency is the number of objects transferred in parallel per bucket. | +| `transfer.maxConcurrentBuckets` | integer | default: 1; format: "int32"; min: 1; max: 50 | MaxConcurrentBuckets is the number of buckets replicated in parallel. | +| `verification` | string | default: "None"; enum: \["None", "Check"\] | Verification runs a comparison of source and destination after each successful copy. Check compares sizes and MD5 hashes (size only for multipart objects without a stored MD5) and fails the run on any difference. | + +### Cross-field validation + +- mirror must be set if and only if mode is Mirror: `has(self.mirror) == (has(self.mode) && self.mode == 'Mirror')` + +- a bucket cannot be replicated onto itself: source and destination are the same endpoint: `!(has(self.source.endpoint) && has(self.destination.endpoint) && self.source.endpoint.url == self.destination.endpoint.url) || self.buckets.all(b, b.source != b.destination)` + +## Status + +GarageReplicationStatus defines the observed state of GarageReplication. + +| Field path | Type | Constraints | Description | +| --- | --- | --- | --- | +| `buckets` | array of object | | Buckets reports per-bucket results, keyed by destination bucket. | +| `buckets[].activeJob` | string | | ActiveJob is the Job currently replicating this bucket. | +| `buckets[].destination` | string | required | Destination bucket; the list key. | +| `buckets[].hold` | string | enum: \["SourceEmpty", "MaxDeletesExceeded", "AwaitingTrigger"\] | Hold is set when this bucket must not run again without a person asking. Scheduled runs skip a held bucket. Only the trigger-sync annotation releases it; editing the spec does not. | +| `buckets[].lastCompletionTime` | string | format: "date-time" | LastCompletionTime is when LastJob finished. | +| `buckets[].lastError` | string | enum: \["AuthFailed", "BucketNotFound", "EndpointUnreachable", "SourceEmpty", "MaxDeletesExceeded", "VerificationFailed", "DeadlineExceeded", "TransferErrors", "Unknown"\] | LastError is a fixed reason for a failed LastJob. | +| `buckets[].lastJob` | string | | LastJob is the last finished Job recorded for this bucket. | +| `buckets[].lastMode` | string | | LastMode is the mode LastJob ran in. A Mirror bucket whose last Job did not run in Mirror mode waits for a trigger-sync (hold AwaitingTrigger) before its first Mirror run. | +| `buckets[].lastResult` | string | enum: \["Succeeded", "Failed", "Cancelled"\] | LastResult is the outcome of LastJob. | +| `buckets[].lastRunId` | string | | LastRunID is the run of LastJob. | +| `buckets[].lastStatistics` | object | | LastStatistics are the copy engine's counters for LastJob. Unset when the engine reported none (e.g. the Pod was killed). | +| `buckets[].lastStatistics.bytes` | integer | required; format: "int64" | Bytes is the number of bytes written to the destination. | +| `buckets[].lastStatistics.checks` | integer | required; format: "int64" | Checks is the number of objects compared and found up to date. | +| `buckets[].lastStatistics.deletes` | integer | required; format: "int64" | Deletes is the number of destination objects deleted (Mirror). | +| `buckets[].lastStatistics.differences` | integer | format: "int64" | Differences is the number of differences found by verification. | +| `buckets[].lastStatistics.elapsedSeconds` | integer | required; format: "int64" | ElapsedSeconds is the duration of the copy. | +| `buckets[].lastStatistics.errors` | integer | required; format: "int64" | Errors is the number of errors. | +| `buckets[].lastStatistics.transfers` | integer | required; format: "int64" | Transfers is the number of objects written to the destination. | +| `buckets[].lastSuccessfulTime` | string | format: "date-time" | LastSuccessfulTime is when the last successful Job for this bucket finished. | +| `buckets[].lastVerification` | string | enum: \["Passed", "Failed"\] | LastVerification is the verification outcome when verification ran. | +| `buckets[].source` | string | required | Source bucket. | +| `conditions` | array of object | | Conditions: Ready (configuration resolved; reasons include CredentialsNotAllowed and DestinationConflict), Synced (every bucket succeeded in the last run on the current configuration), Suspended. | +| `conditions[].lastTransitionTime` | string | required; format: "date-time" | lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. | +| `conditions[].message` | string | required; max length: 32768 | message is a human readable message indicating details about the transition. This may be an empty string. | +| `conditions[].observedGeneration` | integer | format: "int64"; min: 0 | observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions\[x\].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. | +| `conditions[].reason` | string | required; min length: 1; max length: 1024; pattern: "^\[A-Za-z\](\[A-Za-z0-9_,:\]*\[A-Za-z0-9_\])?$" | reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. | +| `conditions[].status` | string | required; enum: \["True", "False", "Unknown"\] | status of the condition, one of True, False, Unknown. | +| `conditions[].type` | string | required; max length: 316; pattern: "^(\[a-z0-9\](\[-a-z0-9\]*\[a-z0-9\])?(\\.\[a-z0-9\](\[-a-z0-9\]*\[a-z0-9\])?)*/)?((\[A-Za-z0-9\]\[-A-Za-z0-9_.\]*)?\[A-Za-z0-9\])$" | type of condition in CamelCase or in foo.example.com/CamelCase. | +| `currentRun` | object | | CurrentRun describes the run in progress. | +| `currentRun.buckets` | array of string | | Buckets lists the destination buckets taking part in the run. | +| `currentRun.completionTime` | string | format: "date-time" | CompletionTime is when the run finished. | +| `currentRun.id` | string | required | ID identifies the run; it is part of the Job names. | +| `currentRun.reason` | string | required; enum: \["Initial", "Scheduled", "Triggered"\] | Reason is why the run started. | +| `currentRun.result` | string | enum: \["Succeeded", "Failed", "Cancelled"\] | Result is Succeeded when every bucket of the run succeeded, else Failed. | +| `currentRun.specHash` | string | | SpecHash identifies the configuration the run started with. | +| `currentRun.startTime` | string | required; format: "date-time" | StartTime is when the run started. | +| `currentRun.trigger` | string | | Trigger is the trigger-sync annotation value that started the run. | +| `lastOperation` | object | | LastOperation records the outcome of the last trigger-sync annotation. | +| `lastOperation.error` | string | | Error contains the error message when Succeeded is false | +| `lastOperation.succeeded` | boolean | | Succeeded indicates the operation completed without error | +| `lastOperation.triggeredAt` | string | format: "date-time" | TriggeredAt is when the operation was triggered | +| `lastOperation.type` | string | | Type is the operation type (e.g. "Snapshot", "Repair:Blocks", "Scrub:start") | +| `lastRun` | object | | LastRun describes the most recent finished run. | +| `lastRun.buckets` | array of string | | Buckets lists the destination buckets taking part in the run. | +| `lastRun.completionTime` | string | format: "date-time" | CompletionTime is when the run finished. | +| `lastRun.id` | string | required | ID identifies the run; it is part of the Job names. | +| `lastRun.reason` | string | required; enum: \["Initial", "Scheduled", "Triggered"\] | Reason is why the run started. | +| `lastRun.result` | string | enum: \["Succeeded", "Failed", "Cancelled"\] | Result is Succeeded when every bucket of the run succeeded, else Failed. | +| `lastRun.specHash` | string | | SpecHash identifies the configuration the run started with. | +| `lastRun.startTime` | string | required; format: "date-time" | StartTime is when the run started. | +| `lastRun.trigger` | string | | Trigger is the trigger-sync annotation value that started the run. | +| `lastScheduleTime` | string | format: "date-time" | LastScheduleTime is the start time of the most recent run. | +| `lastSuccessfulSyncTime` | string | format: "date-time" | LastSuccessfulSyncTime is when the most recent run in which every bucket succeeded finished. | +| `lastSyncTime` | string | format: "date-time" | LastSyncTime is when the most recent run finished, whatever its result. | +| `nextSyncTime` | string | format: "date-time" | NextSyncTime is when the next scheduled run starts. Unset without a schedule or while suspended. | +| `observedGeneration` | integer | format: "int64" | ObservedGeneration is the last observed generation. | +| `phase` | string | enum: \["Pending", "Idle", "Syncing", "Suspended", "Failed"\] | Phase summarizes the state. | diff --git a/docs/reference/api/index.md b/docs/reference/api/index.md index b2723796..c72351a8 100644 --- a/docs/reference/api/index.md +++ b/docs/reference/api/index.md @@ -14,3 +14,4 @@ These field tables come from the CRD schemas. Current version schemas are genera | GarageKey | [v1beta1](garagekey-v1beta1.md) | Namespaced | | GarageNode | [v1beta1](garagenode-v1beta1.md) | Namespaced | | GarageReferenceGrant | [v1beta1](garagereferencegrant-v1beta1.md) | Namespaced | +| GarageReplication | [v1beta1](garagereplication-v1beta1.md) | Namespaced | diff --git a/docs/reference/custom-resources.md b/docs/reference/custom-resources.md index f83d171b..a98da4a0 100644 --- a/docs/reference/custom-resources.md +++ b/docs/reference/custom-resources.md @@ -21,8 +21,9 @@ freshness in CI. | `GarageNode` | `garage.rajsingh.info/v1beta1` | Namespaced | One managed, gateway, external, or node-local Garage identity | | `GarageAdminToken` | `garage.rajsingh.info/v1beta1` | Namespaced | Static Admin token Secret template | | `GarageReferenceGrant` | `garage.rajsingh.info/v1beta1` | Namespaced | Allow exact or label-selected namespaces/kinds to make cross-namespace references | +| `GarageReplication` | `garage.rajsingh.info/v1beta1` | Namespaced | Copy objects between two independent Garage clusters through S3 (replication and migration, not backup) | -Short names include `gc`, `gb`, `gk`, `gn`, `gat`, and `grg` as published in +Short names include `gc`, `gb`, `gk`, `gn`, `gat`, `grg`, and `grepl` as published in the CRDs. Confirm them on the target cluster with `kubectl api-resources`. References to a `GarageCluster` use `name` and an optional `namespace`. For @@ -31,7 +32,9 @@ when a `GarageReferenceGrant` in the destination namespace matches the source kind plus its exact namespace or Namespace labels, and the target kind/name. `GarageAdminToken` is namespace-local: its `clusterRef.namespace` must be empty or match the token's namespace, and a `GarageReferenceGrant` does not override that restriction. -`GarageNode` does not support cross-namespace cluster references. +`GarageNode` does not support cross-namespace cluster references. A +`GarageReplication` `clusterRef` (source or destination) follows the same +grant rule as `GarageBucket`, with source kind `GarageReplication`. ## GarageCluster v1beta2 @@ -567,7 +570,8 @@ selected Namespace label can change who the grant authorizes. For Helm namespace-scoped installations, the chart adds only a read-only ClusterRole for Namespace labels; managed Garage resources remain limited to the configured namespaces while `namespaceSelector` entries are evaluated. -Allowed source kinds are `GarageBucket`, `GarageKey`, and `GarageAdminToken`. +Allowed source kinds are `GarageBucket`, `GarageKey`, `GarageAdminToken`, and +`GarageReplication`. `spec.to` narrows the destination kind and optional name; omitted names allow all resources of that kind. Omitting `spec.to` preserves the original grant behavior and allows @@ -584,6 +588,39 @@ Conditions report whether the grant is in use. Removing a grant makes future reconciliation fail closed; it does not itself revoke already-issued Garage permissions. +## GarageReplication + +Copies objects between two independent Garage clusters through their S3 APIs, +for replication and migration. It is not a backup: there are no point-in-time +snapshots and no restore. Each run starts one rclone Job per bucket. The +controller is off by default: start the operator with `--enable-replication` +(chart value `replication.enabled`); without it objects are validated but not +reconciled, and admission returns a warning saying so. See +[Replicate between clusters](../how-to/replication.md) and the +[design record](../design/2026-11-09-garage-replication-design.md). + +| Field | Notes | +| --- | --- | +| `source`, `destination` | Exactly one of `clusterRef` (endpoint and region resolved from the GarageCluster, management handles included) or `endpoint.url` (scheme and host only, + `region`, default `garage`, and optional `caBundleRef` to a ConfigMap). `credentialsSecretRef` names a Secret in the replication's namespace labelled `garage.rajsingh.info/replication-credentials: "true"`, GarageKey-generated Secrets included (set the label in `GarageKey.spec.secretTemplate.labels`). Default keys `access-key-id` / `secret-access-key` | +| `buckets[]` | `source` and `destination`, both required; a list keyed by `destination` (1–50 entries). Buckets must exist. A destination bucket can be written by only one GarageReplication, across all namespaces: the first one keeps it | +| `schedule` | Cron expression (`@hourly`, `CRON_TZ=` accepted). Without a schedule, runs only start from the `trigger-sync` annotation | +| `suspend` | Stops scheduled runs; a running run finishes; `trigger-sync` still works | +| `mode` | `Copy` (default, never deletes) or `Mirror` (requires `mirror.maxDeletes`; the first run waits for `trigger-sync`) | +| `verification` | `None` (default) or `Check` (re-compare after the copy, fail on differences) | +| `transfer` | `concurrency` (4), `checkers` (8), `bandwidthLimit`, `maxConcurrentBuckets` (1) | +| `retry` | `backoffLimit` (2), `activeDeadlineSeconds` (21600) per bucket Job | +| `resources`, `podTemplate` | Job container resources; Pod labels, annotations, node selector, tolerations, affinity, priority class | +| `successfulJobsHistoryLimit`, `failedJobsHistoryLimit` | Finished Jobs kept (1 and 3) | + +Status: `phase` (`Pending`, `Idle`, `Syncing`, `Suspended`, `Failed`), +`currentRun`, `lastRun`, `lastScheduleTime`, `lastSyncTime`, +`lastSuccessfulSyncTime`, `nextSyncTime`, `lastOperation`, and `buckets[]` +keyed by `destination` (`activeJob`, `lastJob`, `lastResult`, `lastError`, +`hold`, `lastVerification`, `lastStatistics`). A `hold` (`SourceEmpty`, +`MaxDeletesExceeded`, `AwaitingTrigger`) is only released by the +`trigger-sync` annotation, never by a spec edit. Conditions: `Ready`, `Synced`, +`Suspended`. + ## Validation and compatibility-only fields Some fields remain in schemas to support conversion or old manifests but are diff --git a/docs/reference/helm.md b/docs/reference/helm.md index a3c4ec77..eb9d6edf 100644 --- a/docs/reference/helm.md +++ b/docs/reference/helm.md @@ -44,6 +44,13 @@ The chart is `oci://ghcr.io/rajsinghtech/charts/garage-operator`. The complete, | `ingress.enabled` | `false` | Renders `--enable-ingress` and the `networking.k8s.io/ingresses` RBAC rules, and watches the Ingresses the operator creates. Required for `GarageBucket.spec.websiteExposure.ingress`; without it the bucket reports `WebsiteExposed=False` / `IngressDisabled` and no Ingress is created or touched. | | `gatewayAPI.enabled` | `false` | Renders `--enable-gateway-api` and the `gateway.networking.k8s.io/httproutes` RBAC rules. Required for `spec.websiteExposure.gateway`; the Gateway API CRDs must already be installed. | +## Cross-cluster replication + +| Value | Default | Notes | +| --- | --- | --- | +| `replication.enabled` | `false` | Renders `--enable-replication`, RBAC on `batch/jobs` and on `GarageReplication` status, and starts the `GarageReplication` controller and its Job informer. Without it, `GarageReplication` objects are validated but not reconciled, and admission warns about it. See [Replicate between clusters](../how-to/replication.md). | +| `replication.image` | `""` | rclone image for replication Jobs (`--replication-image`). Empty uses the built-in digest-pinned `docker.io/rclone/rclone` image; set it to a mirror for air-gapped installs. The image must provide `/bin/sh` and `rclone`. | + ## Operator metrics | Value | Default | Notes | diff --git a/docs/reference/operations.md b/docs/reference/operations.md index 8a9c2632..59f557cc 100644 --- a/docs/reference/operations.md +++ b/docs/reference/operations.md @@ -47,6 +47,12 @@ Maintenance suspension is not an annotation. Set | `cleanup-mpu` | `true` | Delete old incomplete multipart uploads | | `cleanup-mpu-older-than` | duration such as `48h` | Threshold used with `cleanup-mpu`; invalid values default to `24h` | +## `GarageReplication` annotations + +| Annotation | Value | Effect | +| --- | --- | --- | +| `trigger-sync` | any unique value, e.g. a timestamp | Start a run of every bucket as soon as no run is in progress, also while `spec.suspend` is true. It is the only way to release a hold (`AwaitingTrigger`, `SourceEmpty`, `MaxDeletesExceeded`). Removed once the run starts; recorded in `status.lastOperation` | + ## Important conditions The controllers currently write the conditions below. A condition may be absent @@ -90,6 +96,9 @@ package for compatibility but are not emitted as independent status conditions. | `GarageNode` | `Suspended` | Literal condition set while `spec.maintenance.suspended` pauses reconciliation | | `GarageAdminToken` | `Ready` | Static Admin bootstrap material is ready and referenced by the cluster | | `GarageReferenceGrant` | `Ready` / `InUse` | Grant validity and whether resources currently reference it | +| `GarageReplication` | `Ready` | Endpoints, credentials and grants resolve. `False` reasons include `CredentialsNotAllowed` (a credentials Secret lacks the `garage.rajsingh.info/replication-credentials: "true"` label), `DestinationConflict` (an older GarageReplication writes the same destination bucket), `SecretNotFound`, `ReferenceNotGranted`, `ClusterNotFound`, `CABundleNotFound`. Written only with `--enable-replication` | +| `GarageReplication` | `Synced` | `True` when every bucket succeeded in the last run on the current configuration. `False` reasons: `NeverSynced`, `SyncFailed`, `SafetyHold` (a guard stopped a bucket), `AwaitingTrigger` (a Mirror bucket waits for its first trigger), `ConfigChanged` | +| `GarageReplication` | `Suspended` | `True` while `spec.suspend` is set | Key expiry is represented by `GarageKey.status.phase: Expired`, not by a `KeyExpired` condition. `GarageAdminToken` is static bootstrap material, so its diff --git a/docs/reference/samples.md b/docs/reference/samples.md index cd9c7903..0b29e542 100644 --- a/docs/reference/samples.md +++ b/docs/reference/samples.md @@ -36,6 +36,7 @@ new `GarageCluster` manifests should use v1beta2. | [`garage_v1beta1_garagenode_maintenance.yaml`](https://github.com/rajsinghtech/garage-operator/blob/main/config/samples/garage_v1beta1_garagenode_maintenance.yaml) | Per-node reconciliation suspension for hardware/PVC maintenance | | [`garage_v1beta1_garageadmintoken.yaml`](https://github.com/rajsinghtech/garage-operator/blob/main/config/samples/garage_v1beta1_garageadmintoken.yaml) | Static Admin bootstrap Secret material | | [`garage_v1beta1_garagereferencegrant.yaml`](https://github.com/rajsinghtech/garage-operator/blob/main/config/samples/garage_v1beta1_garagereferencegrant.yaml) | Cross-namespace reference authorization | +| [`garage_v1beta1_garagereplication.yaml`](https://github.com/rajsinghtech/garage-operator/blob/main/config/samples/garage_v1beta1_garagereplication.yaml) | Scheduled replication from a managed cluster to an external S3 endpoint | ## COSI samples diff --git a/go.mod b/go.mod index d05b89af..92f5958e 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/onsi/gomega v1.44.0 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.95.2 github.com/prometheus/client_golang v1.25.0 + github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.12.1 go.yaml.in/yaml/v3 v3.0.6 k8s.io/api v0.37.2 diff --git a/go.sum b/go.sum index 51fc2784..849e2f00 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk= github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/internal/controller/garagecluster_factor_migration.go b/internal/controller/garagecluster_factor_migration.go index 386d11af..d32964d6 100644 --- a/internal/controller/garagecluster_factor_migration.go +++ b/internal/controller/garagecluster_factor_migration.go @@ -989,7 +989,7 @@ func factorMigrationPurgeInitContainer(sts *appsv1.StatefulSet, purgeID string) func purgeInitSecurityContext(sts *appsv1.StatefulSet) *corev1.SecurityContext { sc := &corev1.SecurityContext{ AllowPrivilegeEscalation: ptr.To(false), - Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{capabilityAll}}, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, } if uid := purgeInitRunAsUser(sts); uid != nil { diff --git a/internal/controller/garagereplication_controller.go b/internal/controller/garagereplication_controller.go new file mode 100644 index 00000000..befcf7dc --- /dev/null +++ b/internal/controller/garagereplication_controller.go @@ -0,0 +1,970 @@ +/* +Copyright 2026 Raj Singh. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strconv" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + garagev1beta1 "github.com/rajsinghtech/garage-operator/api/v1beta1" + garagev1beta2 "github.com/rajsinghtech/garage-operator/api/v1beta2" +) + +const ( + // AnnotationTriggerSync starts a run of every bucket (including held + // ones) as soon as no run is in progress. It is the only way to release a + // hold. It is removed once the run starts. Use a unique value (e.g. a + // timestamp) for each trigger. + AnnotationTriggerSync = "garage.rajsingh.info/trigger-sync" + + kindGarageReplication = "GarageReplication" + + // Replication conditions. + ConditionReplicationSynced = "Synced" + ConditionReplicationSuspended = "Suspended" + + // Ready=False reasons for an unusable configuration. + ReplicationReasonCredentialsNotAllowed = "CredentialsNotAllowed" + ReplicationReasonDestinationConflict = "DestinationConflict" + + replicationActiveRequeue = 30 * time.Second + replicationNotReadyRequeue = 30 * time.Second + replicationIdleRequeue = time.Hour + + labelJobName = "batch.kubernetes.io/job-name" +) + +// GarageReplicationReconciler reconciles a GarageReplication object. It only +// reads GarageClusters (to resolve S3 endpoints) and owns the per-bucket +// Jobs it creates; it never talks to Garage itself. +type GarageReplicationReconciler struct { + client.Client + // APIReader reads uncached: GarageReferenceGrants (like the bucket and + // key controllers), and Jobs the cache does not show yet. + APIReader client.Reader + Scheme *runtime.Scheme + EventRecorder record.EventRecorder + ClusterDomain string + // Image is the rclone image of the replication Jobs. + Image string + // Now is overridable in tests. + Now func() time.Time +} + +// +kubebuilder:rbac:groups=garage.rajsingh.info,resources=garagereplications,verbs=get;list;watch;patch +// +kubebuilder:rbac:groups=garage.rajsingh.info,resources=garagereplications/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch + +func (r *GarageReplicationReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + return time.Now() +} + +func (r *GarageReplicationReconciler) image() string { + if r.Image != "" { + return r.Image + } + return DefaultReplicationImage +} + +func (r *GarageReplicationReconciler) apiReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +// Reconcile drives the replication runs of one GarageReplication. +func (r *GarageReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + repl := &garagev1beta1.GarageReplication{} + if err := r.Get(ctx, req.NamespacedName, repl); err != nil { + if apierrors.IsNotFound(err) { + deleteReplicationMetrics(req.Namespace, req.Name) + } + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if !repl.DeletionTimestamp.IsZero() { + // Jobs are garbage-collected through their owner reference; object + // data is never touched. + deleteReplicationMetrics(repl.Namespace, repl.Name) + return ctrl.Result{}, nil + } + + now := r.now() + status := &repl.Status + original := status.DeepCopy() + // A spec edit never releases a hold: only trigger-sync does. + status.ObservedGeneration = repl.Generation + + jobs, err := r.listJobs(ctx, repl) + if err != nil { + return ctrl.Result{}, err + } + r.syncBucketStatuses(repl) + if err := r.recordFinishedJobs(ctx, repl, jobs, now); err != nil { + return ctrl.Result{}, err + } + + src, dst, specHash, resolveErr := r.resolve(ctx, repl) + if err := r.cancelObsoleteJobs(ctx, repl, jobs, now); err != nil { + return ctrl.Result{}, err + } + if err := r.pruneJobHistory(ctx, repl, jobs); err != nil { + return ctrl.Result{}, err + } + r.completeRunIfDone(repl, jobs, now) + + result := ctrl.Result{} + if resolveErr != nil { + var cfgErr *replicationConfigError + reason, message := "ResolveFailed", "the replication configuration could not be resolved" + if errors.As(resolveErr, &cfgErr) { + reason, message = cfgErr.reason, cfgErr.message + } else { + log.Error(resolveErr, "resolving GarageReplication") + } + meta.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: garagev1beta1.ConditionReady, Status: metav1.ConditionFalse, + Reason: reason, Message: message, ObservedGeneration: repl.Generation, + }) + result.RequeueAfter = replicationNotReadyRequeue + } else { + meta.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: garagev1beta1.ConditionReady, Status: metav1.ConditionTrue, + Reason: "Resolved", Message: "source, destination and credentials are resolved", ObservedGeneration: repl.Generation, + }) + trigger := repl.Annotations[AnnotationTriggerSync] + consumeTrigger := false + if status.CurrentRun == nil { + r.holdUntriggeredMirrorBuckets(repl) + r.startRunIfDue(repl, trigger, specHash, now) + } + if trigger != "" && status.CurrentRun != nil && status.CurrentRun.Reason == garagev1beta1.ReplicationRunTriggered && status.CurrentRun.Trigger == trigger { + consumeTrigger = true + } + if status.CurrentRun != nil { + // Persist the run before creating its Jobs so a crash never + // starts the same run twice with different IDs. + if err := r.writeStatus(ctx, repl, original, specHash, now); err != nil { + return ctrl.Result{}, err + } + original = repl.Status.DeepCopy() + if consumeTrigger { + if err := r.removeTriggerAnnotation(ctx, repl, trigger); err != nil { + return ctrl.Result{}, err + } + } + if err := r.createRunJobs(ctx, repl, jobs, src, dst, specHash); err != nil { + return ctrl.Result{}, err + } + } + } + + if err := r.writeStatus(ctx, repl, original, specHash, now); err != nil { + return ctrl.Result{}, err + } + updateReplicationMetrics(repl) + + switch { + case result.RequeueAfter > 0: + case status.CurrentRun != nil: + result.RequeueAfter = replicationActiveRequeue + case status.NextSyncTime != nil: + wait := status.NextSyncTime.Sub(now) + result.RequeueAfter = min(max(wait, time.Second), replicationIdleRequeue) + default: + result.RequeueAfter = replicationIdleRequeue + } + return result, nil +} + +// replicationConfigError is a resolution failure caused by the user's +// configuration, reported on the Ready condition with a fixed reason. +type replicationConfigError struct { + reason string + message string +} + +func (e *replicationConfigError) Error() string { return e.reason + ": " + e.message } + +func configErrorf(reason, format string, args ...any) error { + return &replicationConfigError{reason: reason, message: fmt.Sprintf(format, args...)} +} + +// resolve validates the spec, resolves both endpoints, checks that every +// referenced Secret/ConfigMap key exists and that the credentials Secrets +// opted in, and that no older GarageReplication writes the same destination +// buckets. Secret values are never read beyond checking that the keys are +// present and non-empty. +func (r *GarageReplicationReconciler) resolve(ctx context.Context, repl *garagev1beta1.GarageReplication) (resolvedReplicationEndpoint, resolvedReplicationEndpoint, string, error) { + var src, dst resolvedReplicationEndpoint + if _, err := garagev1beta1.ValidateGarageReplicationSpec(repl); err != nil { + return src, dst, "", configErrorf("InvalidSpec", "%s", err.Error()) + } + var err error + if src, err = r.resolveEndpoint(ctx, repl, repl.Spec.Source, "source"); err != nil { + return src, dst, "", err + } + if dst, err = r.resolveEndpoint(ctx, repl, repl.Spec.Destination, "destination"); err != nil { + return src, dst, "", err + } + if garagev1beta1.NormalizeReplicationURL(src.URL) == garagev1beta1.NormalizeReplicationURL(dst.URL) { + for _, b := range repl.Spec.Buckets { + if b.Source == b.Destination { + return src, dst, "", configErrorf("InvalidSpec", "spec.buckets: bucket %q would be replicated onto itself: source and destination resolve to the same endpoint", b.Source) + } + } + } + if err := r.checkDestinationConflict(ctx, repl, dst); err != nil { + return src, dst, "", err + } + return src, dst, replicationSpecHash(repl, src, dst, r.image()), nil +} + +func (r *GarageReplicationReconciler) resolveEndpoint(ctx context.Context, repl *garagev1beta1.GarageReplication, ep garagev1beta1.ReplicationEndpoint, side string) (resolvedReplicationEndpoint, error) { + out := resolvedReplicationEndpoint{Credentials: ep.CredentialsSecretRef} + switch { + case ep.ClusterRef != nil: + name, namespace := ep.ClusterRef.Name, ep.ClusterRef.Namespace + if namespace == "" { + namespace = repl.Namespace + } + if err := garagev1beta1.CheckReferenceGrant(ctx, r.apiReader(), kindGarageReplication, repl.Namespace, kindGarageCluster, namespace, name); err != nil { + if garagev1beta1.IsReferenceGrantDenied(err) { + return out, configErrorf("ReferenceNotGranted", "spec.%s.clusterRef %s/%s needs a GarageReferenceGrant in namespace %s allowing kind GarageReplication from namespace %s", side, namespace, name, namespace, repl.Namespace) + } + return out, err + } + url, region, err := r.clusterS3Endpoint(ctx, name, namespace) + if err != nil { + if apierrors.IsNotFound(err) { + return out, configErrorf("ClusterNotFound", "spec.%s.clusterRef: GarageCluster %s/%s not found", side, namespace, name) + } + if errors.Is(err, errReplicationEndpointUnresolved) { + return out, configErrorf("EndpointUnresolved", "spec.%s.clusterRef: the S3 endpoint of GarageCluster %s/%s is not known yet", side, namespace, name) + } + return out, err + } + out.URL, out.Region = url, region + case ep.Endpoint != nil: + out.URL = ep.Endpoint.URL + out.Region = ep.Endpoint.Region + if out.Region == "" { + out.Region = garagev1beta1.ReplicationDefaultRegion + } + if ca := ep.Endpoint.CABundleRef; ca != nil { + out.CABundle = ca + if err := r.checkCABundle(ctx, repl.Namespace, ca, side); err != nil { + return out, err + } + } + } + + secret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: ep.CredentialsSecretRef.Name, Namespace: repl.Namespace}, secret); err != nil { + if apierrors.IsNotFound(err) { + return out, configErrorf("SecretNotFound", "spec.%s.credentialsSecretRef: Secret %s not found in namespace %s", side, ep.CredentialsSecretRef.Name, repl.Namespace) + } + return out, err + } + // The Job sends the access key ID to an endpoint the creator of the + // GarageReplication chooses: without this opt-in, the right to create one + // would amount to reading any Secret of the namespace. + if secret.Labels[garagev1beta1.LabelReplicationCredentials] != garagev1beta1.LabelReplicationCredentialsValue { + return out, configErrorf(ReplicationReasonCredentialsNotAllowed, "spec.%s.credentialsSecretRef: Secret %s is not labelled %s=%s", side, secret.Name, garagev1beta1.LabelReplicationCredentials, garagev1beta1.LabelReplicationCredentialsValue) + } + idKey, secretKey := garagev1beta1.ReplicationCredentialKeys(ep.CredentialsSecretRef) + for _, key := range []string{idKey, secretKey} { + if len(secret.Data[key]) == 0 { + return out, configErrorf("SecretKeyMissing", "spec.%s.credentialsSecretRef: Secret %s has no non-empty key %q", side, secret.Name, key) + } + } + return out, nil +} + +var errReplicationEndpointUnresolved = errors.New("S3 endpoint not known yet") + +// clusterS3Endpoint returns the S3 URL (no path) and region of a GarageCluster. +func (r *GarageReplicationReconciler) clusterS3Endpoint(ctx context.Context, name, namespace string) (string, string, error) { + cluster := &garagev1beta2.GarageCluster{} + if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, cluster); err != nil { + return "", "", err + } + u, err := ResolveS3Endpoint(cluster, r.ClusterDomain) + if err != nil { + return "", "", errReplicationEndpointUnresolved + } + u.Path = "" + region := defaultS3Region + if cluster.Spec.S3API != nil && cluster.Spec.S3API.Region != "" { + region = cluster.Spec.S3API.Region + } + return u.String(), region, nil +} + +func (r *GarageReplicationReconciler) checkCABundle(ctx context.Context, namespace string, ca *garagev1beta1.ReplicationCABundleRef, side string) error { + key := ca.Key + if key == "" { + key = garagev1beta1.ReplicationDefaultCABundleKey + } + cm := &corev1.ConfigMap{} + if err := r.Get(ctx, types.NamespacedName{Name: ca.Name, Namespace: namespace}, cm); err != nil { + if apierrors.IsNotFound(err) { + return configErrorf("CABundleNotFound", "spec.%s.endpoint.caBundleRef: ConfigMap %s not found", side, ca.Name) + } + return err + } + if cm.Data[key] == "" { + return configErrorf("CABundleNotFound", "spec.%s.endpoint.caBundleRef: ConfigMap %s has no key %q", side, cm.Name, key) + } + return nil +} + +// checkDestinationConflict applies destination exclusivity again after +// admission (two objects admitted concurrently, or created while the webhook +// was unavailable), and also compares resolved URLs, so a clusterRef and an +// endpoint that reach the same cluster are detected. The older object keeps +// the bucket; the other one runs nothing. +func (r *GarageReplicationReconciler) checkDestinationConflict(ctx context.Context, repl *garagev1beta1.GarageReplication, dst resolvedReplicationEndpoint) error { + list := &garagev1beta1.GarageReplicationList{} + if err := r.List(ctx, list); err != nil { + return err + } + selfKey := garagev1beta1.ReplicationEndpointKey(repl.Spec.Destination, repl.Namespace) + selfURL := garagev1beta1.NormalizeReplicationURL(dst.URL) + for i := range list.Items { + other := &list.Items[i] + if other.UID == repl.UID || !other.DeletionTimestamp.IsZero() || !garagev1beta1.ReplicationPrecedes(other, repl) { + continue + } + if garagev1beta1.ReplicationEndpointKey(other.Spec.Destination, other.Namespace) != selfKey && r.destinationURL(ctx, other) != selfURL { + continue + } + for _, b := range repl.Spec.Buckets { + if !slices.ContainsFunc(other.Spec.Buckets, func(ob garagev1beta1.ReplicationBucket) bool { return ob.Destination == b.Destination }) { + continue + } + owner := "a GarageReplication in another namespace" + if other.Namespace == repl.Namespace { + owner = fmt.Sprintf("GarageReplication %q", other.Name) + } + return configErrorf(ReplicationReasonDestinationConflict, "destination bucket %q is already written by %s; a destination bucket can be written by only one GarageReplication", b.Destination, owner) + } + } + return nil +} + +// destinationURL returns the normalized destination URL of another +// GarageReplication, or "" when it cannot be resolved (then only its spec +// key is compared). +func (r *GarageReplicationReconciler) destinationURL(ctx context.Context, repl *garagev1beta1.GarageReplication) string { + ep := repl.Spec.Destination + switch { + case ep.Endpoint != nil: + return garagev1beta1.NormalizeReplicationURL(ep.Endpoint.URL) + case ep.ClusterRef != nil: + namespace := ep.ClusterRef.Namespace + if namespace == "" { + namespace = repl.Namespace + } + url, _, err := r.clusterS3Endpoint(ctx, ep.ClusterRef.Name, namespace) + if err != nil { + return "" + } + return garagev1beta1.NormalizeReplicationURL(url) + } + return "" +} + +// replicationSpecHash covers everything that changes what a Job does. +// Schedule and suspend only change when runs happen and are excluded. +func replicationSpecHash(repl *garagev1beta1.GarageReplication, src, dst resolvedReplicationEndpoint, image string) string { + spec := repl.Spec.DeepCopy() + spec.Schedule = "" + spec.Suspend = false + spec.SuccessfulJobsHistoryLimit = nil + spec.FailedJobsHistoryLimit = nil + raw, _ := json.Marshal(struct { + Spec *garagev1beta1.GarageReplicationSpec + Src resolvedReplicationEndpoint + Dst resolvedReplicationEndpoint + Image string + }{spec, src, dst, image}) + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:])[:16] +} + +func (r *GarageReplicationReconciler) listJobs(ctx context.Context, repl *garagev1beta1.GarageReplication) ([]batchv1.Job, error) { + list := &batchv1.JobList{} + if err := r.List(ctx, list, client.InNamespace(repl.Namespace), client.MatchingLabels{labelReplicationUID: string(repl.UID)}); err != nil { + return nil, err + } + jobs := make([]batchv1.Job, 0, len(list.Items)) + for _, job := range list.Items { + if metav1.IsControlledBy(&job, repl) { + jobs = append(jobs, job) + } + } + sort.Slice(jobs, func(i, j int) bool { + if !jobs[i].CreationTimestamp.Equal(&jobs[j].CreationTimestamp) { + return jobs[i].CreationTimestamp.Before(&jobs[j].CreationTimestamp) + } + return jobs[i].Name < jobs[j].Name + }) + return jobs, nil +} + +// jobDestination is the destination bucket a Job writes, the key of its +// bucket status. +func jobDestination(job *batchv1.Job) string { + return job.Annotations[annotationReplicationDestination] +} + +// syncBucketStatuses keeps one status entry per spec bucket, in spec order, +// keyed by destination. The source follows the spec. +func (r *GarageReplicationReconciler) syncBucketStatuses(repl *garagev1beta1.GarageReplication) { + existing := map[string]garagev1beta1.ReplicationBucketStatus{} + for _, b := range repl.Status.Buckets { + existing[b.Destination] = b + } + out := make([]garagev1beta1.ReplicationBucketStatus, 0, len(repl.Spec.Buckets)) + for _, b := range repl.Spec.Buckets { + entry, ok := existing[b.Destination] + if !ok { + entry = garagev1beta1.ReplicationBucketStatus{Destination: b.Destination} + } + entry.Source = b.Source + out = append(out, entry) + } + repl.Status.Buckets = out +} + +func bucketStatusFor(repl *garagev1beta1.GarageReplication, destination string) *garagev1beta1.ReplicationBucketStatus { + for i := range repl.Status.Buckets { + if repl.Status.Buckets[i].Destination == destination { + return &repl.Status.Buckets[i] + } + } + return nil +} + +// recordCancelled records a Job that stopped before it finished. +func recordCancelled(bucket *garagev1beta1.ReplicationBucketStatus, job, runID string, mode garagev1beta1.ReplicationMode, now time.Time) { + t := metav1.NewTime(now) + bucket.ActiveJob = "" + bucket.LastJob = job + bucket.LastRunID = runID + bucket.LastResult = garagev1beta1.ReplicationResultCancelled + bucket.LastMode = mode + bucket.LastCompletionTime = &t + bucket.LastStatistics = nil + bucket.LastVerification = "" + bucket.LastError = "" +} + +// recordFinishedJobs copies the outcome of every finished Job not recorded +// yet into the bucket status, oldest first, and notes active Jobs. +func (r *GarageReplicationReconciler) recordFinishedJobs(ctx context.Context, repl *garagev1beta1.GarageReplication, jobs []batchv1.Job, now time.Time) error { + listed := map[string]bool{} + for i := range jobs { + listed[jobs[i].Name] = true + } + for i := range repl.Status.Buckets { + bucket := &repl.Status.Buckets[i] + previous := bucket.ActiveJob + bucket.ActiveJob = "" + if previous == "" || listed[previous] { + continue + } + // The Job recorded as active is gone from the cache. Confirm with an + // uncached read so a Job created a moment ago is not mistaken for a + // deleted one; then the deletion was someone stopping the run. + job := &batchv1.Job{} + err := r.apiReader().Get(ctx, types.NamespacedName{Name: previous, Namespace: repl.Namespace}, job) + if err == nil { + bucket.ActiveJob = previous + continue + } + if !apierrors.IsNotFound(err) { + return err + } + runID := "" + if run := repl.Status.CurrentRun; run != nil { + runID = run.ID + } + recordCancelled(bucket, previous, runID, garagev1beta1.EffectiveReplicationMode(&repl.Spec), now) + r.event(repl, corev1.EventTypeNormal, "SyncCancelled", "Job %s was deleted before it finished", previous) + } + for i := range jobs { + job := &jobs[i] + bucket := bucketStatusFor(repl, jobDestination(job)) + if bucket == nil { + continue + } + finished, _, _ := replicationJobFinished(job) + if !finished { + if job.DeletionTimestamp.IsZero() { + bucket.ActiveJob = job.Name + } + continue + } + runID := job.Annotations[annotationReplicationRunID] + if bucket.LastJob == job.Name || replicationRunIDLess(runID, bucket.LastRunID) { + continue + } + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(job.Namespace), client.MatchingLabels{labelJobName: job.Name}); err != nil { + return err + } + outcome := summarizeReplicationJob(job, pods.Items) + completed := metav1.NewTime(now) + if job.Status.CompletionTime != nil { + completed = *job.Status.CompletionTime + } else { + for _, c := range job.Status.Conditions { + if c.Type == batchv1.JobFailed && c.Status == corev1.ConditionTrue { + completed = c.LastTransitionTime + } + } + } + bucket.LastJob = job.Name + bucket.LastRunID = runID + bucket.LastResult = outcome.Result + bucket.LastMode = garagev1beta1.ReplicationMode(job.Annotations[annotationReplicationMode]) + bucket.LastCompletionTime = &completed + bucket.LastStatistics = outcome.Stats + bucket.LastVerification = outcome.Verification + bucket.LastError = outcome.Reason + if outcome.Result == garagev1beta1.ReplicationResultSucceeded { + bucket.LastSuccessfulTime = &completed + continue + } + if hold := replicationSafetyHold(outcome.Reason); hold != "" { + bucket.Hold = hold + } + r.event(repl, corev1.EventTypeWarning, "SyncFailed", "bucket %s -> %s failed: %s", bucket.Source, bucket.Destination, outcome.Reason) + } + return nil +} + +// cancelObsoleteJobs deletes active Jobs whose destination left the spec or +// now has another source, or whose mode deletes more than the current spec +// allows. +func (r *GarageReplicationReconciler) cancelObsoleteJobs(ctx context.Context, repl *garagev1beta1.GarageReplication, jobs []batchv1.Job, now time.Time) error { + wanted := map[string]string{} + for _, b := range repl.Spec.Buckets { + wanted[b.Destination] = b.Source + } + mode := garagev1beta1.EffectiveReplicationMode(&repl.Spec) + for i := range jobs { + job := &jobs[i] + if finished, _, _ := replicationJobFinished(job); finished || !job.DeletionTimestamp.IsZero() { + continue + } + destination := jobDestination(job) + jobMode := garagev1beta1.ReplicationMode(job.Annotations[annotationReplicationMode]) + why := "" + switch source, ok := wanted[destination]; { + case !ok: + why = "its destination bucket was removed from spec.buckets" + case source != job.Annotations[annotationReplicationSource]: + why = "its destination bucket now has another source" + case jobMode == garagev1beta1.ReplicationModeMirror && mode != garagev1beta1.ReplicationModeMirror: + why = fmt.Sprintf("it runs in mode %s and spec.mode is now %s", jobMode, mode) + default: + continue + } + if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)); client.IgnoreNotFound(err) != nil { + return err + } + r.event(repl, corev1.EventTypeNormal, "SyncCancelled", "cancelled Job %s because %s", job.Name, why) + if bucket := bucketStatusFor(repl, destination); bucket != nil { + recordCancelled(bucket, job.Name, job.Annotations[annotationReplicationRunID], jobMode, now) + } + } + return nil +} + +// pruneJobHistory keeps the configured number of finished Jobs. Only Jobs +// already recorded in status are deleted. +func (r *GarageReplicationReconciler) pruneJobHistory(ctx context.Context, repl *garagev1beta1.GarageReplication, jobs []batchv1.Job) error { + keepSucceeded, keepFailed := int32(1), int32(3) + if repl.Spec.SuccessfulJobsHistoryLimit != nil { + keepSucceeded = *repl.Spec.SuccessfulJobsHistoryLimit + } + if repl.Spec.FailedJobsHistoryLimit != nil { + keepFailed = *repl.Spec.FailedJobsHistoryLimit + } + var succeeded, failed []*batchv1.Job + for i := len(jobs) - 1; i >= 0; i-- { // newest first + job := &jobs[i] + finished, ok, _ := replicationJobFinished(job) + if !finished || !job.DeletionTimestamp.IsZero() { + continue + } + if ok { + succeeded = append(succeeded, job) + } else { + failed = append(failed, job) + } + } + prune := func(list []*batchv1.Job, keep int32) error { + for idx, job := range list { + if int32(idx) < keep { + continue + } + bucket := bucketStatusFor(repl, jobDestination(job)) + if bucket != nil && bucket.LastJob != job.Name && !replicationRunIDLess(job.Annotations[annotationReplicationRunID], bucket.LastRunID) { + continue // not recorded yet + } + if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationBackground)); client.IgnoreNotFound(err) != nil { + return err + } + } + return nil + } + if err := prune(succeeded, keepSucceeded); err != nil { + return err + } + return prune(failed, keepFailed) +} + +// completeRunIfDone closes the current run once every participating bucket +// still in the spec has a recorded result for it and no Job of the run is +// active. +func (r *GarageReplicationReconciler) completeRunIfDone(repl *garagev1beta1.GarageReplication, jobs []batchv1.Job, now time.Time) { + run := repl.Status.CurrentRun + if run == nil { + return + } + for i := range jobs { + if finished, _, _ := replicationJobFinished(&jobs[i]); !finished && jobs[i].Annotations[annotationReplicationRunID] == run.ID { + return + } + } + allSucceeded := true + for _, destination := range run.Buckets { + bucket := bucketStatusFor(repl, destination) + if bucket == nil { + continue // removed from spec during the run + } + if bucket.LastRunID != run.ID { + return // not run yet + } + if bucket.LastResult != garagev1beta1.ReplicationResultSucceeded { + allSucceeded = false + } + } + t := metav1.NewTime(now) + done := run.DeepCopy() + done.CompletionTime = &t + done.Result = garagev1beta1.ReplicationResultSucceeded + repl.Status.LastSyncTime = &t + if allSucceeded { + repl.Status.LastSuccessfulSyncTime = &t + r.event(repl, corev1.EventTypeNormal, "SyncSucceeded", "%s run %s replicated %d bucket(s)", run.Reason, run.ID, len(run.Buckets)) + } else { + done.Result = garagev1beta1.ReplicationResultFailed + r.event(repl, corev1.EventTypeWarning, "SyncFailed", "%s run %s finished with failed buckets", run.Reason, run.ID) + } + repl.Status.LastRun = done + repl.Status.CurrentRun = nil +} + +// holdUntriggeredMirrorBuckets puts the AwaitingTrigger hold on every bucket +// that has not run in Mirror mode yet while spec.mode is Mirror, so the first +// deletions only happen after a person triggers them. The hold is dropped +// again when the mode goes back to Copy, which never deletes. Other holds are +// left alone. +func (r *GarageReplicationReconciler) holdUntriggeredMirrorBuckets(repl *garagev1beta1.GarageReplication) { + mirror := garagev1beta1.EffectiveReplicationMode(&repl.Spec) == garagev1beta1.ReplicationModeMirror + for i := range repl.Status.Buckets { + bucket := &repl.Status.Buckets[i] + switch { + case !mirror && bucket.Hold == garagev1beta1.ReplicationHoldAwaitingTrigger: + bucket.Hold = "" + case mirror && bucket.Hold == "" && bucket.LastMode != garagev1beta1.ReplicationModeMirror: + bucket.Hold = garagev1beta1.ReplicationHoldAwaitingTrigger + } + } +} + +// startRunIfDue starts a run when one is triggered, when the object was +// just created, or when the schedule says so. Runs never overlap: missed +// schedule times while a run is active collapse into one. A trigger releases +// every hold; other runs skip held buckets. +func (r *GarageReplicationReconciler) startRunIfDue(repl *garagev1beta1.GarageReplication, trigger, specHash string, now time.Time) { + status := &repl.Status + var reason garagev1beta1.ReplicationRunReason + switch { + case trigger != "": + reason = garagev1beta1.ReplicationRunTriggered + case repl.Spec.Suspend: + return + case status.LastScheduleTime == nil: + reason = garagev1beta1.ReplicationRunInitial + default: + sched, err := garagev1beta1.ParseReplicationSchedule(repl.Spec.Schedule) + if err != nil || sched == nil { + return + } + if now.Before(sched.Next(status.LastScheduleTime.Time)) { + return + } + reason = garagev1beta1.ReplicationRunScheduled + } + + var buckets []string + for _, b := range repl.Spec.Buckets { + bucket := bucketStatusFor(repl, b.Destination) + if reason == garagev1beta1.ReplicationRunTriggered { + if bucket != nil { + bucket.Hold = "" + } + } else if bucket != nil && bucket.Hold != "" { + continue + } + buckets = append(buckets, b.Destination) + } + start := metav1.NewTime(now) + status.LastScheduleTime = &start + if reason == garagev1beta1.ReplicationRunTriggered { + status.LastOperation = &garagev1beta1.LastOperationStatus{Type: "TriggerSync", TriggeredAt: &start, Succeeded: true} + } + if len(buckets) == 0 { + r.event(repl, corev1.EventTypeWarning, "SyncSkipped", "every bucket is held (see status.buckets[].hold); set the %s annotation to run them", AnnotationTriggerSync) + return + } + runID := replicationRunID(now) + if status.LastRun != nil && !replicationRunIDLess(status.LastRun.ID, runID) { + // Two runs in the same second (a trigger right after a run): + // keep run IDs, and therefore Job names, strictly increasing. + runID = replicationRunID(time.Unix(parseReplicationRunID(status.LastRun.ID)+1, 0)) + } + status.CurrentRun = &garagev1beta1.ReplicationRun{ + ID: runID, Reason: reason, StartTime: start, SpecHash: specHash, Buckets: buckets, + } + if reason == garagev1beta1.ReplicationRunTriggered { + status.CurrentRun.Trigger = trigger + } + r.event(repl, corev1.EventTypeNormal, "SyncStarted", "%s run %s started for %d bucket(s)", reason, runID, len(buckets)) +} + +// createRunJobs creates the current run's missing Jobs, at most +// maxConcurrentBuckets active at a time. +func (r *GarageReplicationReconciler) createRunJobs(ctx context.Context, repl *garagev1beta1.GarageReplication, jobs []batchv1.Job, src, dst resolvedReplicationEndpoint, specHash string) error { + run := repl.Status.CurrentRun + maxActive := int32(1) + if t := repl.Spec.Transfer; t != nil && t.MaxConcurrentBuckets > 0 { + maxActive = t.MaxConcurrentBuckets + } + existing := map[string]bool{} + active := int32(0) + for i := range jobs { + existing[jobs[i].Name] = true + if finished, _, _ := replicationJobFinished(&jobs[i]); !finished { + active++ + } + } + for _, b := range repl.Spec.Buckets { + if !slices.Contains(run.Buckets, b.Destination) { + continue + } + bucket := bucketStatusFor(repl, b.Destination) + name := replicationJobName(repl.Name, run.ID, b) + if existing[name] || (bucket != nil && !replicationRunIDLess(bucket.LastRunID, run.ID)) { + continue + } + if bucket != nil && bucket.ActiveJob != "" { + continue // a Job of an earlier run still runs for this bucket + } + if active >= maxActive { + return nil + } + job := buildReplicationJob(replicationJobInput{ + Replication: repl, Bucket: b, RunID: run.ID, + SpecHash: specHash, Image: r.image(), Source: src, Destination: dst, + }) + if err := controllerutil.SetControllerReference(repl, job, r.Scheme); err != nil { + return err + } + if err := r.Create(ctx, job); err != nil { + if apierrors.IsAlreadyExists(err) { + continue + } + return err + } + active++ + if bucket != nil { + bucket.ActiveJob = job.Name + } + } + return nil +} + +func (r *GarageReplicationReconciler) removeTriggerAnnotation(ctx context.Context, repl *garagev1beta1.GarageReplication, value string) error { + latest := &garagev1beta1.GarageReplication{} + if err := r.Get(ctx, client.ObjectKeyFromObject(repl), latest); err != nil { + return err + } + if latest.Annotations[AnnotationTriggerSync] != value { + return nil + } + patch := client.MergeFrom(latest.DeepCopy()) + delete(latest.Annotations, AnnotationTriggerSync) + if err := r.Patch(ctx, latest, patch); err != nil { + return err + } + repl.Annotations = latest.Annotations + repl.ResourceVersion = latest.ResourceVersion + return nil +} + +// writeStatus derives phase, conditions and the next sync time, then writes +// status when it changed. +func (r *GarageReplicationReconciler) writeStatus(ctx context.Context, repl *garagev1beta1.GarageReplication, original *garagev1beta1.GarageReplicationStatus, specHash string, now time.Time) error { + status := &repl.Status + ready := meta.IsStatusConditionTrue(status.Conditions, garagev1beta1.ConditionReady) + + status.NextSyncTime = nil + if sched, err := garagev1beta1.ParseReplicationSchedule(repl.Spec.Schedule); err == nil && sched != nil && !repl.Spec.Suspend { + base := now + if status.LastScheduleTime != nil { + base = status.LastScheduleTime.Time + } + next := metav1.NewTime(sched.Next(base)) + status.NextSyncTime = &next + } + + safetyHold, awaitingTrigger := false, false + for _, b := range status.Buckets { + switch b.Hold { + case "": + case garagev1beta1.ReplicationHoldAwaitingTrigger: + awaitingTrigger = true + default: + safetyHold = true + } + } + switch { + case status.CurrentRun != nil: + status.Phase = garagev1beta1.ReplicationPhaseSyncing + case !ready: + status.Phase = garagev1beta1.ReplicationPhasePending + case repl.Spec.Suspend: + status.Phase = garagev1beta1.ReplicationPhaseSuspended + case status.LastRun != nil && status.LastRun.Result == garagev1beta1.ReplicationResultFailed: + status.Phase = garagev1beta1.ReplicationPhaseFailed + default: + status.Phase = garagev1beta1.ReplicationPhaseIdle + } + + synced := metav1.Condition{Type: ConditionReplicationSynced, Status: metav1.ConditionFalse, ObservedGeneration: repl.Generation} + switch { + case safetyHold: + synced.Reason, synced.Message = "SafetyHold", fmt.Sprintf("a safety guard stopped at least one bucket (see status.buckets[].hold); check both sides, then set the %s annotation to resume", AnnotationTriggerSync) + case awaitingTrigger: + synced.Reason, synced.Message = "AwaitingTrigger", fmt.Sprintf("mode Mirror deletes destination objects; its first run starts on the %s annotation", AnnotationTriggerSync) + case status.LastRun == nil: + synced.Reason, synced.Message = "NeverSynced", "no run has finished yet" + case status.LastRun.Result != garagev1beta1.ReplicationResultSucceeded: + synced.Reason, synced.Message = "SyncFailed", "the last run had failed buckets (see status.buckets[].lastError)" + case specHash != "" && status.LastRun.SpecHash != specHash: + synced.Reason, synced.Message = "ConfigChanged", "the configuration changed since the last successful run" + default: + synced.Status, synced.Reason, synced.Message = metav1.ConditionTrue, "Synced", "every bucket succeeded in the last run" + } + meta.SetStatusCondition(&status.Conditions, synced) + + suspended := metav1.Condition{Type: ConditionReplicationSuspended, Status: metav1.ConditionFalse, Reason: "Active", Message: "runs start on schedule", ObservedGeneration: repl.Generation} + if repl.Spec.Suspend { + suspended.Status, suspended.Reason, suspended.Message = metav1.ConditionTrue, "Suspended", "spec.suspend is true; no scheduled run starts" + } + meta.SetStatusCondition(&status.Conditions, suspended) + + if equality.Semantic.DeepEqual(original, status) { + return nil + } + return r.Status().Update(ctx, repl) +} + +func (r *GarageReplicationReconciler) event(repl *garagev1beta1.GarageReplication, eventType, reason, format string, args ...any) { + if r.EventRecorder == nil { + return + } + r.EventRecorder.Eventf(repl, eventType, reason, format, args...) +} + +// parseReplicationRunID returns the Unix time of a run ID, or -1 for an +// empty or malformed ID. +func parseReplicationRunID(id string) int64 { + n, err := strconv.ParseInt(id, 36, 64) + if err != nil { + return -1 + } + return n +} + +// replicationRunIDLess orders run IDs by time; an empty ID sorts first. +func replicationRunIDLess(a, b string) bool { + return parseReplicationRunID(a) < parseReplicationRunID(b) +} + +// SetupWithManager sets up the controller with the Manager. Status-only +// writes do not re-enter Reconcile; every path returns RequeueAfter. +func (r *GarageReplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&garagev1beta1.GarageReplication{}, builder.WithPredicates(predicate.Or( + predicate.GenerationChangedPredicate{}, + predicate.AnnotationChangedPredicate{}, + ))). + Owns(&batchv1.Job{}). + Named("garagereplication"). + Complete(r) +} diff --git a/internal/controller/garagereplication_controller_test.go b/internal/controller/garagereplication_controller_test.go new file mode 100644 index 00000000..8d29194b --- /dev/null +++ b/internal/controller/garagereplication_controller_test.go @@ -0,0 +1,908 @@ +/* +Copyright 2026 Raj Singh. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + garagev1beta1 "github.com/rajsinghtech/garage-operator/api/v1beta1" + garagev1beta2 "github.com/rajsinghtech/garage-operator/api/v1beta2" +) + +const ( + replTestNamespace = "dr" + replTestName = "prod-to-dr" + replTestSrcSecret = "src-creds" + replTestDstSecret = "dst-creds" + replTestSecretID = "GKsecretmaterialid0000000" + replTestSecretKey = "super-secret-access-key-material-0123456789" + replTestDstURL = "https://s3.dr.example.com" +) + +type replicationHarness struct { + t *testing.T + r *GarageReplicationReconciler + c client.Client + now time.Time + scheme *runtime.Scheme +} + +func newReplicationHarness(t *testing.T, repl *garagev1beta1.GarageReplication, extra ...client.Object) *replicationHarness { + t.Helper() + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{ + corev1.AddToScheme, batchv1.AddToScheme, garagev1beta1.AddToScheme, garagev1beta2.AddToScheme, + } { + if err := add(scheme); err != nil { + t.Fatal(err) + } + } + objects := []client.Object{ + replicationTestSecret(replTestSrcSecret, true), + replicationTestSecret(replTestDstSecret, true), + } + if repl != nil { + objects = append(objects, repl) + } + objects = append(objects, extra...) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&garagev1beta1.GarageReplication{}, &batchv1.Job{}). + WithObjects(objects...).Build() + h := &replicationHarness{t: t, c: c, scheme: scheme, now: time.Date(2026, 11, 9, 12, 0, 0, 0, time.UTC)} + h.r = &GarageReplicationReconciler{Client: c, APIReader: c, Scheme: scheme, ClusterDomain: "cluster.local", Now: func() time.Time { return h.now }} + return h +} + +func replicationTestSecret(name string, optedIn bool) *corev1.Secret { + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: replTestNamespace}, + Data: map[string][]byte{ + garagev1beta1.ReplicationDefaultAccessKeyIDKey: []byte(replTestSecretID), + garagev1beta1.ReplicationDefaultSecretAccessKeyKey: []byte(replTestSecretKey), + }, + } + if optedIn { + s.Labels = map[string]string{garagev1beta1.LabelReplicationCredentials: "true"} + } + return s +} + +func replicationFixture() *garagev1beta1.GarageReplication { + return &garagev1beta1.GarageReplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: replTestName, Namespace: replTestNamespace, UID: types.UID("repl-uid"), Generation: 1, + CreationTimestamp: metav1.NewTime(time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)), + }, + Spec: garagev1beta1.GarageReplicationSpec{ + Source: garagev1beta1.ReplicationEndpoint{ + Endpoint: &garagev1beta1.ReplicationS3Endpoint{URL: "https://s3.prod.example.com"}, + CredentialsSecretRef: garagev1beta1.ReplicationCredentialsRef{Name: replTestSrcSecret}, + }, + Destination: garagev1beta1.ReplicationEndpoint{ + Endpoint: &garagev1beta1.ReplicationS3Endpoint{URL: replTestDstURL, Region: "dr"}, + CredentialsSecretRef: garagev1beta1.ReplicationCredentialsRef{Name: replTestDstSecret}, + }, + Buckets: []garagev1beta1.ReplicationBucket{ + {Source: "medias", Destination: "medias"}, + {Source: "pg-backups", Destination: "pg-backups-dr"}, + }, + Schedule: "*/15 * * * *", + }, + } +} + +// mirrorFixture is a one-bucket Mirror replication. +func mirrorFixture(maxDeletes int64) *garagev1beta1.GarageReplication { + repl := replicationFixture() + repl.Spec.Mode = garagev1beta1.ReplicationModeMirror + repl.Spec.Mirror = &garagev1beta1.ReplicationMirrorConfig{MaxDeletes: maxDeletes} + repl.Spec.Buckets = repl.Spec.Buckets[:1] + return repl +} + +func (h *replicationHarness) reconcile() ctrl.Result { + h.t.Helper() + res, err := h.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Name: replTestName, Namespace: replTestNamespace}}) + if err != nil { + h.t.Fatalf("reconcile: %v", err) + } + return res +} + +func (h *replicationHarness) get() *garagev1beta1.GarageReplication { + h.t.Helper() + repl := &garagev1beta1.GarageReplication{} + if err := h.c.Get(context.Background(), types.NamespacedName{Name: replTestName, Namespace: replTestNamespace}, repl); err != nil { + h.t.Fatal(err) + } + return repl +} + +func (h *replicationHarness) update(mutate func(*garagev1beta1.GarageReplication)) { + h.t.Helper() + repl := h.get() + mutate(repl) + repl.Generation++ + if err := h.c.Update(context.Background(), repl); err != nil { + h.t.Fatal(err) + } +} + +func (h *replicationHarness) trigger(value string) { + h.t.Helper() + h.update(func(r *garagev1beta1.GarageReplication) { + if r.Annotations == nil { + r.Annotations = map[string]string{} + } + r.Annotations[AnnotationTriggerSync] = value + }) +} + +func (h *replicationHarness) jobs() []batchv1.Job { + h.t.Helper() + list := &batchv1.JobList{} + if err := h.c.List(context.Background(), list, client.InNamespace(replTestNamespace)); err != nil { + h.t.Fatal(err) + } + return list.Items +} + +func (h *replicationHarness) activeJobs() []batchv1.Job { + var out []batchv1.Job + for _, j := range h.jobs() { + if finished, _, _ := replicationJobFinished(&j); !finished { + out = append(out, j) + } + } + return out +} + +// finishJob marks a Job finished the way the Job controller would and +// creates its Pod with the given termination message. +func (h *replicationHarness) finishJob(name string, succeeded bool, jobReason, message string) { + h.t.Helper() + job := &batchv1.Job{} + if err := h.c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: replTestNamespace}, job); err != nil { + h.t.Fatal(err) + } + t := metav1.NewTime(h.now) + cond := batchv1.JobCondition{Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Reason: jobReason, LastTransitionTime: t} + if succeeded { + cond.Type = batchv1.JobComplete + job.Status.CompletionTime = &t + } + job.Status.Conditions = append(job.Status.Conditions, cond) + if err := h.c.Status().Update(context.Background(), job); err != nil { + h.t.Fatal(err) + } + exit := int32(0) + if !succeeded { + exit = 1 + } + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name + "-pod", Namespace: replTestNamespace, Labels: map[string]string{labelJobName: name}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: replicationContainerName, + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: exit, Message: message}}, + }}}, + } + if err := h.c.Create(context.Background(), pod); err != nil { + h.t.Fatal(err) + } +} + +const ( + replTestSuccessMessage = "garage-replication/v1 exit=0 reason= verify= differences= stats=1 transfers=6 bytes=37 checks=2 deletes=0 errors=0 elapsedTime=1.6\n" + replTestMaxDeletesMessage = "garage-replication/v1 exit=65 reason=MaxDeletesExceeded verify= differences= stats=1 transfers=0 bytes=0 checks=5 deletes=10 errors=0 elapsedTime=1" +) + +func bucketStatus(t *testing.T, repl *garagev1beta1.GarageReplication, destination string) garagev1beta1.ReplicationBucketStatus { + t.Helper() + for _, b := range repl.Status.Buckets { + if b.Destination == destination { + return b + } + } + t.Fatalf("no status for destination bucket %s", destination) + return garagev1beta1.ReplicationBucketStatus{} +} + +func readyCondition(t *testing.T, repl *garagev1beta1.GarageReplication) *metav1.Condition { + t.Helper() + c := meta.FindStatusCondition(repl.Status.Conditions, garagev1beta1.ConditionReady) + if c == nil { + t.Fatal("no Ready condition") + } + return c +} + +func TestReplicationInitialRunRunsBucketsOneAtATimeAndRecordsResults(t *testing.T) { + h := newReplicationHarness(t, replicationFixture()) + + h.reconcile() + h.reconcile() // idempotent: no second Job for the same work + jobs := h.jobs() + if len(jobs) != 1 { + t.Fatalf("want 1 Job (maxConcurrentBuckets defaults to 1), got %d", len(jobs)) + } + repl := h.get() + if repl.Status.Phase != garagev1beta1.ReplicationPhaseSyncing || repl.Status.CurrentRun == nil || repl.Status.CurrentRun.Reason != garagev1beta1.ReplicationRunInitial { + t.Fatalf("want an Initial run in progress, got phase=%s run=%+v", repl.Status.Phase, repl.Status.CurrentRun) + } + if !meta.IsStatusConditionTrue(repl.Status.Conditions, garagev1beta1.ConditionReady) { + t.Fatalf("Ready should be True: %+v", repl.Status.Conditions) + } + + h.now = h.now.Add(time.Minute) + h.finishJob(jobs[0].Name, true, "", replTestSuccessMessage) + h.reconcile() + repl = h.get() + medias := bucketStatus(t, repl, "medias") + if medias.LastResult != garagev1beta1.ReplicationResultSucceeded || medias.LastMode != garagev1beta1.ReplicationModeCopy || + medias.LastStatistics == nil || medias.LastStatistics.Transfers != 6 || medias.LastStatistics.Bytes != 37 || medias.LastStatistics.ElapsedSeconds != 2 { + t.Fatalf("medias not recorded: %+v stats=%+v", medias, medias.LastStatistics) + } + active := h.activeJobs() + if len(active) != 1 || active[0].Annotations[annotationReplicationDestination] != "pg-backups-dr" { + t.Fatalf("want the second bucket's Job next, got %+v", active) + } + + h.now = h.now.Add(time.Minute) + h.finishJob(active[0].Name, true, "", replTestSuccessMessage) + res := h.reconcile() + repl = h.get() + if repl.Status.CurrentRun != nil || repl.Status.LastRun == nil || repl.Status.LastRun.Result != garagev1beta1.ReplicationResultSucceeded { + t.Fatalf("run should be complete: current=%+v last=%+v", repl.Status.CurrentRun, repl.Status.LastRun) + } + if repl.Status.LastSuccessfulSyncTime == nil || !repl.Status.LastSuccessfulSyncTime.Time.Equal(h.now) { + t.Fatalf("lastSuccessfulSyncTime = %v, want %v", repl.Status.LastSuccessfulSyncTime, h.now) + } + if !meta.IsStatusConditionTrue(repl.Status.Conditions, ConditionReplicationSynced) || repl.Status.Phase != garagev1beta1.ReplicationPhaseIdle { + t.Fatalf("want Synced=True and Idle, got phase=%s conditions=%+v", repl.Status.Phase, repl.Status.Conditions) + } + wantNext := time.Date(2026, 11, 9, 12, 15, 0, 0, time.UTC) + if repl.Status.NextSyncTime == nil || !repl.Status.NextSyncTime.Time.Equal(wantNext) { + t.Fatalf("nextSyncTime = %v, want %v", repl.Status.NextSyncTime, wantNext) + } + if res.RequeueAfter != wantNext.Sub(h.now) { + t.Fatalf("requeue %v, want until next sync %v", res.RequeueAfter, wantNext.Sub(h.now)) + } +} + +func completeRun(t *testing.T, h *replicationHarness, message string, succeeded bool) { + t.Helper() + for i := 0; i < 10; i++ { + h.reconcile() + active := h.activeJobs() + if len(active) == 0 { + if h.get().Status.CurrentRun == nil { + return + } + continue + } + for _, j := range active { + h.finishJob(j.Name, succeeded, "", message) + } + } + t.Fatal("run did not complete") +} + +func TestReplicationScheduledRunsCollapseAndNeverOverlap(t *testing.T) { + h := newReplicationHarness(t, replicationFixture()) + completeRun(t, h, replTestSuccessMessage, true) + + h.now = h.now.Add(10 * time.Minute) + h.reconcile() + if len(h.activeJobs()) != 0 { + t.Fatal("no run is due before the schedule") + } + + // Three schedule ticks pass while the operator is down: one run. + h.now = h.now.Add(50 * time.Minute) + h.reconcile() + repl := h.get() + if repl.Status.CurrentRun == nil || repl.Status.CurrentRun.Reason != garagev1beta1.ReplicationRunScheduled { + t.Fatalf("want a Scheduled run, got %+v", repl.Status.CurrentRun) + } + firstRun := repl.Status.CurrentRun.ID + + // A long run passes the next tick: no second run starts meanwhile. + h.now = h.now.Add(20 * time.Minute) + h.reconcile() + if got := h.get().Status.CurrentRun.ID; got != firstRun { + t.Fatalf("runs overlapped: %s then %s", firstRun, got) + } + if len(h.activeJobs()) != 1 { + t.Fatalf("want exactly one active Job, got %d", len(h.activeJobs())) + } +} + +func TestReplicationSuspendStopsScheduleButTriggerStillRuns(t *testing.T) { + h := newReplicationHarness(t, replicationFixture()) + completeRun(t, h, replTestSuccessMessage, true) + h.update(func(r *garagev1beta1.GarageReplication) { r.Spec.Suspend = true }) + + h.now = h.now.Add(2 * time.Hour) + h.reconcile() + repl := h.get() + if repl.Status.CurrentRun != nil || repl.Status.Phase != garagev1beta1.ReplicationPhaseSuspended || repl.Status.NextSyncTime != nil { + t.Fatalf("suspended replication must not run: phase=%s run=%+v next=%v", repl.Status.Phase, repl.Status.CurrentRun, repl.Status.NextSyncTime) + } + if !meta.IsStatusConditionTrue(repl.Status.Conditions, ConditionReplicationSuspended) { + t.Fatal("Suspended condition should be True") + } + + h.trigger("final-sync-1") + h.reconcile() + repl = h.get() + if repl.Status.CurrentRun == nil || repl.Status.CurrentRun.Reason != garagev1beta1.ReplicationRunTriggered { + t.Fatalf("trigger should start a run even while suspended, got %+v", repl.Status.CurrentRun) + } + if _, ok := repl.Annotations[AnnotationTriggerSync]; ok { + t.Fatal("trigger annotation should be consumed") + } + if repl.Status.LastOperation == nil || repl.Status.LastOperation.Type != "TriggerSync" || !repl.Status.LastOperation.Succeeded { + t.Fatalf("lastOperation not recorded: %+v", repl.Status.LastOperation) + } + + // Resume: the schedule takes over again. + completeRun(t, h, replTestSuccessMessage, true) + h.update(func(r *garagev1beta1.GarageReplication) { r.Spec.Suspend = false }) + h.reconcile() + if h.get().Status.NextSyncTime == nil { + t.Fatal("resumed replication should have a next sync time") + } +} + +func TestReplicationTriggerDuringRunWaitsForIt(t *testing.T) { + h := newReplicationHarness(t, replicationFixture()) + h.reconcile() + initial := h.get().Status.CurrentRun.ID + h.trigger("after-freeze") + h.reconcile() + repl := h.get() + if repl.Status.CurrentRun.ID != initial || repl.Annotations[AnnotationTriggerSync] != "after-freeze" { + t.Fatal("a trigger must wait for the active run and stay pending") + } + completeRun(t, h, replTestSuccessMessage, true) + repl = h.get() + if repl.Status.LastRun == nil || repl.Status.LastRun.Reason != garagev1beta1.ReplicationRunTriggered || repl.Status.LastRun.Trigger != "after-freeze" { + t.Fatalf("the triggered run should have run after the initial one, last run %+v", repl.Status.LastRun) + } +} + +func TestReplicationFailureIsReportedWithFixedReason(t *testing.T) { + h := newReplicationHarness(t, replicationFixture()) + completeRun(t, h, "garage-replication/v1 exit=1 reason=AuthFailed verify= differences= stats=1 transfers=0 bytes=0 checks=0 deletes=0 errors=1 elapsedTime=0.1", false) + repl := h.get() + b := bucketStatus(t, repl, "medias") + if b.LastResult != garagev1beta1.ReplicationResultFailed || b.LastError != garagev1beta1.ReplicationFailureAuthFailed || b.Hold != "" { + t.Fatalf("want Failed/AuthFailed without hold, got %+v", b) + } + if repl.Status.Phase != garagev1beta1.ReplicationPhaseFailed || repl.Status.LastSuccessfulSyncTime != nil { + t.Fatalf("phase=%s lastSuccess=%v", repl.Status.Phase, repl.Status.LastSuccessfulSyncTime) + } + synced := meta.FindStatusCondition(repl.Status.Conditions, ConditionReplicationSynced) + if synced == nil || synced.Status != metav1.ConditionFalse || synced.Reason != "SyncFailed" { + t.Fatalf("Synced condition %+v", synced) + } + + // A Pod killed before writing its message yields no invented statistics. + h.now = h.now.Add(time.Hour) + h.reconcile() + active := h.activeJobs() + h.finishJob(active[0].Name, false, batchv1.JobReasonDeadlineExceeded, "") + h.reconcile() + b = bucketStatus(t, h.get(), "medias") + if b.LastError != garagev1beta1.ReplicationFailureDeadlineExceeded || b.LastStatistics != nil { + t.Fatalf("want DeadlineExceeded without statistics, got %+v", b) + } +} + +func TestReplicationFirstMirrorRunWaitsForTrigger(t *testing.T) { + h := newReplicationHarness(t, mirrorFixture(10)) + h.reconcile() + if len(h.jobs()) != 0 { + t.Fatal("a new Mirror replication must not run before a trigger") + } + repl := h.get() + if b := bucketStatus(t, repl, "medias"); b.Hold != garagev1beta1.ReplicationHoldAwaitingTrigger { + t.Fatalf("want hold AwaitingTrigger, got %+v", b) + } + if c := meta.FindStatusCondition(repl.Status.Conditions, ConditionReplicationSynced); c == nil || c.Reason != "AwaitingTrigger" { + t.Fatalf("want Synced reason AwaitingTrigger, got %+v", c) + } + + // Schedule ticks and spec edits do not start it. + h.now = h.now.Add(time.Hour) + h.update(func(r *garagev1beta1.GarageReplication) { r.Spec.Mirror.MaxDeletes = 20 }) + h.reconcile() + if len(h.jobs()) != 0 { + t.Fatal("only a trigger may start the first Mirror run") + } + + h.trigger("first-mirror") + completeRun(t, h, replTestSuccessMessage, true) + b := bucketStatus(t, h.get(), "medias") + if b.Hold != "" || b.LastMode != garagev1beta1.ReplicationModeMirror || b.LastResult != garagev1beta1.ReplicationResultSucceeded { + t.Fatalf("triggered Mirror run not recorded: %+v", b) + } + + // Later scheduled Mirror runs need no trigger. + h.now = h.now.Add(time.Hour) + h.reconcile() + if len(h.activeJobs()) != 1 { + t.Fatal("a scheduled Mirror run should start once a Mirror run happened") + } +} + +func TestReplicationSwitchToMirrorWaitsForTrigger(t *testing.T) { + repl := replicationFixture() + repl.Spec.Buckets = repl.Spec.Buckets[:1] + h := newReplicationHarness(t, repl) + completeRun(t, h, replTestSuccessMessage, true) + + h.update(func(r *garagev1beta1.GarageReplication) { + r.Spec.Mode = garagev1beta1.ReplicationModeMirror + r.Spec.Mirror = &garagev1beta1.ReplicationMirrorConfig{MaxDeletes: 10} + }) + h.now = h.now.Add(time.Hour) + h.reconcile() + if len(h.activeJobs()) != 0 || bucketStatus(t, h.get(), "medias").Hold != garagev1beta1.ReplicationHoldAwaitingTrigger { + t.Fatal("switching from Copy to Mirror must wait for a trigger") + } + + // Going back to Copy drops the hold: Copy never deletes. + h.update(func(r *garagev1beta1.GarageReplication) { + r.Spec.Mode = garagev1beta1.ReplicationModeCopy + r.Spec.Mirror = nil + }) + h.reconcile() + if bucketStatus(t, h.get(), "medias").Hold != "" { + t.Fatal("AwaitingTrigger must not hold a Copy bucket") + } +} + +func TestReplicationSafetyHoldIsReleasedOnlyByTrigger(t *testing.T) { + h := newReplicationHarness(t, mirrorFixture(10)) + h.trigger("first-mirror") + completeRun(t, h, replTestMaxDeletesMessage, false) + b := bucketStatus(t, h.get(), "medias") + if b.Hold != garagev1beta1.ReplicationHoldMaxDeletesExceeded || b.LastError != garagev1beta1.ReplicationFailureMaxDeletesExceeded { + t.Fatalf("want hold MaxDeletesExceeded, got %+v", b) + } + + h.now = h.now.Add(time.Hour) + h.reconcile() + if len(h.activeJobs()) != 0 { + t.Fatal("a scheduled run must skip a held bucket") + } + got := h.get() + if got.Status.CurrentRun != nil || meta.FindStatusCondition(got.Status.Conditions, ConditionReplicationSynced).Reason != "SafetyHold" { + t.Fatalf("want SafetyHold, got %+v", got.Status.Conditions) + } + + // Raising the cap is a spec edit: it must not release the hold, and + // neither must an unrelated edit. + h.update(func(r *garagev1beta1.GarageReplication) { r.Spec.Mirror.MaxDeletes = 1000 }) + h.update(func(r *garagev1beta1.GarageReplication) { + r.Spec.Transfer = &garagev1beta1.ReplicationTransferConfig{BandwidthLimit: "10M"} + }) + h.now = h.now.Add(time.Hour) + h.reconcile() + if len(h.activeJobs()) != 0 || bucketStatus(t, h.get(), "medias").Hold != garagev1beta1.ReplicationHoldMaxDeletesExceeded { + t.Fatal("a spec edit must not release a safety hold") + } + + h.trigger("checked-both-sides") + h.reconcile() + if len(h.activeJobs()) != 1 || bucketStatus(t, h.get(), "medias").Hold != "" { + t.Fatal("trigger-sync should release the hold and run the bucket") + } +} + +func TestReplicationModeDowngradeCancelsDestructiveJob(t *testing.T) { + h := newReplicationHarness(t, mirrorFixture(10)) + h.trigger("first-mirror") + h.reconcile() + running := h.activeJobs()[0].Name + + h.update(func(r *garagev1beta1.GarageReplication) { + r.Spec.Mode = garagev1beta1.ReplicationModeCopy + r.Spec.Mirror = nil + }) + h.reconcile() + for _, j := range h.jobs() { + if j.Name == running { + t.Fatal("the Mirror Job should have been deleted after switching to Copy") + } + } + b := bucketStatus(t, h.get(), "medias") + if b.LastResult != garagev1beta1.ReplicationResultCancelled || b.LastMode != garagev1beta1.ReplicationModeMirror { + t.Fatalf("want Cancelled Mirror Job, got %+v", b) + } +} + +func TestReplicationNonDestructiveSpecChangeLetsJobFinish(t *testing.T) { + h := newReplicationHarness(t, replicationFixture()) + h.reconcile() + running := h.activeJobs()[0].Name + h.update(func(r *garagev1beta1.GarageReplication) { + r.Spec.Transfer = &garagev1beta1.ReplicationTransferConfig{Concurrency: 16} + }) + h.reconcile() + if active := h.activeJobs(); len(active) != 1 || active[0].Name != running { + t.Fatal("a non-destructive change must not cancel the running Job") + } +} + +func TestReplicationRemovedOrRepointedBucketCancelsItsJob(t *testing.T) { + h := newReplicationHarness(t, replicationFixture()) + h.reconcile() + h.update(func(r *garagev1beta1.GarageReplication) { r.Spec.Buckets = r.Spec.Buckets[1:] }) + h.reconcile() + for _, j := range h.activeJobs() { + if j.Annotations[annotationReplicationDestination] == "medias" { + t.Fatal("Job of a removed bucket should be deleted") + } + } + if len(h.get().Status.Buckets) != 1 { + t.Fatal("status should only list spec buckets") + } + + h.reconcile() + active := h.activeJobs() + if len(active) != 1 || active[0].Annotations[annotationReplicationSource] != "pg-backups" { + t.Fatalf("want the pg-backups Job, got %+v", active) + } + h.update(func(r *garagev1beta1.GarageReplication) { r.Spec.Buckets[0].Source = "pg-backups-v2" }) + h.reconcile() + for _, j := range h.activeJobs() { + if j.Name == active[0].Name { + t.Fatal("a Job copying the old source into the destination should be deleted") + } + } + if b := bucketStatus(t, h.get(), "pg-backups-dr"); b.Source != "pg-backups-v2" || b.LastResult != garagev1beta1.ReplicationResultCancelled { + t.Fatalf("status should follow the new source and record the cancel, got %+v", b) + } +} + +func TestReplicationManuallyDeletedJobIsNotRecreated(t *testing.T) { + h := newReplicationHarness(t, replicationFixture()) + h.reconcile() + job := h.activeJobs()[0] + if err := h.c.Delete(context.Background(), &job); err != nil { + t.Fatal(err) + } + h.reconcile() + for _, j := range h.jobs() { + if j.Name == job.Name { + t.Fatal("a Job someone deleted must not be recreated") + } + } + if b := bucketStatus(t, h.get(), job.Annotations[annotationReplicationDestination]); b.LastResult != garagev1beta1.ReplicationResultCancelled { + t.Fatalf("want Cancelled, got %+v", b) + } +} + +func TestReplicationMissingSecretBlocksJobs(t *testing.T) { + repl := replicationFixture() + repl.Spec.Destination.CredentialsSecretRef.Name = "missing" + h := newReplicationHarness(t, repl) + res := h.reconcile() + if len(h.jobs()) != 0 { + t.Fatal("no Job without credentials") + } + got := h.get() + ready := readyCondition(t, got) + if ready.Status != metav1.ConditionFalse || ready.Reason != "SecretNotFound" || got.Status.Phase != garagev1beta1.ReplicationPhasePending { + t.Fatalf("want Ready=False/SecretNotFound and Pending, got %+v phase=%s", ready, got.Status.Phase) + } + if res.RequeueAfter != replicationNotReadyRequeue { + t.Fatalf("requeue %v", res.RequeueAfter) + } + if strings.Contains(ready.Message, replTestSecretKey) { + t.Fatal("secret material leaked into status") + } +} + +func TestReplicationCredentialsSecretMustOptIn(t *testing.T) { + // A GarageKey-generated Secret without the label is refused too: being + // generated for a key does not make it replication credentials. + keySecret := replicationTestSecret("other-app-key", false) + keySecret.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: garagev1beta1.GroupVersion.String(), Kind: "GarageKey", Name: "other-app", UID: "key-uid", Controller: ptr.To(true), + }} + repl := replicationFixture() + repl.Spec.Source.CredentialsSecretRef.Name = keySecret.Name + h := newReplicationHarness(t, repl, keySecret) + h.reconcile() + if len(h.jobs()) != 0 { + t.Fatal("no Job may use a Secret that did not opt in") + } + ready := readyCondition(t, h.get()) + if ready.Reason != ReplicationReasonCredentialsNotAllowed || !strings.Contains(ready.Message, garagev1beta1.LabelReplicationCredentials) { + t.Fatalf("want CredentialsNotAllowed naming the label, got %+v", ready) + } + + keySecret.Labels = map[string]string{garagev1beta1.LabelReplicationCredentials: "true"} + if err := h.c.Update(context.Background(), keySecret); err != nil { + t.Fatal(err) + } + h.reconcile() + if len(h.jobs()) != 1 { + t.Fatal("an opted-in Secret should be used") + } +} + +func TestReplicationCABundleComesFromConfigMap(t *testing.T) { + repl := replicationFixture() + repl.Spec.Destination.Endpoint.CABundleRef = &garagev1beta1.ReplicationCABundleRef{Name: "dr-ca"} + h := newReplicationHarness(t, repl) + h.reconcile() + if ready := readyCondition(t, h.get()); ready.Reason != "CABundleNotFound" { + t.Fatalf("want CABundleNotFound, got %+v", ready) + } + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "dr-ca", Namespace: replTestNamespace}, + Data: map[string]string{garagev1beta1.ReplicationDefaultCABundleKey: "-----BEGIN CERTIFICATE-----"}, + } + if err := h.c.Create(context.Background(), cm); err != nil { + t.Fatal(err) + } + h.reconcile() + jobs := h.jobs() + if len(jobs) != 1 { + t.Fatalf("want a Job once the CA exists, got %d", len(jobs)) + } + vols := jobs[0].Spec.Template.Spec.Volumes + found := false + for _, v := range vols { + if v.Secret != nil { + t.Fatal("no Secret volume may be mounted for a CA") + } + if v.ConfigMap != nil && v.ConfigMap.Name == "dr-ca" && v.ConfigMap.Items[0].Key == garagev1beta1.ReplicationDefaultCABundleKey { + found = true + } + } + if !found { + t.Fatalf("CA ConfigMap not mounted: %+v", vols) + } +} + +func TestReplicationDestinationConflictComparesResolvedURLs(t *testing.T) { + // The older object writes "medias" on the destination cluster through a + // clusterRef; the newer one names the same cluster by URL. + dr := &garagev1beta2.GarageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "garage", Namespace: replTestNamespace}, + Spec: garagev1beta2.GarageClusterSpec{S3API: &garagev1beta2.S3APIConfig{BindPort: 3900}}, + } + older := replicationFixture() + older.Name, older.UID = "older", "older-uid" + older.CreationTimestamp = metav1.NewTime(time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)) + older.Spec.Destination = garagev1beta1.ReplicationEndpoint{ + ClusterRef: &garagev1beta1.ClusterReference{Name: "garage"}, + CredentialsSecretRef: garagev1beta1.ReplicationCredentialsRef{Name: replTestDstSecret}, + } + older.Spec.Buckets = older.Spec.Buckets[:1] + repl := replicationFixture() + repl.Spec.Destination.Endpoint.URL = "HTTP://garage.dr.svc.cluster.local:3900/" + h := newReplicationHarness(t, repl, dr, older) + h.reconcile() + if len(h.jobs()) != 0 { + t.Fatal("the newer GarageReplication must not write a taken destination bucket") + } + ready := readyCondition(t, h.get()) + if ready.Reason != ReplicationReasonDestinationConflict || !strings.Contains(ready.Message, `"older"`) { + t.Fatalf("want DestinationConflict naming the same-namespace owner, got %+v", ready) + } + + // Another namespace's owner is not named. + h2 := newReplicationHarness(t, replicationFixture(), func() client.Object { + o := replicationFixture() + o.Name, o.Namespace, o.UID = "secret-project", "elsewhere", "other-uid" + o.CreationTimestamp = older.CreationTimestamp + return o + }()) + h2.reconcile() + ready = readyCondition(t, h2.get()) + if ready.Reason != ReplicationReasonDestinationConflict || strings.Contains(ready.Message, "secret-project") || strings.Contains(ready.Message, "elsewhere") { + t.Fatalf("want a DestinationConflict that does not reveal the other namespace, got %+v", ready) + } +} + +func TestReplicationClusterRefResolvesEndpointAndRequiresGrant(t *testing.T) { + cluster := &garagev1beta2.GarageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "storage"}, + Spec: garagev1beta2.GarageClusterSpec{S3API: &garagev1beta2.S3APIConfig{BindPort: 3900, Region: "eu"}}, + } + repl := replicationFixture() + repl.Spec.Source.Endpoint = nil + repl.Spec.Source.ClusterRef = &garagev1beta1.ClusterReference{Name: "prod", Namespace: "storage"} + h := newReplicationHarness(t, repl, cluster) + h.reconcile() + if ready := readyCondition(t, h.get()); ready.Reason != "ReferenceNotGranted" || len(h.jobs()) != 0 { + t.Fatalf("cross-namespace clusterRef needs a grant, got %+v", ready) + } + + grant := &garagev1beta1.GarageReferenceGrant{ + ObjectMeta: metav1.ObjectMeta{Name: "allow-dr", Namespace: "storage"}, + Spec: garagev1beta1.GarageReferenceGrantSpec{ + From: []garagev1beta1.ReferenceGrantFrom{{Kind: kindGarageReplication, Namespace: replTestNamespace}}, + To: []garagev1beta1.ReferenceGrantTo{{Kind: kindGarageCluster, Name: "prod"}}, + }, + } + if err := h.c.Create(context.Background(), grant); err != nil { + t.Fatal(err) + } + h.reconcile() + jobs := h.jobs() + if len(jobs) != 1 { + t.Fatalf("want a Job once granted, got %d", len(jobs)) + } + env := map[string]string{} + for _, e := range jobs[0].Spec.Template.Spec.Containers[0].Env { + env[e.Name] = e.Value + } + if env["RCLONE_CONFIG_SRC_ENDPOINT"] != "http://prod.storage.svc.cluster.local:3900" || env["RCLONE_CONFIG_SRC_REGION"] != "eu" { + t.Fatalf("source endpoint not resolved from the GarageCluster: %q %q", env["RCLONE_CONFIG_SRC_ENDPOINT"], env["RCLONE_CONFIG_SRC_REGION"]) + } +} + +func TestReplicationPrunesRecordedJobHistory(t *testing.T) { + repl := replicationFixture() + repl.Spec.Buckets = repl.Spec.Buckets[:1] + h := newReplicationHarness(t, repl) + for i := 0; i < 4; i++ { + completeRun(t, h, replTestSuccessMessage, true) + h.now = h.now.Add(15 * time.Minute) + } + h.reconcile() + finished := 0 + for _, j := range h.jobs() { + if done, _, _ := replicationJobFinished(&j); done { + finished++ + } + } + if finished != 1 { + t.Fatalf("successfulJobsHistoryLimit defaults to 1, %d finished Jobs kept", finished) + } +} + +func TestReplicationJobCarriesNoSecretMaterial(t *testing.T) { + repl := mirrorFixture(100) + repl.Spec.Mirror.MaxDeleteSize = "10G" + repl.Spec.Transfer = &garagev1beta1.ReplicationTransferConfig{BandwidthLimit: "1024"} + repl.Spec.Destination.Endpoint.CABundleRef = &garagev1beta1.ReplicationCABundleRef{Name: "dr-ca", Key: "bundle.pem"} + job := buildReplicationJob(replicationJobInput{ + Replication: repl, Bucket: repl.Spec.Buckets[0], RunID: replicationRunID(time.Unix(1762689600, 0)), + SpecHash: "abc", Image: DefaultReplicationImage, + Source: resolvedReplicationEndpoint{URL: "https://s3.prod.example.com", Region: "garage", Credentials: repl.Spec.Source.CredentialsSecretRef}, + Destination: resolvedReplicationEndpoint{URL: replTestDstURL, Region: "dr", Credentials: repl.Spec.Destination.CredentialsSecretRef, CABundle: repl.Spec.Destination.Endpoint.CABundleRef}, + }) + raw, _ := json.Marshal(job) + if strings.Contains(string(raw), replTestSecretKey) || strings.Contains(string(raw), replTestSecretID) { + t.Fatal("Job must reference credentials, never embed them") + } + c := job.Spec.Template.Spec.Containers[0] + refs := 0 + env := map[string]string{} + for _, e := range c.Env { + env[e.Name] = e.Value + if e.ValueFrom != nil && e.ValueFrom.SecretKeyRef != nil { + refs++ + } + } + if refs != 4 { + t.Fatalf("want 4 secretKeyRef env vars (2 per side), got %d", refs) + } + want := map[string]string{ + "GR_MODE": "Mirror", "GR_MAX_DELETES": "100", "GR_MAX_DELETE_SIZE": "10G", + // rclone reads a bare number as KiB; the API means bytes. + "GR_BWLIMIT": "1024B", + "GR_CA_FILES": "/etc/garage-replication/ca/destination/ca.crt", + "GR_TRANSFERS": "4", "GR_CHECKERS": "8", "GR_VERIFY": "None", + } + for k, v := range want { + if env[k] != v { + t.Fatalf("%s = %q, want %q (env %+v)", k, env[k], v, env) + } + } + for _, v := range job.Spec.Template.Spec.Volumes { + if v.Name == "ca-destination" && (v.ConfigMap == nil || v.ConfigMap.Items[0].Key != "bundle.pem") { + t.Fatalf("CA volume %+v", v) + } + } + sc := c.SecurityContext + if sc == nil || !*sc.ReadOnlyRootFilesystem || *sc.AllowPrivilegeEscalation || !*job.Spec.Template.Spec.SecurityContext.RunAsNonRoot || *job.Spec.Template.Spec.AutomountServiceAccountToken { + t.Fatal("replication Pods must run locked down") + } + if p := job.Spec.PodFailurePolicy; p == nil || len(p.Rules) == 0 || p.Rules[0].Action != batchv1.PodFailurePolicyActionFailJob { + t.Fatal("safety-guard exit codes must fail the Job without retry") + } + if len(job.Name) > 63 { + t.Fatalf("Job name %q too long", job.Name) + } +} + +func TestReplicationJobNameIsBoundedAndDeterministic(t *testing.T) { + long := strings.Repeat("a", 253) + b := garagev1beta1.ReplicationBucket{Source: "medias", Destination: "medias"} + n1 := replicationJobName(long, "t2b5wg", b) + if n1 != replicationJobName(long, "t2b5wg", b) || len(n1) > 63 { + t.Fatalf("name %q", n1) + } + if n1 == replicationJobName(long, "t2b5wg", garagev1beta1.ReplicationBucket{Source: "medias", Destination: "other"}) { + t.Fatal("different destinations must get different Jobs") + } + if !replicationRunIDLess("", "t2b5wg") || !replicationRunIDLess("zzzzzz", "1000000") { + t.Fatal("run IDs must order by time, not by string") + } +} + +func TestParseReplicationTermination(t *testing.T) { + got, ok := parseReplicationTermination("garage-replication/v1 exit=1 reason=VerificationFailed verify=Failed differences=3 stats=1 transfers=1 bytes=2 checks=3 deletes=4 errors=6 elapsedTime=7.4") + if !ok || got.ExitCode != 1 || got.Reason != string(garagev1beta1.ReplicationFailureVerificationFailed) || got.Verification != "Failed" { + t.Fatalf("got %+v", got) + } + s := got.Stats + if s.Transfers != 1 || s.Bytes != 2 || s.Checks != 3 || s.Deletes != 4 || s.Errors != 6 || s.ElapsedSeconds != 7 || s.Differences == nil || *s.Differences != 3 { + t.Fatalf("stats %+v", s) + } + if got, ok := parseReplicationTermination("garage-replication/v1 exit=143 reason=Terminated verify= differences= stats=0"); !ok || got.Stats != nil { + t.Fatalf("no stats must stay nil, got %+v", got) + } + for _, bad := range []string{"", "panic: boom", "garage-replication/v1 exit=abc"} { + if _, ok := parseReplicationTermination(bad); ok { + t.Fatalf("%q should not parse", bad) + } + } + failed := &batchv1.Job{Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}}}} + pod := func(msg string) []corev1.Pod { + return []corev1.Pod{{Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: replicationContainerName, State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Message: msg}}, + }}}}} + } + for msg, want := range map[string]garagev1beta1.ReplicationFailureReason{ + "garage-replication/v1 exit=1 reason=