Add OpenTelemetry SDK metrics foundation with dual export

Dockerfile

@@ -18,9 +18,14 @@ COPY api/ api/
 COPY pkg/ pkg/
 COPY internal/ internal/
 
+# VERSION is stamped into the binary as the service.version resource
+# attribute on every metric and span. Without it the image reports "dev",
+# so pass --build-arg VERSION=... when building a release.
+ARG VERSION=dev
+
 # Build
 RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
-    go build -a -o manager cmd/main.go
+    go build -a -ldflags "-X main.version=${VERSION}" -o manager cmd/main.go
 
 # Use Red Hat UBI minimal as runtime image
 FROM registry.access.redhat.com/ubi9/ubi-minimal:latest

Dockerfile.mcp

@@ -18,9 +18,14 @@ COPY pkg/ pkg/
 COPY internal/ internal/
 COPY api/ api/
 
+# VERSION is stamped into the binary as the service.version resource
+# attribute on every metric and span. Without it the image reports "dev",
+# so pass --build-arg VERSION=... when building a release.
+ARG VERSION=dev
+
 # Build
 RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
-    go build -a -o unstructured-data-mcp-server cmd/unstructured-data-mcp-server/main.go
+    go build -a -ldflags "-X main.version=${VERSION}" -o unstructured-data-mcp-server cmd/unstructured-data-mcp-server/main.go
 
 # Use Red Hat UBI minimal as runtime image
 FROM registry.access.redhat.com/ubi9/ubi-minimal:latest

Makefile

@@ -162,9 +162,21 @@ lint-config: golangci-lint ## Verify golangci-lint linter configuration
 
 ##@ Build
 
+# Stamps service.version on every metric and span emitted by a binary built
+# through these targets and the image targets below. A build that bypasses
+# them reports "dev", which is at least honest about being unstamped. VERSION
+# is a Makefile constant, so remember to bump it for a release.
+# := not ?=, and a dedicated name: a developer with LDFLAGS exported in
+# their shell would otherwise silently produce an unstamped build.
+VERSION_LDFLAGS := -X main.version=$(VERSION)
+
 .PHONY: build
 build: manifests generate fmt vet ## Build manager binary.
-	go build -o bin/manager cmd/main.go
+	go build -ldflags "$(VERSION_LDFLAGS)" -o bin/manager cmd/main.go
+
+.PHONY: build-mcp-server
+build-mcp-server: fmt vet ## Build the MCP server binary.
+	go build -ldflags "$(VERSION_LDFLAGS)" -o bin/unstructured-data-mcp-server ./cmd/unstructured-data-mcp-server
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
@@ -175,15 +187,15 @@ run: manifests generate fmt vet ## Run a controller from your host.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager.
-	$(CONTAINER_TOOL) build -t ${IMG} .
+	$(CONTAINER_TOOL) build --build-arg VERSION=$(VERSION) -t ${IMG} .
 
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
 	$(CONTAINER_TOOL) push ${IMG}
 
 .PHONY: docker-build-mcp
 docker-build-mcp: ## Build docker image for the MCP server.
-	$(CONTAINER_TOOL) build -f Dockerfile.mcp -t ${IMG_MCP} .
+	$(CONTAINER_TOOL) build --build-arg VERSION=$(VERSION) -f Dockerfile.mcp -t ${IMG_MCP} .
 
 .PHONY: docker-push-mcp
 docker-push-mcp: ## Push docker image for the MCP server.
@@ -202,7 +214,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform
 	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
 	- $(CONTAINER_TOOL) buildx create --name unstructured-data-controller-builder
 	$(CONTAINER_TOOL) buildx use unstructured-data-controller-builder
-	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx build --push --build-arg VERSION=$(VERSION) --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
 	- $(CONTAINER_TOOL) buildx rm unstructured-data-controller-builder
 	rm Dockerfile.cross
 
@@ -211,7 +223,7 @@ docker-buildx-mcp: ## Build and push docker image for the MCP server for cross-p
 	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile.mcp > Dockerfile.mcp.cross
 	- $(CONTAINER_TOOL) buildx create --name unstructured-data-controller-builder
 	$(CONTAINER_TOOL) buildx use unstructured-data-controller-builder
-	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG_MCP} -f Dockerfile.mcp.cross .
+	- $(CONTAINER_TOOL) buildx build --push --build-arg VERSION=$(VERSION) --platform=$(PLATFORMS) --tag ${IMG_MCP} -f Dockerfile.mcp.cross .
 	- $(CONTAINER_TOOL) buildx rm unstructured-data-controller-builder
 	rm Dockerfile.mcp.cross
 

cmd/main.go

@@ -17,10 +17,12 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
 	"os"
 	"path/filepath"
+	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -40,6 +42,8 @@ import (
 
 	operatorv1alpha1 "github.com/redhat-data-and-ai/unstructured-data-controller/api/v1alpha1"
 	"github.com/redhat-data-and-ai/unstructured-data-controller/internal/controller"
+	udctelemetry "github.com/redhat-data-and-ai/unstructured-data-controller/pkg/telemetry"
+	crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -48,6 +52,22 @@ var (
 	setupLog = ctrl.Log.WithName("setup")
 )
 
+// version is the build version reported as the service.version resource
+// attribute on every metric and span. Overridden at build time with
+// -ldflags "-X main.version=..."; see the Makefile. The default is honest
+// rather than a stale literal, so an unstamped build is identifiable in the
+// telemetry backend instead of masquerading as a release.
+var version = "dev"
+
+// Shutdown is budgeted against the pod's terminationGracePeriodSeconds of 10s
+// (config/manager/manager.yaml): reconcilers drain first, then telemetry
+// flushes, and the two together have to finish before the kubelet SIGKILLs.
+// The margin absorbs process teardown.
+var (
+	gracefulShutdownTimeout  = 5 * time.Second
+	telemetryShutdownTimeout = 3 * time.Second
+)
+
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 
@@ -92,6 +112,30 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Initialize OTel metrics. udc_* metrics are registered on
+	// controller-runtime's Prometheus registry, so they appear beside its
+	// built-in metrics wherever that endpoint is served. --metrics-bind-address
+	// defaults to "0"; config/default sets it to :8443, but config/deploy -
+	// the overlay `make deploy` builds - does not, so a default install
+	// serves nothing and OTLP push is its only delivery path.
+	metricsProvider, err := udctelemetry.Init(context.Background(), udctelemetry.Config{
+		ServiceName:          "unstructured-data-controller",
+		ServiceVersion:       version,
+		PrometheusRegisterer: crmetrics.Registry,
+		// Reconcile loops run continuously across every watched resource, so
+		// exporting every trace would flood the collector. Operators can raise
+		// this at runtime with OTEL_TRACES_SAMPLER_ARG when debugging.
+		TraceSampleRatio: 0.1,
+		// Send SDK diagnostics through the same logger as everything else, so
+		// a failing exporter is visible in the pod logs instead of only on
+		// stderr in a format the log pipeline cannot parse.
+		Logger: ctrl.Log.WithName("otel"),
+	})
+	if err != nil {
+		setupLog.Error(err, "unable to initialize metrics provider")
+		os.Exit(1)
+	}
+
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
@@ -188,6 +232,13 @@ func main() {
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "6dc1af3d.dataverse.redhat.com",
+		// Bounded so that mgr.Start returns, and the telemetry flush that
+		// follows it runs, inside the pod's terminationGracePeriodSeconds
+		// (10s in config/manager/manager.yaml). controller-runtime defaults
+		// this to 30s, which would let the kubelet SIGKILL the process while
+		// reconcilers were still draining - taking the final metric interval
+		// and every buffered span with it.
+		GracefulShutdownTimeout: &gracefulShutdownTimeout,
 		Cache: cache.Options{
 			DefaultNamespaces: map[string]cache.Config{
 				watchNamespace: {},
@@ -211,53 +262,60 @@ func main() {
 	}
 
 	if err := (&controller.ControllerConfigReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:  mgr.GetClient(),
+		Scheme:  mgr.GetScheme(),
+		Metrics: metricsProvider,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ControllerConfig")
 		os.Exit(1)
 	}
 
 	if err := (&controller.UnstructuredDataPipelineReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:  mgr.GetClient(),
+		Scheme:  mgr.GetScheme(),
+		Metrics: metricsProvider,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "UnstructuredDataPipeline")
 		os.Exit(1)
 	}
 
 	if err := (&controller.DocumentProcessorReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:  mgr.GetClient(),
+		Scheme:  mgr.GetScheme(),
+		Metrics: metricsProvider,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "DocumentProcessor")
 		os.Exit(1)
 	}
 
 	if err := (&controller.ChunksGeneratorReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:  mgr.GetClient(),
+		Scheme:  mgr.GetScheme(),
+		Metrics: metricsProvider,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ChunksGenerator")
 		os.Exit(1)
 	}
 	if err := (&controller.VectorEmbeddingsGeneratorReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:  mgr.GetClient(),
+		Scheme:  mgr.GetScheme(),
+		Metrics: metricsProvider,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "VectorEmbeddingsGenerator")
 		os.Exit(1)
 	}
 	if err := (&controller.SourceCrawlerReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:  mgr.GetClient(),
+		Scheme:  mgr.GetScheme(),
+		Metrics: metricsProvider,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "SourceCrawler")
 		os.Exit(1)
 	}
 	if err := (&controller.DestinationSyncerReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:  mgr.GetClient(),
+		Scheme:  mgr.GetScheme(),
+		Metrics: metricsProvider,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "DestinationSyncer")
 		os.Exit(1)
@@ -290,7 +348,26 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	err = mgr.Start(ctrl.SetupSignalHandler())
+
+	// Shut telemetry down after Start returns, not as a manager Runnable.
+	// A Runnable with NeedLeaderElection()==false lands in controller-runtime's
+	// `Others` group, which internal.go stops BEFORE the LeaderElection group
+	// that holds the reconcilers - so the providers would be torn down while
+	// all seven controllers were still draining, and every reconcile recorded
+	// during shutdown would be dropped. Start returns only once every runnable
+	// group has stopped, which is the point at which there is nothing left to
+	// record.
+	// Fresh, bounded context: the manager's own context is already cancelled
+	// by this point, and an unbounded one would let a hung OTLP exporter
+	// block process exit indefinitely.
+	shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), telemetryShutdownTimeout)
+	if shutdownErr := metricsProvider.Shutdown(shutdownCtx); shutdownErr != nil {
+		setupLog.Error(shutdownErr, "failed to flush telemetry on shutdown")
+	}
+	cancelShutdown()
+
+	if err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}