From 96d978f09db1c2f936cab149c902e3ff66533246 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:00:26 +0000 Subject: [PATCH 1/3] feat(hub): add optional OpenTelemetry server tracing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- hub/Gemfile | 5 + hub/Gemfile.lock | 91 +++++++++++ hub/config/application.rb | 3 + hub/docs/README.md | 2 +- hub/docs/deployment.md | 51 ++++++ hub/lib/hub_tracing.rb | 65 ++++++++ hub/lib/hub_tracing/exporter.rb | 47 ++++++ hub/test/integration/hub_tracing_test.rb | 200 +++++++++++++++++++++++ hub/test/lib/hub_tracing_test.rb | 148 +++++++++++++++++ 9 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 hub/lib/hub_tracing.rb create mode 100644 hub/lib/hub_tracing/exporter.rb create mode 100644 hub/test/integration/hub_tracing_test.rb create mode 100644 hub/test/lib/hub_tracing_test.rb diff --git a/hub/Gemfile b/hub/Gemfile index 62b1bee..d57f5b3 100644 --- a/hub/Gemfile +++ b/hub/Gemfile @@ -7,6 +7,11 @@ gem "bootsnap", require: false gem "json", "~> 3.0" gem "jwt", "~> 3.3" gem "oaken", "1.0.0" +gem "opentelemetry-exporter-otlp", require: false +gem "opentelemetry-instrumentation-action_pack", require: false +gem "opentelemetry-instrumentation-active_job", require: false +gem "opentelemetry-instrumentation-net_http", require: false +gem "opentelemetry-sdk", require: false gem "propshaft" gem "puma", ">= 5.0" gem "rails", "~> 8.1.4" diff --git a/hub/Gemfile.lock b/hub/Gemfile.lock index 36d13ba..d17da70 100644 --- a/hub/Gemfile.lock +++ b/hub/Gemfile.lock @@ -123,6 +123,26 @@ GEM raabro (~> 1.4) globalid (1.4.0) activesupport (>= 6.1) + google-protobuf (4.36.2) + bigdecimal + rake (~> 13.3) + google-protobuf (4.36.2-aarch64-linux-gnu) + bigdecimal + rake (~> 13.3) + google-protobuf (4.36.2-aarch64-linux-musl) + bigdecimal + rake (~> 13.3) + google-protobuf (4.36.2-arm64-darwin) + bigdecimal + rake (~> 13.3) + google-protobuf (4.36.2-x86_64-linux-gnu) + bigdecimal + rake (~> 13.3) + google-protobuf (4.36.2-x86_64-linux-musl) + bigdecimal + rake (~> 13.3) + googleapis-common-protos-types (1.23.0) + google-protobuf (~> 4.26) hashdiff (1.2.1) i18n (1.15.2) concurrent-ruby (~> 1.0) @@ -182,6 +202,51 @@ GEM oj (3.17.6) bigdecimal (>= 3.0) ostruct (>= 0.2) + opentelemetry-api (1.11.1) + logger + opentelemetry-common (0.25.2) + opentelemetry-api (~> 1.0) + opentelemetry-exporter-otlp (0.37.0) + google-protobuf (>= 3.18) + googleapis-common-protos-types (~> 1.3) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-exporter-otlp-common (~> 0.2) + opentelemetry-sdk (~> 1.10) + opentelemetry-semantic_conventions + opentelemetry-exporter-otlp-common (0.3.0) + google-protobuf (>= 3.18) + googleapis-common-protos-types (~> 1.3) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-instrumentation-action_pack (0.18.1) + opentelemetry-instrumentation-rack (~> 0.29) + opentelemetry-instrumentation-active_job (0.13.0) + opentelemetry-instrumentation-base (~> 0.25) + opentelemetry-instrumentation-base (0.27.0) + opentelemetry-api (~> 1.7) + opentelemetry-common (~> 0.21) + opentelemetry-logs-api (~> 0.6) + opentelemetry-metrics-api (~> 0.9) + opentelemetry-registry (~> 0.1) + opentelemetry-instrumentation-net_http (0.29.2) + opentelemetry-instrumentation-base (~> 0.25) + opentelemetry-instrumentation-rack (0.31.2) + opentelemetry-instrumentation-base (~> 0.25) + opentelemetry-logs-api (0.6.0) + opentelemetry-api (~> 1.0) + opentelemetry-metrics-api (0.9.0) + opentelemetry-api (~> 1.0) + opentelemetry-registry (0.6.3) + opentelemetry-api (~> 1.1) + opentelemetry-sdk (1.13.1) + logger + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-registry (~> 0.2) + opentelemetry-semantic_conventions + opentelemetry-semantic_conventions (1.44.0) + opentelemetry-api (~> 1.0) ostruct (0.6.3) parallel (2.2.0) parser (3.3.12.0) @@ -377,6 +442,11 @@ DEPENDENCIES json (~> 3.0) jwt (~> 3.3) oaken (= 1.0.0) + opentelemetry-exporter-otlp + opentelemetry-instrumentation-action_pack + opentelemetry-instrumentation-active_job + opentelemetry-instrumentation-net_http + opentelemetry-sdk propshaft puma (>= 5.0) rails (~> 8.1.4) @@ -435,6 +505,13 @@ CHECKSUMS form_props (0.2.2) sha256=a46f8f7fd216b26ed6d6416d22334344a531188aee0e854aebc81761ee280cad fugit (1.13.0) sha256=a4f093fce740da52f216740a5041e2a594ea763cdb89e8b2754ca4399634ab18 globalid (1.4.0) sha256=037f12fbf1d9d7a014d501c2d5c77356fd4ddd96d7a7991d6700bba96706f427 + google-protobuf (4.36.2) sha256=2f8daff50ae14aad36ab0ccdcf284065fa71d726c444efe114e6db82ffe62494 + google-protobuf (4.36.2-aarch64-linux-gnu) sha256=c3279ec6eaa59f6ed6276eda2a33cffb4306471183ab79a332e698b8564f5169 + google-protobuf (4.36.2-aarch64-linux-musl) sha256=0772c9dde1e6dce9a6b44d1d35c28c29adee99d9bd6aff5e21840daeb22161ce + google-protobuf (4.36.2-arm64-darwin) sha256=f70d810e6bbab9d8ad9c1d5fd685a21de4c10ad3ad32efb9455e17e5b1286917 + google-protobuf (4.36.2-x86_64-linux-gnu) sha256=d25a820873e423f1d209ae8c672a4a28b019e2429debd0901d26670a0b4e14db + google-protobuf (4.36.2-x86_64-linux-musl) sha256=0fa17686c896da96a60a705e56ddc7d4ff56413f52787c655bd4c9fb33ffe14f + googleapis-common-protos-types (1.23.0) sha256=992e740a523794d9fc5f29a504465d8fc737aaa16c930fe7228e3346860faf0a hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5 io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08 @@ -465,6 +542,20 @@ CHECKSUMS nokogiri (1.19.4-x86_64-linux-musl) sha256=17dfb7c1fa194ae02fbf7c51a7afc8d278045ab3fdacfd86f91d02d7b274470b oaken (1.0.0) sha256=e89249bc4f6cd3ab9b5e3abdc06c377fa772e6bcb3005cc09ff044bb8d3f1dc2 oj (3.17.6) sha256=f511257e01a12030c3ceb8023cd2e96fca924ea9c741f57ea366816f1fd8e68c + opentelemetry-api (1.11.1) sha256=d07f9987ea6240b3fd5ed5dd7450f93c58e403e0553e804069c23ae18ba61ff7 + opentelemetry-common (0.25.2) sha256=c035d44987789356fe8feeeb870708b44fe7564ffc032dec7200c39da92a911d + opentelemetry-exporter-otlp (0.37.0) sha256=edc8bda657dce5f4d527ac735f75d44043bb5924de6df0e3f01260108aa76136 + opentelemetry-exporter-otlp-common (0.3.0) sha256=855d10d788a7272dfe5b14efa229507619abbef29085ef4a8acfd2c0d3af6171 + opentelemetry-instrumentation-action_pack (0.18.1) sha256=0b827d703baa2951458e36f2bdfa4a2cdb17a0130cfb0e1b6fe5d67fadc85c2d + opentelemetry-instrumentation-active_job (0.13.0) sha256=42da156924bbabdaca6268be1e46963a04829bd0b2b82fc6e368597a3525cde2 + opentelemetry-instrumentation-base (0.27.0) sha256=e55edb82cd0b018e82e746be61fa36cb05b7b68b2fa37f0e4a96b923ad3d7735 + opentelemetry-instrumentation-net_http (0.29.2) sha256=4256105f058404eb0bd43e4a987e48f9e48fcade06b40a388092c380b2e946eb + opentelemetry-instrumentation-rack (0.31.2) sha256=4a183066110455711a0c77754e3f46652f646221d7704d968a48d155b47f8309 + opentelemetry-logs-api (0.6.0) sha256=b889ac61d90326bffadb83bf7278d246d1086e2349c9052aeb072ed375ce73d0 + opentelemetry-metrics-api (0.9.0) sha256=e11a12055113e11421313f71c758e5e10e4662e9e5c3b91786218a692e81206d + opentelemetry-registry (0.6.3) sha256=568fa8eaea94394f29f4fce3f0339ab10c3bf6d6c8beed6b3cfee6fd37fcc608 + opentelemetry-sdk (1.13.1) sha256=91c9cc315ac7ef30243549eeff8b27cd0d39bd02bdf9c9ae481ab96a1cec6754 + opentelemetry-semantic_conventions (1.44.0) sha256=b09b8793422f13b01c08f7a6f29b5f511f47f37bcdb57c0513465a0464168513 ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912 parallel (2.2.0) sha256=e1059c5fd7b649558a0aec38a769f06a42942bdb40503d005a59c352fe011cd8 parser (3.3.12.0) sha256=21a6d7f755d5a24dfbdc6e6b772e4e879a52e7631a88bc5a3a134606052c9828 diff --git a/hub/config/application.rb b/hub/config/application.rb index ac8fcab..b48369a 100644 --- a/hub/config/application.rb +++ b/hub/config/application.rb @@ -85,4 +85,7 @@ class Application < Rails::Application # only to replica-learning and never stores it. See docs/copilot.md. config.x.copilot_token = ENV["REPLICA_COPILOT_TOKEN"].presence end + + require_relative "../lib/hub_tracing" + HubTracing.start! end diff --git a/hub/docs/README.md b/hub/docs/README.md index 91add69..9637091 100644 --- a/hub/docs/README.md +++ b/hub/docs/README.md @@ -4,7 +4,7 @@ Start with [Deployment](deployment.md) to run Replica Hub or [Development](devel The [shared contributor guides](../../docs/README.md) apply across Replica. This index covers Hub-specific behavior and operations. -- **Operate Hub:** [Deployment](deployment.md), [GitHub App setup](github-app-setup.md), [security](security.md), and [backups](backups-and-restore.md). +- **Operate Hub:** [Deployment](deployment.md), [server-side tracing](deployment.md#optional-server-side-tracing), [GitHub App setup](github-app-setup.md), [security](security.md), and [backups](backups-and-restore.md). - **GitHub and Builds:** [App setup](github-app-setup.md), [access](authentication-and-repository-authorization.md), [discovery](build-discovery.md), [Actions](github-actions-builds.md), and [storage](artifact-storage.md). - **Review:** [Gallery](gallery.md), [previews](preview-serving.md), [collaboration](review-collaboration.md), and [share links](share-links.md). - **Analytics:** [Repository activity events](analytics.md). diff --git a/hub/docs/deployment.md b/hub/docs/deployment.md index f6851d9..e5cfd10 100644 --- a/hub/docs/deployment.md +++ b/hub/docs/deployment.md @@ -13,6 +13,57 @@ The recommended deployment needs: The default deployment runs one Replica Hub container and stores its SQLite databases and Build files in one persistent Docker volume. +## Optional server-side tracing + +Hub can send server-side traces to an OpenTelemetry Protocol (OTLP) endpoint. Traces describe Rails requests, Active Job enqueueing and execution, instrumented Active Record operations, and outbound Net::HTTP calls. The published image includes OpenTelemetry instrumentation and its HTTP/protobuf exporter, not vendor-specific APM SDKs. + +Tracing is off by default. To enable it, add these settings to the private deployment `.env` or your container platform's environment: + +```dotenv +OTEL_TRACES_EXPORTER=otlp +OTEL_SERVICE_NAME=replica-hub +OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=production +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel-collector:4318/v1/traces +OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf +``` + +This example assumes a Collector named `otel-collector` is reachable from Hub on a trusted private container network. Hub does not install or start that service. Use HTTPS for endpoints outside a trusted private network. + +Restart every Hub web and job container after changing these settings. No image rebuild is needed. Docker Compose already passes the private `.env` to Hub; other container platforms must pass the variables to every process themselves. + +| Setting | Purpose | +| --- | --- | +| `OTEL_TRACES_EXPORTER` | `otlp` enables export; `none` disables tracing and is Hub's default. Other exporters are not included. | +| `OTEL_SERVICE_NAME` | Service identity; defaults to `replica-hub`. | +| `OTEL_RESOURCE_ATTRIBUTES` | Comma-separated deployment metadata, such as `deployment.environment.name=production,service.version=1.0.0`. Do not put secrets or personal data here. | +| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Complete HTTP(S) trace endpoint, including its required path. Credentials, query strings, and fragments are not accepted in the URL. | +| `OTEL_EXPORTER_OTLP_TRACES_HEADERS` | Authentication headers required by the recipient, in the exporter's comma-separated `key=value` format. Keep this value private. | +| `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` | Only `http/protobuf` is supported. A gRPC-only endpoint needs a Collector or gateway that accepts HTTP/protobuf. | +| `OTEL_TRACES_SAMPLER` | Standard SDK sampling policy, for example `parentbased_traceidratio`. The SDK default is parent-based, with all new root traces sampled. | +| `OTEL_TRACES_SAMPLER_ARG` | Sampling-policy argument; for `parentbased_traceidratio`, a fraction from `0` to `1`, such as `0.1`. | +| `OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE` | Optional path inside the container to a trusted CA certificate for a private endpoint. Mount the certificate when needed; keep TLS verification enabled. | +| `OTEL_SDK_DISABLED` | Set to `true` to disable tracing regardless of the exporter setting. | + +The exporter also accepts the common `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, and `OTEL_EXPORTER_OTLP_PROTOCOL` settings. Trace-specific values take precedence. The common endpoint is a base URL to which the exporter appends `/v1/traces`; the trace-specific endpoint is used as supplied. Enabling export requires an explicit endpoint. Hub rejects unsupported exporters, protocols, and malformed endpoint URLs at startup. + +### Choose a recipient + +Send directly to an APM provider that accepts OTLP over HTTP/protobuf, using its documented trace endpoint and authentication headers. Alternatively, send to a company-managed OpenTelemetry Collector. The Collector can filter, sample, translate, and forward traces to one or more APMs without changing Hub's image. Choose a Collector distribution that includes the exporters your providers need. + +Sentry documents a direct OTLP trace endpoint. Datadog provides a Collector exporter. These paths do not guarantee the same features as each provider's native SDK; check the provider's supported signals, release status, and deployment version. + +The deployment operator controls the destination and credentials. Restrict recipients and network access through your deployment's outbound-network policy. Hub's repository analytics destination settings do not configure or restrict tracing. + +### Data and delivery limits + +Hub exports an allowlist of operational span attributes. It removes raw URL paths and queries, request and response headers, bodies, SQL text and parameter values, job arguments, span events, span links, and error status descriptions before export. HTTP span names use methods and route templates instead of concrete URLs. Trace context can cross service boundaries, but Hub does not propagate incoming OpenTelemetry baggage. Operator-supplied resource attributes are exported as configured. + +Active Job execution continues the enqueueing trace when the job's serialized context is available. The SDK's bounded background queue exports batches outside the request path and resets its worker after process forks. Hub allows up to five seconds to flush at normal process exit. Collector or provider outages are logged and may drop traces; delivery is not durable and does not gate successful Hub requests. Forced termination can lose pending spans. + +Health checks at `/up` are excluded. Asset compilation with `SECRET_KEY_BASE_DUMMY` does not initialize tracing. This integration does not enable browser monitoring, metric or log export, exception stack-trace export, or vendor-native error reporting. + +For exporter settings and their formats, see the [Ruby OTLP exporter](https://github.com/open-telemetry/opentelemetry-ruby/tree/main/exporter/otlp) and the [Ruby SDK documentation](https://opentelemetry.io/docs/languages/ruby/). Provider setup examples are available for [Sentry OTLP](https://docs.sentry.io/concepts/otlp/direct/traces/) and the [Datadog Collector exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/datadogexporter). + ## Choose an isolated domain Hub displays prototypes inside its review pages. It requires each preview to use a separate HTTPS subdomain, such as `b123.preview.hub.example.com` under `hub.example.com`. This lets preview cookies work while the prototype is embedded. Hub keeps its own sign-in cookie separate. diff --git a/hub/lib/hub_tracing.rb b/hub/lib/hub_tracing.rb new file mode 100644 index 0000000..af3422d --- /dev/null +++ b/hub/lib/hub_tracing.rb @@ -0,0 +1,65 @@ +require "uri" + +module HubTracing + def self.start! + return if ENV["SECRET_KEY_BASE_DUMMY"] || ENV["OTEL_SDK_DISABLED"]&.casecmp?("true") + + exporter = ENV.fetch("OTEL_TRACES_EXPORTER", "none") + unless %w[none otlp].include?(exporter) + raise ArgumentError, "OTEL_TRACES_EXPORTER must be none or otlp" + end + return if exporter == "none" + + protocol = ENV["OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"] || + ENV.fetch("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + unless protocol == "http/protobuf" + raise ArgumentError, "Hub tracing supports only the http/protobuf OTLP protocol" + end + endpoint = ENV["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] || ENV["OTEL_EXPORTER_OTLP_ENDPOINT"] + begin + uri = URI.parse(endpoint.to_s) + rescue URI::InvalidURIError + raise ArgumentError, "OTLP endpoint must be an absolute HTTP(S) URL" + end + unless %w[http https].include?(uri.scheme) && uri.host && + !uri.userinfo && !uri.query && !uri.fragment + raise ArgumentError, "OTLP endpoint must be an absolute HTTP(S) URL without credentials, query, or fragment" + end + + require "opentelemetry/sdk" + require "opentelemetry/exporter/otlp" + require "opentelemetry/instrumentation/action_pack" + require "opentelemetry/instrumentation/active_job" + require "opentelemetry/instrumentation/net/http" + require_relative "hub_tracing/exporter" + + processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + Exporter.new(OpenTelemetry::Exporter::OTLP::Exporter.new) + ) + OpenTelemetry::SDK.configure do |config| + config.service_name = ENV.fetch("OTEL_SERVICE_NAME", "replica-hub") + config.propagators = [ OpenTelemetry::Trace::Propagation::TraceContext.text_map_propagator ] + config.add_span_processor(processor) + config.use "OpenTelemetry::Instrumentation::Rack", untraced_endpoints: [ "/up" ] + config.use "OpenTelemetry::Instrumentation::ActionPack", span_naming: :class + config.use "OpenTelemetry::Instrumentation::ActiveJob", + propagation_style: :child, span_naming: :job_class + config.use "OpenTelemetry::Instrumentation::Net::HTTP" + end + + # ORM instrumentation does not cover Hub's direct SQLite calls. + tracer = OpenTelemetry.tracer_provider.tracer("replica-hub") + ActiveSupport::Notifications.subscribe("sql.active_record") do |_, started, finished, _, payload| + next if payload[:name] == "SCHEMA" || payload[:cached] + + span = tracer.start_span \ + "database query", + kind: :client, + start_timestamp: started, + attributes: { "db.system.name" => "sqlite" } + span.status = OpenTelemetry::Trace::Status.error if payload[:exception] + span.finish(end_timestamp: finished) + end + at_exit { OpenTelemetry.tracer_provider.shutdown(timeout: 5) } + end +end diff --git a/hub/lib/hub_tracing/exporter.rb b/hub/lib/hub_tracing/exporter.rb new file mode 100644 index 0000000..a891b79 --- /dev/null +++ b/hub/lib/hub_tracing/exporter.rb @@ -0,0 +1,47 @@ +module HubTracing + class Exporter + # Export operational metadata, not request content or SDK-generated exception messages. + ATTRIBUTES = %w[ + http.request.method http.response.status_code http.method http.status_code http.route + url.scheme http.scheme server.address server.port net.peer.name net.peer.port + db.system db.system.name db.operation db.operation.name + messaging.system messaging.operation messaging.operation.name messaging.operation.type + messaging.destination.name messaging.destination_kind code.namespace code.function + ].freeze + + def initialize(exporter) + @exporter = exporter + end + + def export(spans, timeout: nil) + @exporter.export(spans.map { |span| sanitized(span) }, timeout:) + end + + def shutdown(timeout: nil) + @exporter.shutdown(timeout:) + end + + def force_flush(timeout: nil) + @exporter.force_flush(timeout:) + end + + private + + def sanitized(span) + result = span.dup + result.attributes = (span.attributes || {}).slice(*ATTRIBUTES) + result.events = [] + result.links = [] + result.status = case span.status.code + when OpenTelemetry::Trace::Status::ERROR then OpenTelemetry::Trace::Status.error + when OpenTelemetry::Trace::Status::OK then OpenTelemetry::Trace::Status.ok + else OpenTelemetry::Trace::Status.unset + end + method = result.attributes["http.request.method"] || result.attributes["http.method"] + if span.kind == :server || method + result.name = [ method || "HTTP", result.attributes["http.route"] ].compact.join(" ") + end + result + end + end +end diff --git a/hub/test/integration/hub_tracing_test.rb b/hub/test/integration/hub_tracing_test.rb new file mode 100644 index 0000000..710f768 --- /dev/null +++ b/hub/test/integration/hub_tracing_test.rb @@ -0,0 +1,200 @@ +require "test_helper" +require "open3" +require "base64" +require "opentelemetry/exporter/otlp" + +class HubTracingIntegrationTest < ActiveSupport::TestCase + self.use_transactional_tests = false + + test "normal boot traces requests serialized child jobs database and outbound HTTP without private content" do + result = run_traced_request + + assert_equal 200, result.fetch("status") + assert_equal "completed", result.fetch("body") + assert result.fetch("job_performed") + assert_equal 200, result.fetch("health_status") + refute result.fetch("baggage_propagated"), "incoming baggage reached the outbound service" + payloads = result.fetch("payloads").map { |payload| Base64.strict_decode64(payload) } + assert_not_empty payloads, result.fetch("diagnostics") + payloads.each { |payload| refute_includes payload, "private-value-canary" } + spans = payloads.flat_map do |payload| + request = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(payload) + request.resource_spans.flat_map(&:scope_spans).flat_map(&:spans) + end + server = spans.find { |span| span.kind == :SPAN_KIND_SERVER } + assert server, "Rails boot did not produce a server span" + assert_equal "POST /tracing-probe", server.name + assert_equal "1234567890abcdef1234567890abcdef", server.trace_id.unpack1("H*") + assert_equal 1, spans.count { |span| span.kind == :SPAN_KIND_SERVER }, "health check was traced" + jobs = spans.select { |span| span.name.include?("TracingProbeJob") } + assert_operator jobs.size, :>=, 2, "enqueue and serialized execution must both be traced" + assert jobs.all? { |span| span.trace_id == server.trace_id }, "job execution lost its request trace" + jobs.each do |job| + assert spans.any? { |parent| parent.span_id == job.parent_span_id }, "job has no exported parent" + end + metadata = spans.flat_map(&:attributes) + assert metadata.any? { |attribute| attribute.key.start_with?("db.system") } + assert spans.any? { |span| span.kind == :SPAN_KIND_CLIENT }, "outbound HTTP was not traced" + spans.each do |span| + assert_empty span.events + assert_empty span.status.message if span.has_status? + end + end + + test "an unavailable collector does not interrupt a request or serialized job and flush is bounded" do + result = run_traced_request(collector_failure: true) + + assert_equal 200, result.fetch("status") + assert_equal "completed", result.fetch("body") + assert result.fetch("job_performed") + assert_operator result.fetch("export_attempts"), :>, 0 + assert_operator result.fetch("request_seconds"), :<, 5 + assert_operator result.fetch("flush_seconds"), :<, 5 + end + + test "a forked worker exports new job spans without replaying its parents pending request" do + result = run_traced_request(fork_worker: true) + + assert_equal 0, result.fetch("worker_exitstatus") + assert result.fetch("worker_performed") + spans = result.fetch("worker_payloads").flat_map do |payload| + request = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode \ + Base64.strict_decode64(payload) + request.resource_spans.flat_map(&:scope_spans).flat_map(&:spans) + end + assert spans.any? { |span| span.name.include?("TracingProbeJob") }, "forked worker did not export" + refute spans.any? { |span| span.kind == :SPAN_KIND_SERVER }, "child re-exported its parent's request" + spans.each { |span| assert_empty span.events } + end + + private + + def run_traced_request(collector_failure: false, fork_worker: false) + environment = ENV.keys.grep(/\AOTEL_/).to_h { |key| [ key, nil ] } + environment.merge! \ + "SECRET_KEY_BASE_DUMMY" => nil, + "RAILS_ENV" => "test", + "OTEL_TRACES_EXPORTER" => "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT" => "http://collector.example/v1/traces", + "OTEL_EXPORTER_OTLP_TIMEOUT" => "1", + "OTEL_BSP_SCHEDULE_DELAY" => "60000", + "OTEL_BSP_EXPORT_TIMEOUT" => "1000" + script = <<~RUBY + require "webmock" + require "json" + require "base64" + WebMock.enable! + WebMock.disable_net_connect! + require_relative "config/environment" + payloads = [] + export_attempts = 0 + baggage_propagated = false + intake = WebMock.stub_request(:post, "http://collector.example/v1/traces") + if #{collector_failure} + intake.to_return do + export_attempts += 1 + raise Errno::ECONNREFUSED + end + else + intake.to_return do |request| + export_attempts += 1 + payloads << Base64.strict_encode64(request.body) + { status: 200, body: "" } + end + end + WebMock.stub_request(:get, "http://upstream.example/private-value-canary?token=private-value-canary") + .to_return do |request| + baggage_propagated ||= request.headers.key?("Baggage") + { status: 200, body: "private-value-canary" } + end + class TracingProbeJob < ActiveJob::Base + def perform(secret) + raise "job argument changed" unless secret == "private-value-canary" + self.class.performed = true + end + class_attribute :performed, default: false + end + TracingProbeJob.queue_adapter = :test + class TracingProbeController < ActionController::Base + def create + ActiveRecord::Base.connection.select_value("SELECT 'private-value-canary'") + Net::HTTP.get(URI("http://upstream.example/private-value-canary?token=private-value-canary")) + TracingProbeJob.perform_later("private-value-canary") + queued = TracingProbeJob.queue_adapter.enqueued_jobs.shift.select { |key, _| key.is_a?(String) } + serialized = JSON.parse(JSON.generate(queued)) + ActiveJob::Base.execute(serialized) + render plain: "completed" + end + end + Rails.application.routes.draw do + post "/tracing-probe", to: "tracing_probe#create" + get "/up", to: "rails/health#show" + end + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + request = Rack::MockRequest.env_for \ + "/tracing-probe?token=private-value-canary", + method: "POST", + "HTTP_AUTHORIZATION" => "Bearer private-value-canary", + "HTTP_BAGGAGE" => "secret=private-value-canary", + "HTTP_TRACEPARENT" => "00-1234567890abcdef1234567890abcdef-1234567890abcdef-01", + input: "private-value-canary", + "CONTENT_TYPE" => "text/plain" + status, headers, body = Rails.application.call(request) + response_body = body.each.to_a.join + body.close if body.respond_to?(:close) + request_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + health_status, _, health_body = Rails.application.call(Rack::MockRequest.env_for("/up")) + health_body.each { |_part| } + health_body.close if health_body.respond_to?(:close) + worker = {} + if #{fork_worker} + reader, writer = IO.pipe + pid = fork do + reader.close + payloads.clear + TracingProbeJob.performed = false + TracingProbeJob.perform_now("private-value-canary") + OpenTelemetry.tracer_provider.force_flush(timeout: 1) + OpenTelemetry.tracer_provider.shutdown(timeout: 1) + writer.write(JSON.generate(payloads: payloads, performed: TracingProbeJob.performed)) + writer.close + exit! 0 + end + writer.close + worker = JSON.parse(reader.read) + reader.close + _, worker_status = Process.wait2(pid) + worker["exitstatus"] = worker_status.exitstatus + end + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + OpenTelemetry.tracer_provider.force_flush(timeout: 1) + flush_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + result = JSON.generate \ + status: status, + body: response_body, + job_performed: TracingProbeJob.performed, + health_status: health_status, + baggage_propagated: baggage_propagated, + payloads: payloads, + export_attempts: export_attempts, + request_seconds: request_seconds, + flush_seconds: flush_seconds, + worker_payloads: worker["payloads"], + worker_performed: worker["performed"], + worker_exitstatus: worker["exitstatus"] + puts "HUB_TRACING_RESULT=" + result + RUBY + output, error, status = Open3.capture3 \ + environment, + RbConfig.ruby, + "-rbundler/setup", + "-e", + script, + chdir: Rails.root + assert status.success?, "Tracing subprocess failed:\n#{output}\n#{error}" + result = output.lines.find { |line| line.start_with?("HUB_TRACING_RESULT=") } + assert result, "Tracing subprocess returned no result:\n#{output}\n#{error}" + JSON.parse(result.delete_prefix("HUB_TRACING_RESULT=")) + .merge("diagnostics" => "#{output}\n#{error}") + end +end diff --git a/hub/test/lib/hub_tracing_test.rb b/hub/test/lib/hub_tracing_test.rb new file mode 100644 index 0000000..aa8d912 --- /dev/null +++ b/hub/test/lib/hub_tracing_test.rb @@ -0,0 +1,148 @@ +require "test_helper" +require "open3" +require "opentelemetry/sdk" +require "opentelemetry/exporter/otlp" +require "opentelemetry/sdk/trace/export/in_memory_span_exporter" +require_relative "../../lib/hub_tracing/exporter" + +class HubTracingTest < ActiveSupport::TestCase + self.use_transactional_tests = false + + test "default disabled and dummy builds do not load the tracing SDK" do + [ + {}, + { "OTEL_TRACES_EXPORTER" => "none" }, + { "OTEL_TRACES_EXPORTER" => "otlp", "OTEL_SDK_DISABLED" => "true" }, + { "OTEL_TRACES_EXPORTER" => "otlp", "SECRET_KEY_BASE_DUMMY" => "1" } + ].each do |configuration| + output, error, status = run_startup(configuration, <<~RUBY) + HubTracing.start! + abort "SDK initialized while disabled" if defined?(OpenTelemetry::SDK) + puts "disabled" + RUBY + + assert status.success?, error + assert_equal "disabled", output.strip + end + end + + test "invalid enabled configuration fails without printing endpoint credentials" do + [ + { "OTEL_EXPORTER_OTLP_ENDPOINT" => nil }, + { "OTEL_EXPORTER_OTLP_ENDPOINT" => "https://user:credential-canary@collector.example" }, + { "OTEL_EXPORTER_OTLP_ENDPOINT" => "https://collector.example?token=credential-canary" }, + { "OTEL_EXPORTER_OTLP_ENDPOINT" => "https://collector.example/#credential-canary" }, + { "OTEL_EXPORTER_OTLP_ENDPOINT" => "https://[credential-canary" }, + { "OTEL_EXPORTER_OTLP_ENDPOINT" => "file:///credential-canary" }, + { "OTEL_EXPORTER_OTLP_PROTOCOL" => "grpc" }, + { "OTEL_TRACES_EXPORTER" => "console,otlp" } + ].each do |invalid| + configuration = { + "OTEL_TRACES_EXPORTER" => "otlp", + "OTEL_EXPORTER_OTLP_ENDPOINT" => "http://collector.example" + }.merge(invalid) + output, error, status = run_startup(configuration, <<~RUBY) + begin + HubTracing.start! + rescue ArgumentError => error + puts error.message + exit 42 + end + RUBY + + assert_equal 42, status.exitstatus, "#{configuration.keys}: #{output} #{error}" + refute_includes output + error, "credential-canary" + end + end + + test "serialized OTLP removes sensitive span data while preserving operational context" do + memory = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new + provider = OpenTelemetry::SDK::Trace::TracerProvider.new + provider.add_span_processor(OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(memory)) + tracer = provider.tracer("hub-tracing-test") + sensitive = "private-value-canary" + attributes = { + "http.request.method" => "POST", + "http.response.status_code" => 500, + "http.route" => "/builds/:id", + "db.system" => "sqlite", + "db.operation" => "SELECT", + "messaging.system" => "active_job", + "messaging.operation" => "process", + "code.namespace" => "TracingProbeJob", + "http.request.header.authorization" => "Bearer #{sensitive}", + "http.request.body" => sensitive, + "url.full" => "https://user:#{sensitive}@example.com/#{sensitive}?token=#{sensitive}", + "db.statement" => "SELECT '#{sensitive}'", + "db.query.parameter.0" => sensitive, + "job.arguments" => sensitive + } + link = OpenTelemetry::Trace::Link.new \ + OpenTelemetry::Trace::SpanContext.new, + { "secret" => sensitive } + span = tracer.start_span("POST /#{sensitive}", kind: :server, attributes:, links: [ link ]) + span.add_event(sensitive, attributes: { "exception.message" => sensitive }) + span.status = OpenTelemetry::Trace::Status.error(sensitive) + span.finish + client = tracer.start_span \ + "GET https://example.com/#{sensitive}", + kind: :client, + attributes: attributes.merge("http.request.method" => "GET") + client.finish + originals = memory.finished_spans + requests = [] + intake = stub_request(:post, "http://collector.example/v1/traces").with do |request| + requests << request + true + end.to_return(status: 200, body: "") + exporter = HubTracing::Exporter.new \ + OpenTelemetry::Exporter::OTLP::Exporter.new(endpoint: "http://collector.example/v1/traces") + + assert_equal OpenTelemetry::SDK::Trace::Export::SUCCESS, exporter.export(originals, timeout: 1) + + assert_requested intake, times: 1 + assert_equal "application/x-protobuf", requests.first.headers["Content-Type"] + payload = requests.first.body + refute_includes payload, sensitive + decoded = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(payload) + exported = decoded.resource_spans.flat_map(&:scope_spans).flat_map(&:spans) + assert_equal [ "POST /builds/:id", "GET /builds/:id" ], exported.map(&:name) + exported.zip(originals).each do |actual, original| + assert_equal original.trace_id, actual.trace_id + assert_equal original.span_id, actual.span_id + assert_equal original.start_timestamp, actual.start_time_unix_nano + assert_equal original.end_timestamp, actual.end_time_unix_nano + assert_empty actual.events + actual.links.each { |exported_link| assert_empty exported_link.attributes } + metadata = actual.attributes.to_h { |attribute| [ attribute.key, attribute.value ] } + assert_equal "SELECT", metadata.fetch("db.operation").string_value + assert_equal "TracingProbeJob", metadata.fetch("code.namespace").string_value + assert_equal 500, metadata.fetch("http.response.status_code").int_value + end + assert_equal :STATUS_CODE_ERROR, exported.first.status.code + assert_empty exported.first.status.message + assert_equal "POST /#{sensitive}", originals.first.name + assert_equal sensitive, originals.first.events.first.name + assert_equal sensitive, originals.first.status.description + ensure + exporter&.shutdown(timeout: 1) + provider&.shutdown(timeout: 1) + end + + private + + def run_startup(configuration, script) + environment = ENV.keys.grep(/\AOTEL_/).to_h { |key| [ key, nil ] } + .merge("SECRET_KEY_BASE_DUMMY" => nil) + .merge(configuration) + Open3.capture3 \ + environment, + RbConfig.ruby, + "-rbundler/setup", + "-Ilib", + "-rhub_tracing", + "-e", + script, + chdir: Rails.root + end +end From b9d924310ab8189d6fc0ac2dcf41274eec55e0cf Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:40:13 +0000 Subject: [PATCH 2/3] feat(hub): refine tracing diagnostics and context trust Include filtered job arguments, SQL text, and exception diagnostics, with an operator-controlled identity opt-in. Align exported metadata with bundled instrumentation and isolate public incoming trace context by default. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- hub/lib/hub_tracing.rb | 54 ++++- hub/lib/hub_tracing/data_filter.rb | 21 ++ hub/lib/hub_tracing/exporter.rb | 31 ++- hub/lib/hub_tracing/propagator.rb | 29 +++ hub/test/integration/hub_tracing_test.rb | 235 +++++++++++++++++-- hub/test/lib/hub_tracing_data_filter_test.rb | 109 +++++++++ hub/test/lib/hub_tracing_test.rb | 112 ++++++++- 7 files changed, 545 insertions(+), 46 deletions(-) create mode 100644 hub/lib/hub_tracing/data_filter.rb create mode 100644 hub/lib/hub_tracing/propagator.rb create mode 100644 hub/test/lib/hub_tracing_data_filter_test.rb diff --git a/hub/lib/hub_tracing.rb b/hub/lib/hub_tracing.rb index af3422d..654c162 100644 --- a/hub/lib/hub_tracing.rb +++ b/hub/lib/hub_tracing.rb @@ -10,6 +10,14 @@ def self.start! end return if exporter == "none" + include_sensitive_data = boolean_setting("REPLICA_OTEL_INCLUDE_SENSITIVE_DATA") + trust_incoming_context = boolean_setting("REPLICA_OTEL_TRUST_INCOMING_CONTEXT") + untraced_hosts = ENV.fetch("REPLICA_OTEL_UNTRACED_HOSTS", "").split(",", -1).map(&:strip) + untraced_hosts = [] unless ENV.key?("REPLICA_OTEL_UNTRACED_HOSTS") + if untraced_hosts.any?(&:empty?) + raise ArgumentError, "REPLICA_OTEL_UNTRACED_HOSTS must contain nonempty comma-separated hostnames" + end + protocol = ENV["OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"] || ENV.fetch("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") unless protocol == "http/protobuf" @@ -32,19 +40,39 @@ def self.start! require "opentelemetry/instrumentation/active_job" require "opentelemetry/instrumentation/net/http" require_relative "hub_tracing/exporter" + require_relative "hub_tracing/data_filter" + require_relative "hub_tracing/propagator" - processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new \ Exporter.new(OpenTelemetry::Exporter::OTLP::Exporter.new) - ) OpenTelemetry::SDK.configure do |config| config.service_name = ENV.fetch("OTEL_SERVICE_NAME", "replica-hub") - config.propagators = [ OpenTelemetry::Trace::Propagation::TraceContext.text_map_propagator ] + config.propagators = [ Propagator.new(trust_incoming_context:) ] config.add_span_processor(processor) config.use "OpenTelemetry::Instrumentation::Rack", untraced_endpoints: [ "/up" ] config.use "OpenTelemetry::Instrumentation::ActionPack", span_naming: :class config.use "OpenTelemetry::Instrumentation::ActiveJob", propagation_style: :child, span_naming: :job_class - config.use "OpenTelemetry::Instrumentation::Net::HTTP" + config.use "OpenTelemetry::Instrumentation::Net::HTTP", untraced_hosts: + end + + # Snapshot the original rules before Rails compiles them into combined regular expressions. + Rails.application.config.after_initialize do + data_filter = DataFilter.new(include_sensitive_data:) + ActiveSupport.on_load(:active_job) do + record_arguments = proc do |job| + span = OpenTelemetry::Trace.current_span + next unless span.recording? + + begin + span.set_attribute("replica.job.arguments", data_filter.job_arguments(job.arguments)) + rescue ActiveJob::SerializationError, JSON::GeneratorError + Rails.logger.warn("OpenTelemetry could not serialize job arguments; diagnostic arguments omitted") + end + end + before_enqueue(&record_arguments) + before_perform(&record_arguments) + end end # ORM instrumentation does not cover Hub's direct SQLite calls. @@ -56,10 +84,24 @@ def self.start! "database query", kind: :client, start_timestamp: started, - attributes: { "db.system.name" => "sqlite" } + attributes: { "db.system.name" => "sqlite", "db.query.text" => payload[:sql] } span.status = OpenTelemetry::Trace::Status.error if payload[:exception] + span.record_exception(payload[:exception_object]) if payload[:exception_object] span.finish(end_timestamp: finished) end - at_exit { OpenTelemetry.tracer_provider.shutdown(timeout: 5) } + at_exit do + Timeout.timeout(5) { OpenTelemetry.tracer_provider.shutdown(timeout: 5) } + rescue Timeout::Error + OpenTelemetry.handle_error(message: "OpenTelemetry shutdown exceeded its deadline; pending traces dropped") + end + end + + def self.boolean_setting(name) + value = ENV.fetch(name, "false") + unless %w[true false].include?(value.downcase) + raise ArgumentError, "#{name} must be true or false" + end + + value.casecmp?("true") end end diff --git a/hub/lib/hub_tracing/data_filter.rb b/hub/lib/hub_tracing/data_filter.rb new file mode 100644 index 0000000..49d4b32 --- /dev/null +++ b/hub/lib/hub_tracing/data_filter.rb @@ -0,0 +1,21 @@ +module HubTracing + class DataFilter + IDENTITY_KEYS = /\A(?:email|name|username|user_name|first_name|last_name|display_name|given_name|family_name|login)\z/i + + def initialize(include_sensitive_data: false) + filters = Rails.application.config.filter_parameters + if include_sensitive_data + # Only the baseline email rule is relaxed; deployment-added filters still apply. + filters = filters.reject { |filter| filter == :email || filter == "email" } + else + filters = filters + [ IDENTITY_KEYS ] + end + @filter = ActiveSupport::ParameterFilter.new(filters + [ :cookie ]) + end + + def job_arguments(arguments) + serialized = ActiveJob::Arguments.serialize(arguments) + JSON.generate(@filter.filter("arguments" => serialized).fetch("arguments")) + end + end +end diff --git a/hub/lib/hub_tracing/exporter.rb b/hub/lib/hub_tracing/exporter.rb index a891b79..986cec5 100644 --- a/hub/lib/hub_tracing/exporter.rb +++ b/hub/lib/hub_tracing/exporter.rb @@ -1,20 +1,32 @@ +require "timeout" + module HubTracing class Exporter - # Export operational metadata, not request content or SDK-generated exception messages. + # Keep this list aligned with the bundled instrumentation and Hub's diagnostic collectors. ATTRIBUTES = %w[ http.request.method http.response.status_code http.method http.status_code http.route url.scheme http.scheme server.address server.port net.peer.name net.peer.port - db.system db.system.name db.operation db.operation.name - messaging.system messaging.operation messaging.operation.name messaging.operation.type - messaging.destination.name messaging.destination_kind code.namespace code.function + db.system.name db.query.text + messaging.system messaging.destination messaging.message.id messaging.active_job.adapter.name + messaging.active_job.message.provider_job_id messaging.active_job.message.priority + code.namespace code.function replica.job.arguments ].freeze + EXCEPTION_ATTRIBUTES = %w[exception.type exception.message exception.stacktrace].freeze def initialize(exporter) @exporter = exporter end def export(spans, timeout: nil) - @exporter.export(spans.map { |span| sanitized(span) }, timeout:) + raise Timeout::Error if timeout && timeout <= 0 + + # The bundled OTLP exporter's retry sleeps can outlast its request timeout. + Timeout.timeout(timeout) do + @exporter.export(spans.map { |span| sanitized(span) }, timeout:) + end + rescue Timeout::Error + OpenTelemetry.handle_error(message: "OTLP export exceeded its deadline; trace batch dropped") + OpenTelemetry::SDK::Trace::Export::TIMEOUT end def shutdown(timeout: nil) @@ -30,7 +42,14 @@ def force_flush(timeout: nil) def sanitized(span) result = span.dup result.attributes = (span.attributes || {}).slice(*ATTRIBUTES) - result.events = [] + result.events = (span.events || []).filter_map do |event| + next unless event.name == "exception" + + OpenTelemetry::SDK::Trace::Event.new \ + event.name, + (event.attributes || {}).slice(*EXCEPTION_ATTRIBUTES), + event.timestamp + end result.links = [] result.status = case span.status.code when OpenTelemetry::Trace::Status::ERROR then OpenTelemetry::Trace::Status.error diff --git a/hub/lib/hub_tracing/propagator.rb b/hub/lib/hub_tracing/propagator.rb new file mode 100644 index 0000000..8bdb82b --- /dev/null +++ b/hub/lib/hub_tracing/propagator.rb @@ -0,0 +1,29 @@ +module HubTracing + class Propagator + def initialize(trust_incoming_context: false) + @trust_incoming_context = trust_incoming_context + @trace_context = OpenTelemetry::Trace::Propagation::TraceContext.text_map_propagator + end + + def extract(carrier, context: OpenTelemetry::Context.current, + getter: OpenTelemetry::Context::Propagation.text_map_getter) + if getter.equal?(OpenTelemetry::Common::Propagation.rack_env_getter) && !@trust_incoming_context + return OpenTelemetry::Trace.context_with_span \ + OpenTelemetry::Trace::Span::INVALID, + parent_context: context + end + + # Vendor state and baggage never cross a Hub boundary, even with trusted ingress. + @trace_context.extract({ "traceparent" => getter.get(carrier, "traceparent") }, context:) + end + + def inject(carrier, context: OpenTelemetry::Context.current, + setter: OpenTelemetry::Context::Propagation.text_map_setter) + @trace_context.inject(carrier, context:, setter:) + end + + def fields + [ "traceparent" ] + end + end +end diff --git a/hub/test/integration/hub_tracing_test.rb b/hub/test/integration/hub_tracing_test.rb index 710f768..f35db2d 100644 --- a/hub/test/integration/hub_tracing_test.rb +++ b/hub/test/integration/hub_tracing_test.rb @@ -6,7 +6,7 @@ class HubTracingIntegrationTest < ActiveSupport::TestCase self.use_transactional_tests = false - test "normal boot traces requests serialized child jobs database and outbound HTTP without private content" do + test "normal boot isolates inbound context and traces serialized child jobs SQL and HTTP diagnostics" do result = run_traced_request assert_equal 200, result.fetch("status") @@ -14,17 +14,21 @@ class HubTracingIntegrationTest < ActiveSupport::TestCase assert result.fetch("job_performed") assert_equal 200, result.fetch("health_status") refute result.fetch("baggage_propagated"), "incoming baggage reached the outbound service" + refute result.fetch("tracestate_propagated"), "incoming tracestate reached the outbound service" + assert result.fetch("traceparent_propagated"), "ordinary outbound HTTP lost context injection" payloads = result.fetch("payloads").map { |payload| Base64.strict_decode64(payload) } assert_not_empty payloads, result.fetch("diagnostics") - payloads.each { |payload| refute_includes payload, "private-value-canary" } - spans = payloads.flat_map do |payload| - request = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(payload) - request.resource_spans.flat_map(&:scope_spans).flat_map(&:spans) + payloads.each do |payload| + %w[request-private-canary identity-email-canary identity-name-canary password-canary token-canary].each do |private_value| + refute_includes payload, private_value + end end + spans = decoded_spans(result) server = spans.find { |span| span.kind == :SPAN_KIND_SERVER } assert server, "Rails boot did not produce a server span" assert_equal "POST /tracing-probe", server.name - assert_equal "1234567890abcdef1234567890abcdef", server.trace_id.unpack1("H*") + refute_equal "1234567890abcdef1234567890abcdef", server.trace_id.unpack1("H*") + assert_empty server.parent_span_id assert_equal 1, spans.count { |span| span.kind == :SPAN_KIND_SERVER }, "health check was traced" jobs = spans.select { |span| span.name.include?("TracingProbeJob") } assert_operator jobs.size, :>=, 2, "enqueue and serialized execution must both be traced" @@ -33,7 +37,22 @@ class HubTracingIntegrationTest < ActiveSupport::TestCase assert spans.any? { |parent| parent.span_id == job.parent_span_id }, "job has no exported parent" end metadata = spans.flat_map(&:attributes) - assert metadata.any? { |attribute| attribute.key.start_with?("db.system") } + assert metadata.any? { |attribute| attribute.key == "db.system.name" } + assert metadata.any? { |attribute| attribute.key == "db.query.text" && attribute.value.string_value.include?("sql-diagnostic-canary") } + jobs.each do |job| + attributes = span_attributes(job) + assert_equal "default", attributes.fetch("messaging.destination").string_value + assert attributes.fetch("messaging.message.id").string_value.present? + assert_equal "test", attributes.fetch("messaging.active_job.adapter.name").string_value + assert_equal "3", attributes.fetch("messaging.active_job.message.priority").string_value + arguments = JSON.parse(attributes.fetch("replica.job.arguments").string_value) + assert_equal "argument-diagnostic-canary", arguments.first + assert_equal "[FILTERED]", arguments.last.fetch("email") + assert_equal "[FILTERED]", arguments.last.fetch("name") + assert_equal "[FILTERED]", arguments.last.fetch("password") + assert_equal "[FILTERED]", arguments.last.fetch("token") + assert_equal "ActiveJob::Serializers::DateSerializer", arguments.last.fetch("on").fetch("_aj_serialized") + end assert spans.any? { |span| span.kind == :SPAN_KIND_CLIENT }, "outbound HTTP was not traced" spans.each do |span| assert_empty span.events @@ -41,6 +60,121 @@ class HubTracingIntegrationTest < ActiveSupport::TestCase end end + test "trusted inbound context retains sampled parent but discards tracestate and baggage" do + result = run_traced_request(configuration: { + "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "TrUe", + "OTEL_TRACES_SAMPLER" => "parentbased_traceidratio", + "OTEL_TRACES_SAMPLER_ARG" => "0" + }) + + server = decoded_spans(result).find { |span| span.kind == :SPAN_KIND_SERVER } + assert server, result.fetch("diagnostics") + assert_equal "1234567890abcdef1234567890abcdef", server.trace_id.unpack1("H*") + assert_equal "1234567890abcdef", server.parent_span_id.unpack1("H*") + refute result.fetch("baggage_propagated") + refute result.fetch("tracestate_propagated") + assert result.fetch("traceparent_propagated") + assert decoded_spans(result).all? { |span| span.trace_state.empty? } + end + + test "public inbound sampled context cannot override zero sampling" do + result = run_traced_request(configuration: { + "OTEL_TRACES_SAMPLER" => "parentbased_traceidratio", + "OTEL_TRACES_SAMPLER_ARG" => "0" + }) + + assert_equal 200, result.fetch("status") + assert result.fetch("job_performed") + assert_empty decoded_spans(result), "an untrusted sampled parent bypassed the configured sampler" + end + + test "explicit false keeps identity filtering and incoming context isolation" do + result = run_traced_request(configuration: { + "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "FaLsE", + "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "FaLsE" + }) + + server = decoded_spans(result).find { |span| span.kind == :SPAN_KIND_SERVER } + assert server, result.fetch("diagnostics") + refute_equal "1234567890abcdef1234567890abcdef", server.trace_id.unpack1("H*") + refute_includes result.fetch("payloads").map { |payload| Base64.strict_decode64(payload) }.join, "identity-email-canary" + end + + test "sensitive opt in permits structured identities but not credentials" do + result = run_traced_request(configuration: { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "TrUe" }) + + jobs = decoded_spans(result).select { |span| span.name.include?("TracingProbeJob") } + assert_operator jobs.size, :>=, 2 + jobs.each do |job| + arguments = JSON.parse(span_attributes(job).fetch("replica.job.arguments").string_value).last + assert_equal "identity-email-canary@example.test", arguments.fetch("email"), "#{job.name} did not honor identity opt in" + assert_equal "identity-name-canary", arguments.fetch("name") + assert_equal "[FILTERED]", arguments.fetch("password") + assert_equal "[FILTERED]", arguments.fetch("token") + assert_equal "[FILTERED]", arguments.fetch("customer_secret") + end + assert result.fetch("job_performed"), "diagnostic filtering changed the arguments delivered to the job" + end + + test "sensitive collection setting remains fixed until restart" do + [ [ "true", "false", "identity-email-canary@example.test" ], [ "false", "true", "[FILTERED]" ] ].each do |startup, changed, expected| + result = run_traced_request( + configuration: { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => startup }, + change_sensitive_setting: changed + ) + jobs = decoded_spans(result).select { |span| span.name.include?("TracingProbeJob") } + assert_operator jobs.size, :>=, 2 + jobs.each do |job| + arguments = JSON.parse(span_attributes(job).fetch("replica.job.arguments").string_value).last + assert_equal expected, arguments.fetch("email"), "runtime ENV change overrode startup value #{startup}" + end + end + end + + test "untraced exact hosts suppress HTTP spans and propagation without suppressing other hosts or jobs" do + result = run_traced_request(configuration: { + "REPLICA_OTEL_UNTRACED_HOSTS" => " upstream.example, other.example " + }) + + refute result.fetch("traceparent_propagated"), "untraced host received trace context" + refute result.fetch("tracestate_propagated") + refute result.fetch("baggage_propagated") + assert result.fetch("other_traceparent_propagated"), "hostname suffix incorrectly suppressed another host" + spans = decoded_spans(result) + http = spans.select { |span| span_attributes(span).key?("http.method") || span_attributes(span).key?("http.request.method") } + refute http.any? { |span| span.kind == :SPAN_KIND_CLIENT && span_attributes(span).values.any? { |value| value.string_value == "upstream.example" } } + assert http.any? { |span| span.kind == :SPAN_KIND_CLIENT && span_attributes(span).values.any? { |value| value.string_value == "not-upstream.example" } } + server = spans.find { |span| span.kind == :SPAN_KIND_SERVER } + jobs = spans.select { |span| span.name.include?("TracingProbeJob") } + assert_operator jobs.size, :>=, 2 + assert jobs.all? { |span| span.trace_id == server.trace_id } + end + + test "job and SQL failures retain useful exception and query diagnostics" do + result = run_traced_request(failing_job: true) + + assert_equal 200, result.fetch("status") + spans = decoded_spans(result) + job = spans.find { |span| span.name.include?("TracingProbeJob") && span.events.any? } + assert job, "job failure has no exported exception event" + assert_equal :STATUS_CODE_ERROR, job.status.code + assert_empty job.status.message + exception = job.events.find { |event| event.name == "exception" } + assert exception, "job failure did not preserve the exception event" + attributes = exception.attributes.to_h { |attribute| [ attribute.key, attribute.value.string_value ] } + assert_equal "RuntimeError", attributes.fetch("exception.type") + assert_includes attributes.fetch("exception.message"), "job-failure-diagnostic-canary" + assert attributes.fetch("exception.stacktrace").present? + assert_equal %w[exception.message exception.stacktrace exception.type], attributes.keys.sort + query = spans.find do |span| + span_attributes(span)["db.query.text"]&.string_value&.include?("missing_diagnostic_table") + end + assert query, "failed SQL query was not exported" + assert_equal :STATUS_CODE_ERROR, query.status.code + assert spans.flat_map(&:attributes).none? { |attribute| attribute.key.match?(/bind|parameter/) } + assert spans.flat_map(&:events).all? { |event| event.name == "exception" } + end + test "an unavailable collector does not interrupt a request or serialized job and flush is bounded" do result = run_traced_request(collector_failure: true) @@ -69,8 +203,20 @@ class HubTracingIntegrationTest < ActiveSupport::TestCase private - def run_traced_request(collector_failure: false, fork_worker: false) - environment = ENV.keys.grep(/\AOTEL_/).to_h { |key| [ key, nil ] } + def decoded_spans(result) + result.fetch("payloads").flat_map do |payload| + request = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode \ + Base64.strict_decode64(payload) + request.resource_spans.flat_map(&:scope_spans).flat_map(&:spans) + end + end + + def span_attributes(span) + span.attributes.to_h { |attribute| [ attribute.key, attribute.value ] } + end + + def run_traced_request(collector_failure: false, fork_worker: false, configuration: {}, failing_job: false, change_sensitive_setting: nil) + environment = ENV.keys.grep(/\A(?:OTEL_|REPLICA_OTEL_)/).to_h { |key| [ key, nil ] } environment.merge! \ "SECRET_KEY_BASE_DUMMY" => nil, "RAILS_ENV" => "test", @@ -79,6 +225,7 @@ def run_traced_request(collector_failure: false, fork_worker: false) "OTEL_EXPORTER_OTLP_TIMEOUT" => "1", "OTEL_BSP_SCHEDULE_DELAY" => "60000", "OTEL_BSP_EXPORT_TIMEOUT" => "1000" + environment.merge!(configuration) script = <<~RUBY require "webmock" require "json" @@ -86,9 +233,14 @@ def run_traced_request(collector_failure: false, fork_worker: false) WebMock.enable! WebMock.disable_net_connect! require_relative "config/environment" + Rails.application.config.filter_parameters += [ :customer_secret ] + ENV["REPLICA_OTEL_INCLUDE_SENSITIVE_DATA"] = #{change_sensitive_setting.inspect} unless #{change_sensitive_setting.nil?} payloads = [] export_attempts = 0 baggage_propagated = false + tracestate_propagated = false + traceparent_propagated = false + other_traceparent_propagated = false intake = WebMock.stub_request(:post, "http://collector.example/v1/traces") if #{collector_failure} intake.to_return do @@ -102,14 +254,27 @@ def run_traced_request(collector_failure: false, fork_worker: false) { status: 200, body: "" } end end - WebMock.stub_request(:get, "http://upstream.example/private-value-canary?token=private-value-canary") + WebMock.stub_request(:get, "http://upstream.example/request-private-canary?token=request-private-canary") .to_return do |request| baggage_propagated ||= request.headers.key?("Baggage") - { status: 200, body: "private-value-canary" } + tracestate_propagated ||= request.headers.key?("Tracestate") + traceparent_propagated ||= request.headers.key?("Traceparent") + { status: 200, body: "request-private-canary" } + end + WebMock.stub_request(:get, "http://not-upstream.example/") + .to_return do |request| + other_traceparent_propagated ||= request.headers.key?("Traceparent") + { status: 200, body: "" } end class TracingProbeJob < ActiveJob::Base - def perform(secret) - raise "job argument changed" unless secret == "private-value-canary" + def perform(diagnostic, values) + raise "job argument changed" unless diagnostic == "argument-diagnostic-canary" && + values.fetch("email") == "identity-email-canary@example.test" && + values.fetch("name") == "identity-name-canary" && + values.fetch("password") == "password-canary" && + values.fetch("token") == "token-canary" && + values.fetch("on") == Date.new(2026, 10, 19) + raise "job-failure-diagnostic-canary" if #{failing_job} self.class.performed = true end class_attribute :performed, default: false @@ -117,12 +282,30 @@ def perform(secret) TracingProbeJob.queue_adapter = :test class TracingProbeController < ActionController::Base def create - ActiveRecord::Base.connection.select_value("SELECT 'private-value-canary'") - Net::HTTP.get(URI("http://upstream.example/private-value-canary?token=private-value-canary")) - TracingProbeJob.perform_later("private-value-canary") + ActiveRecord::Base.connection.select_value("SELECT 'sql-diagnostic-canary'") + if #{failing_job} + begin + ActiveRecord::Base.connection.select_value("SELECT * FROM missing_diagnostic_table") + rescue ActiveRecord::StatementInvalid + end + end + Net::HTTP.get(URI("http://upstream.example/request-private-canary?token=request-private-canary")) + Net::HTTP.get(URI("http://not-upstream.example/")) + TracingProbeJob.set(priority: 3).perform_later("argument-diagnostic-canary", { + "email" => "identity-email-canary@example.test", + "name" => "identity-name-canary", + "password" => "password-canary", + "token" => "token-canary", + "customer_secret" => "app-secret-canary", + "on" => Date.new(2026, 10, 19) + }) queued = TracingProbeJob.queue_adapter.enqueued_jobs.shift.select { |key, _| key.is_a?(String) } serialized = JSON.parse(JSON.generate(queued)) - ActiveJob::Base.execute(serialized) + begin + ActiveJob::Base.execute(serialized) + rescue RuntimeError => error + raise unless #{failing_job} && error.message == "job-failure-diagnostic-canary" + end render plain: "completed" end end @@ -132,12 +315,13 @@ def create end started = Process.clock_gettime(Process::CLOCK_MONOTONIC) request = Rack::MockRequest.env_for \ - "/tracing-probe?token=private-value-canary", + "/tracing-probe?token=request-private-canary", method: "POST", "HTTP_AUTHORIZATION" => "Bearer private-value-canary", - "HTTP_BAGGAGE" => "secret=private-value-canary", + "HTTP_BAGGAGE" => "secret=request-private-canary", "HTTP_TRACEPARENT" => "00-1234567890abcdef1234567890abcdef-1234567890abcdef-01", - input: "private-value-canary", + "HTTP_TRACESTATE" => "vendor=request-private-canary", + input: "request-private-canary", "CONTENT_TYPE" => "text/plain" status, headers, body = Rails.application.call(request) response_body = body.each.to_a.join @@ -153,7 +337,13 @@ def create reader.close payloads.clear TracingProbeJob.performed = false - TracingProbeJob.perform_now("private-value-canary") + TracingProbeJob.perform_now("argument-diagnostic-canary", { + "email" => "identity-email-canary@example.test", + "name" => "identity-name-canary", + "password" => "password-canary", + "token" => "token-canary", + "on" => Date.new(2026, 10, 19) + }) OpenTelemetry.tracer_provider.force_flush(timeout: 1) OpenTelemetry.tracer_provider.shutdown(timeout: 1) writer.write(JSON.generate(payloads: payloads, performed: TracingProbeJob.performed)) @@ -175,6 +365,9 @@ def create job_performed: TracingProbeJob.performed, health_status: health_status, baggage_propagated: baggage_propagated, + tracestate_propagated: tracestate_propagated, + traceparent_propagated: traceparent_propagated, + other_traceparent_propagated: other_traceparent_propagated, payloads: payloads, export_attempts: export_attempts, request_seconds: request_seconds, diff --git a/hub/test/lib/hub_tracing_data_filter_test.rb b/hub/test/lib/hub_tracing_data_filter_test.rb new file mode 100644 index 0000000..a90754e --- /dev/null +++ b/hub/test/lib/hub_tracing_data_filter_test.rb @@ -0,0 +1,109 @@ +require "test_helper" +require "json" +require_relative "../../lib/hub_tracing/data_filter" + +class HubTracingDataFilterTest < ActiveSupport::TestCase + test "default argument filtering recursively removes structured identity and credential values without mutation" do + identities = %w[email name username user_name first_name last_name display_name given_name family_name login] + identity_values = identities.to_h { |key| [ key, "#{key}-identity-canary" ] } + arguments = [ + { + "nested" => [ identity_values ], + "password" => "password-canary", + "access_token" => "token-canary", + "operation" => "publish" + }, + "positional-diagnostic-canary" + ] + original = Marshal.load(Marshal.dump(arguments)) + + result = JSON.parse(HubTracing::DataFilter.new.job_arguments(arguments)) + + assert_equal original, arguments + identities.each { |key| assert_equal "[FILTERED]", result.first.fetch("nested").first.fetch(key) } + assert_equal "[FILTERED]", result.first.fetch("password") + assert_equal "[FILTERED]", result.first.fetch("access_token") + assert_equal "publish", result.first.fetch("operation") + assert_equal "positional-diagnostic-canary", result.last + end + + test "sensitive opt in retains identities while preserving additional Rails literal regex and proc filters" do + original_filters = Rails.application.config.filter_parameters + Rails.application.config.filter_parameters = [ + :email, "email", :password, :token, :customer_secret, /private_email/, + ->(key, value) { value.replace("[APP FILTERED]") if key == "internal_note" } + ] + filter = HubTracing::DataFilter.new(include_sensitive_data: true) + arguments = [ + { + "email" => "person@example.test", + "name" => "Example Person", + "login" => "example-person", + "password" => "password-canary", + "token" => "token-canary", + "customer_secret" => "app-secret-canary", + "private_email" => "private@example.test", + "internal_note" => "note-canary" + } + ] + original = Marshal.load(Marshal.dump(arguments)) + + result = JSON.parse(filter.job_arguments(arguments)).first + + assert_equal "person@example.test", result.fetch("email") + assert_equal "Example Person", result.fetch("name") + assert_equal "example-person", result.fetch("login") + %w[password token customer_secret private_email].each do |key| + assert_equal "[FILTERED]", result.fetch(key) + end + assert_equal "[APP FILTERED]", result.fetch("internal_note") + assert_equal original, arguments + ensure + Rails.application.config.filter_parameters = original_filters + end + + test "constructed filters retain identity policy and credentials after Rails replaces its source with precompiled rules" do + original_filters = Rails.application.config.filter_parameters + Rails.application.config.filter_parameters = [ :email, :password, :token, :customer_secret ] + default_filter = HubTracing::DataFilter.new + opted_in_filter = HubTracing::DataFilter.new(include_sensitive_data: true) + source = Rails.application.config.filter_parameters + source.replace(ActiveSupport::ParameterFilter.precompile_filters(source)) + arguments = [ + { + "email" => "person@example.test", + "name" => "Example Person", + "password" => "password-canary", + "token" => "token-canary", + "customer_secret" => "app-secret-canary" + } + ] + + default = JSON.parse(default_filter.job_arguments(arguments)).first + opted_in = JSON.parse(opted_in_filter.job_arguments(arguments)).first + + assert_equal "[FILTERED]", default.fetch("email") + assert_equal "[FILTERED]", default.fetch("name") + assert_equal "person@example.test", opted_in.fetch("email") + assert_equal "Example Person", opted_in.fetch("name") + [ default, opted_in ].each do |result| + %w[password token customer_secret].each { |key| assert_equal "[FILTERED]", result.fetch(key) } + end + ensure + Rails.application.config.filter_parameters = original_filters + end + + test "sensitive opt in does not remove an application rule for another identity field" do + original_filters = Rails.application.config.filter_parameters + Rails.application.config.filter_parameters = [ :email, :name, /email/ ] + arguments = [ { "email" => "person@example.test", "name" => "Example Person", "login" => "example-person" } ] + + result = JSON.parse(HubTracing::DataFilter.new(include_sensitive_data: true).job_arguments(arguments)).first + + assert_equal "[FILTERED]", result.fetch("email") + assert_equal "[FILTERED]", result.fetch("name") + assert_equal "example-person", result.fetch("login") + ensure + Rails.application.config.filter_parameters = original_filters + end +end diff --git a/hub/test/lib/hub_tracing_test.rb b/hub/test/lib/hub_tracing_test.rb index aa8d912..46339a6 100644 --- a/hub/test/lib/hub_tracing_test.rb +++ b/hub/test/lib/hub_tracing_test.rb @@ -13,7 +13,8 @@ class HubTracingTest < ActiveSupport::TestCase {}, { "OTEL_TRACES_EXPORTER" => "none" }, { "OTEL_TRACES_EXPORTER" => "otlp", "OTEL_SDK_DISABLED" => "true" }, - { "OTEL_TRACES_EXPORTER" => "otlp", "SECRET_KEY_BASE_DUMMY" => "1" } + { "OTEL_TRACES_EXPORTER" => "otlp", "SECRET_KEY_BASE_DUMMY" => "1" }, + { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "invalid", "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "invalid" } ].each do |configuration| output, error, status = run_startup(configuration, <<~RUBY) HubTracing.start! @@ -35,7 +36,16 @@ class HubTracingTest < ActiveSupport::TestCase { "OTEL_EXPORTER_OTLP_ENDPOINT" => "https://[credential-canary" }, { "OTEL_EXPORTER_OTLP_ENDPOINT" => "file:///credential-canary" }, { "OTEL_EXPORTER_OTLP_PROTOCOL" => "grpc" }, - { "OTEL_TRACES_EXPORTER" => "console,otlp" } + { "OTEL_TRACES_EXPORTER" => "console,otlp" }, + { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "credential-canary" }, + { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "" }, + { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "1" }, + { "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "credential-canary" }, + { "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "" }, + { "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "yes" }, + { "REPLICA_OTEL_UNTRACED_HOSTS" => "upstream.example," }, + { "REPLICA_OTEL_UNTRACED_HOSTS" => ",upstream.example" }, + { "REPLICA_OTEL_UNTRACED_HOSTS" => "upstream.example, ,other.example" } ].each do |invalid| configuration = { "OTEL_TRACES_EXPORTER" => "otlp", @@ -55,7 +65,41 @@ class HubTracingTest < ActiveSupport::TestCase end end - test "serialized OTLP removes sensitive span data while preserving operational context" do + test "export deadline interrupts a sleeping exporter even when it rescues ordinary errors" do + delegate = Object.new + entered = false + completed = false + delegate.define_singleton_method(:export) do |_, timeout:| + entered = true + sleep 0.25 + completed = true + OpenTelemetry::SDK::Trace::Export::SUCCESS + rescue StandardError + OpenTelemetry::SDK::Trace::Export::SUCCESS + end + exporter = HubTracing::Exporter.new(delegate) + + assert_equal OpenTelemetry::SDK::Trace::Export::TIMEOUT, exporter.export([], timeout: 0.01) + assert entered, "delegate was never called" + refute completed, "delegate continued beyond the export deadline" + end + + test "expired export deadlines do not send spans" do + delegate = Object.new + calls = 0 + delegate.define_singleton_method(:export) do |_, timeout:| + calls += 1 + OpenTelemetry::SDK::Trace::Export::SUCCESS + end + exporter = HubTracing::Exporter.new(delegate) + + [ 0, -1 ].each do |timeout| + assert_equal OpenTelemetry::SDK::Trace::Export::TIMEOUT, exporter.export([], timeout:) + end + assert_equal 0, calls + end + + test "serialized OTLP preserves emitted metadata and exception diagnostics but removes unrelated private content" do memory = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new provider = OpenTelemetry::SDK::Trace::TracerProvider.new provider.add_span_processor(OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(memory)) @@ -65,23 +109,54 @@ class HubTracingTest < ActiveSupport::TestCase "http.request.method" => "POST", "http.response.status_code" => 500, "http.route" => "/builds/:id", - "db.system" => "sqlite", - "db.operation" => "SELECT", + "http.method" => "POST", + "http.status_code" => 500, + "url.scheme" => "https", + "http.scheme" => "https", + "server.address" => "example.com", + "server.port" => 443, + "net.peer.name" => "example.com", + "net.peer.port" => 443, + "db.system.name" => "sqlite", + "db.query.text" => "SELECT 'sql-diagnostic-canary'", "messaging.system" => "active_job", - "messaging.operation" => "process", + "messaging.destination" => "default", + "messaging.message.id" => "job-id", + "messaging.active_job.adapter.name" => "ActiveJob::QueueAdapters::TestAdapter", + "messaging.active_job.message.provider_job_id" => "provider-id", + "messaging.active_job.message.priority" => "3", "code.namespace" => "TracingProbeJob", + "code.function" => "perform", + "replica.job.arguments" => '["argument-diagnostic-canary"]', "http.request.header.authorization" => "Bearer #{sensitive}", "http.request.body" => sensitive, "url.full" => "https://user:#{sensitive}@example.com/#{sensitive}?token=#{sensitive}", "db.statement" => "SELECT '#{sensitive}'", "db.query.parameter.0" => sensitive, - "job.arguments" => sensitive + "job.arguments" => sensitive, + "db.system" => sensitive, + "db.operation" => sensitive, + "db.operation.name" => sensitive, + "messaging.operation" => sensitive, + "messaging.operation.name" => sensitive, + "messaging.operation.type" => sensitive, + "messaging.destination.name" => sensitive, + "messaging.destination_kind" => sensitive } link = OpenTelemetry::Trace::Link.new \ OpenTelemetry::Trace::SpanContext.new, { "secret" => sensitive } span = tracer.start_span("POST /#{sensitive}", kind: :server, attributes:, links: [ link ]) span.add_event(sensitive, attributes: { "exception.message" => sensitive }) + span.add_event \ + "exception", + attributes: { + "exception.type" => "RuntimeError", + "exception.message" => "exception-diagnostic-canary", + "exception.stacktrace" => "probe.rb:1:in `perform'", + "exception.escaped" => true, + "request.body" => sensitive + } span.status = OpenTelemetry::Trace::Status.error(sensitive) span.finish client = tracer.start_span \ @@ -112,13 +187,24 @@ class HubTracingTest < ActiveSupport::TestCase assert_equal original.span_id, actual.span_id assert_equal original.start_timestamp, actual.start_time_unix_nano assert_equal original.end_timestamp, actual.end_time_unix_nano - assert_empty actual.events - actual.links.each { |exported_link| assert_empty exported_link.attributes } + assert_empty actual.links metadata = actual.attributes.to_h { |attribute| [ attribute.key, attribute.value ] } - assert_equal "SELECT", metadata.fetch("db.operation").string_value - assert_equal "TracingProbeJob", metadata.fetch("code.namespace").string_value - assert_equal 500, metadata.fetch("http.response.status_code").int_value + attributes.reject { |_, value| value.to_s.include?(sensitive) } + .merge("http.request.method" => original.attributes["http.request.method"]).each do |key, value| + assert metadata.key?(key), "missing emitted metadata #{key}" + assert_equal value, metadata.fetch(key).public_send(value.is_a?(Integer) ? :int_value : :string_value) + end end + assert_equal [ "exception" ], exported.first.events.map(&:name) + event = exported.first.events.first + assert_equal originals.first.events.last.timestamp, event.time_unix_nano + expected_exception = { + "exception.type" => "RuntimeError", + "exception.message" => "exception-diagnostic-canary", + "exception.stacktrace" => "probe.rb:1:in `perform'" + } + assert_equal expected_exception, event.attributes.to_h { |attribute| [ attribute.key, attribute.value.string_value ] } + assert_empty exported.last.events assert_equal :STATUS_CODE_ERROR, exported.first.status.code assert_empty exported.first.status.message assert_equal "POST /#{sensitive}", originals.first.name @@ -132,7 +218,7 @@ class HubTracingTest < ActiveSupport::TestCase private def run_startup(configuration, script) - environment = ENV.keys.grep(/\AOTEL_/).to_h { |key| [ key, nil ] } + environment = ENV.keys.grep(/\A(?:OTEL_|REPLICA_OTEL_)/).to_h { |key| [ key, nil ] } .merge("SECRET_KEY_BASE_DUMMY" => nil) .merge(configuration) Open3.capture3 \ From 286e4d97a6166208ff4c6144bf2a24c8314c4849 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:40:13 +0000 Subject: [PATCH 3/3] docs(hub): explain tracing data and trust controls Document identity opt-in, trusted ingress, outbound exclusions, and the limits of key-based filtering. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- hub/docs/deployment.md | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/hub/docs/deployment.md b/hub/docs/deployment.md index e5cfd10..70b91e2 100644 --- a/hub/docs/deployment.md +++ b/hub/docs/deployment.md @@ -43,6 +43,9 @@ Restart every Hub web and job container after changing these settings. No image | `OTEL_TRACES_SAMPLER_ARG` | Sampling-policy argument; for `parentbased_traceidratio`, a fraction from `0` to `1`, such as `0.1`. | | `OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE` | Optional path inside the container to a trusted CA certificate for a private endpoint. Mount the certificate when needed; keep TLS verification enabled. | | `OTEL_SDK_DISABLED` | Set to `true` to disable tracing regardless of the exporter setting. | +| `REPLICA_OTEL_INCLUDE_SENSITIVE_DATA` | Set to `true` to include structured job-argument identity fields listed below. Defaults to `false`; credential-keyed fields remain filtered. | +| `REPLICA_OTEL_TRUST_INCOMING_CONTEXT` | Set to `true` only behind a trusted ingress that removes client-supplied trace headers and supplies its own. Defaults to `false`. | +| `REPLICA_OTEL_UNTRACED_HOSTS` | Optional comma-separated exact hostnames to exclude from outbound Net::HTTP tracing and trace-header injection. Subdomains are not included automatically. Empty entries are rejected. | The exporter also accepts the common `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, and `OTEL_EXPORTER_OTLP_PROTOCOL` settings. Trace-specific values take precedence. The common endpoint is a base URL to which the exporter appends `/v1/traces`; the trace-specific endpoint is used as supplied. Enabling export requires an explicit endpoint. Hub rejects unsupported exporters, protocols, and malformed endpoint URLs at startup. @@ -54,13 +57,31 @@ Sentry documents a direct OTLP trace endpoint. Datadog provides a Collector expo The deployment operator controls the destination and credentials. Restrict recipients and network access through your deployment's outbound-network policy. Hub's repository analytics destination settings do not configure or restrict tracing. +For mutual TLS, the exporter accepts `OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE` and `OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY`, pointing to mounted client certificate and key files. Keep certificate verification enabled. + +### Trace-context boundaries + +Public HTTP requests start new traces by default. Hub ignores their incoming `traceparent`, so a client cannot override the deployment's root sampling decision by supplying a sampled parent. Enable `REPLICA_OTEL_TRUST_INCOMING_CONTEXT` only when all traffic passes through an ingress that sanitizes client-supplied trace headers. This setting trusts every incoming request; it does not authenticate a proxy or enforce a network restriction. + +Hub discards incoming `tracestate` and baggage even with trusted ingress. Jobs continue the context serialized by Hub through its trusted job queue. Protect queue storage and access independently. + +Outbound Net::HTTP calls propagate trace context by default. Use `REPLICA_OTEL_UNTRACED_HOSTS` to exclude specific destinations; those calls also lose their HTTP spans. This setting does not block the requests or restrict telemetry export destinations. Hub fixes its propagation policy in code; `OTEL_PROPAGATORS` does not override these boundaries. + ### Data and delivery limits -Hub exports an allowlist of operational span attributes. It removes raw URL paths and queries, request and response headers, bodies, SQL text and parameter values, job arguments, span events, span links, and error status descriptions before export. HTTP span names use methods and route templates instead of concrete URLs. Trace context can cross service boundaries, but Hub does not propagate incoming OpenTelemetry baggage. Operator-supplied resource attributes are exported as configured. +Tracing includes SQL query text, serialized job arguments, and exception type, message, and stack trace by default. Job spans retain queue, job ID, adapter, provider job ID, and priority metadata when the instrumentation supplies them. Hub uses explicit attribute lists matching its bundled instrumentation, including legacy HTTP and code keys that those versions still emit. + +Hub filters structured job arguments with Rails' configured parameter filters. By default it also filters the keys `email`, `name`, `username`, `user_name`, `first_name`, `last_name`, `display_name`, `given_name`, `family_name`, and `login`. Set `REPLICA_OTEL_INCLUDE_SENSITIVE_DATA=true` to include those identity fields. This relaxes only Hub's additional identity filter and Rails' baseline literal `email` rule; other configured filters still apply, including password, token, credential, and authorization rules. Cookie-keyed values remain filtered in either mode. The flag does not attach signed-in user records or enable request bodies or headers. + +Filtering is key-based, not a guarantee that exported data contains no personal information or secrets. SQL text can contain literal values, exception messages and stack traces can contain application data, and positional or free-text job arguments lack field names to filter. SQL bind values are not attached separately. Serialized model arguments may include Global IDs. Configure your APM's access and retention policies accordingly, and apply additional Collector filtering if needed. Operator-supplied resource attributes are exported as configured. + +The deployment operator controls the identity flag through environment configuration. Restart every web and job process when enabling or disabling it. The new setting applies to diagnostics collected by those processes; data already sent remains subject to the recipient's retention policy. + +Hub excludes raw HTTP URL paths and queries, request and response headers and bodies, arbitrary span events, span links, and error status descriptions from export. Exception events retain only `exception.type`, `exception.message`, and `exception.stacktrace`. HTTP span names use methods and route templates instead of concrete URLs. Active Job execution continues the enqueueing trace when the job's serialized context is available. The SDK's bounded background queue exports batches outside the request path and resets its worker after process forks. Hub allows up to five seconds to flush at normal process exit. Collector or provider outages are logged and may drop traces; delivery is not durable and does not gate successful Hub requests. Forced termination can lose pending spans. -Health checks at `/up` are excluded. Asset compilation with `SECRET_KEY_BASE_DUMMY` does not initialize tracing. This integration does not enable browser monitoring, metric or log export, exception stack-trace export, or vendor-native error reporting. +Health checks at `/up` are excluded. Asset compilation with `SECRET_KEY_BASE_DUMMY` does not initialize tracing. This integration does not enable browser monitoring, metric or log export, or vendor-native error reporting. Providers that discard OTLP span events may not display the exported exceptions. For exporter settings and their formats, see the [Ruby OTLP exporter](https://github.com/open-telemetry/opentelemetry-ruby/tree/main/exporter/otlp) and the [Ruby SDK documentation](https://opentelemetry.io/docs/languages/ruby/). Provider setup examples are available for [Sentry OTLP](https://docs.sentry.io/concepts/otlp/direct/traces/) and the [Datadog Collector exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/datadogexporter).