diff --git a/hub/Gemfile b/hub/Gemfile index 62b1bee..d57f5b3 100644 --- a/hub/Gemfile +++ b/hub/Gemfile @@ -7,6 +7,11 @@ gem "bootsnap", require: false gem "json", "~> 3.0" gem "jwt", "~> 3.3" gem "oaken", "1.0.0" +gem "opentelemetry-exporter-otlp", require: false +gem "opentelemetry-instrumentation-action_pack", require: false +gem "opentelemetry-instrumentation-active_job", require: false +gem "opentelemetry-instrumentation-net_http", require: false +gem "opentelemetry-sdk", require: false gem "propshaft" gem "puma", ">= 5.0" gem "rails", "~> 8.1.4" diff --git a/hub/Gemfile.lock b/hub/Gemfile.lock index 36d13ba..d17da70 100644 --- a/hub/Gemfile.lock +++ b/hub/Gemfile.lock @@ -123,6 +123,26 @@ GEM raabro (~> 1.4) globalid (1.4.0) activesupport (>= 6.1) + google-protobuf (4.36.2) + bigdecimal + rake (~> 13.3) + google-protobuf (4.36.2-aarch64-linux-gnu) + bigdecimal + rake (~> 13.3) + google-protobuf (4.36.2-aarch64-linux-musl) + bigdecimal + rake (~> 13.3) + google-protobuf (4.36.2-arm64-darwin) + bigdecimal + rake (~> 13.3) + google-protobuf (4.36.2-x86_64-linux-gnu) + bigdecimal + rake (~> 13.3) + google-protobuf (4.36.2-x86_64-linux-musl) + bigdecimal + rake (~> 13.3) + googleapis-common-protos-types (1.23.0) + google-protobuf (~> 4.26) hashdiff (1.2.1) i18n (1.15.2) concurrent-ruby (~> 1.0) @@ -182,6 +202,51 @@ GEM oj (3.17.6) bigdecimal (>= 3.0) ostruct (>= 0.2) + opentelemetry-api (1.11.1) + logger + opentelemetry-common (0.25.2) + opentelemetry-api (~> 1.0) + opentelemetry-exporter-otlp (0.37.0) + google-protobuf (>= 3.18) + googleapis-common-protos-types (~> 1.3) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-exporter-otlp-common (~> 0.2) + opentelemetry-sdk (~> 1.10) + opentelemetry-semantic_conventions + opentelemetry-exporter-otlp-common (0.3.0) + google-protobuf (>= 3.18) + googleapis-common-protos-types (~> 1.3) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-instrumentation-action_pack (0.18.1) + opentelemetry-instrumentation-rack (~> 0.29) + opentelemetry-instrumentation-active_job (0.13.0) + opentelemetry-instrumentation-base (~> 0.25) + opentelemetry-instrumentation-base (0.27.0) + opentelemetry-api (~> 1.7) + opentelemetry-common (~> 0.21) + opentelemetry-logs-api (~> 0.6) + opentelemetry-metrics-api (~> 0.9) + opentelemetry-registry (~> 0.1) + opentelemetry-instrumentation-net_http (0.29.2) + opentelemetry-instrumentation-base (~> 0.25) + opentelemetry-instrumentation-rack (0.31.2) + opentelemetry-instrumentation-base (~> 0.25) + opentelemetry-logs-api (0.6.0) + opentelemetry-api (~> 1.0) + opentelemetry-metrics-api (0.9.0) + opentelemetry-api (~> 1.0) + opentelemetry-registry (0.6.3) + opentelemetry-api (~> 1.1) + opentelemetry-sdk (1.13.1) + logger + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-registry (~> 0.2) + opentelemetry-semantic_conventions + opentelemetry-semantic_conventions (1.44.0) + opentelemetry-api (~> 1.0) ostruct (0.6.3) parallel (2.2.0) parser (3.3.12.0) @@ -377,6 +442,11 @@ DEPENDENCIES json (~> 3.0) jwt (~> 3.3) oaken (= 1.0.0) + opentelemetry-exporter-otlp + opentelemetry-instrumentation-action_pack + opentelemetry-instrumentation-active_job + opentelemetry-instrumentation-net_http + opentelemetry-sdk propshaft puma (>= 5.0) rails (~> 8.1.4) @@ -435,6 +505,13 @@ CHECKSUMS form_props (0.2.2) sha256=a46f8f7fd216b26ed6d6416d22334344a531188aee0e854aebc81761ee280cad fugit (1.13.0) sha256=a4f093fce740da52f216740a5041e2a594ea763cdb89e8b2754ca4399634ab18 globalid (1.4.0) sha256=037f12fbf1d9d7a014d501c2d5c77356fd4ddd96d7a7991d6700bba96706f427 + google-protobuf (4.36.2) sha256=2f8daff50ae14aad36ab0ccdcf284065fa71d726c444efe114e6db82ffe62494 + google-protobuf (4.36.2-aarch64-linux-gnu) sha256=c3279ec6eaa59f6ed6276eda2a33cffb4306471183ab79a332e698b8564f5169 + google-protobuf (4.36.2-aarch64-linux-musl) sha256=0772c9dde1e6dce9a6b44d1d35c28c29adee99d9bd6aff5e21840daeb22161ce + google-protobuf (4.36.2-arm64-darwin) sha256=f70d810e6bbab9d8ad9c1d5fd685a21de4c10ad3ad32efb9455e17e5b1286917 + google-protobuf (4.36.2-x86_64-linux-gnu) sha256=d25a820873e423f1d209ae8c672a4a28b019e2429debd0901d26670a0b4e14db + google-protobuf (4.36.2-x86_64-linux-musl) sha256=0fa17686c896da96a60a705e56ddc7d4ff56413f52787c655bd4c9fb33ffe14f + googleapis-common-protos-types (1.23.0) sha256=992e740a523794d9fc5f29a504465d8fc737aaa16c930fe7228e3346860faf0a hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5 io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08 @@ -465,6 +542,20 @@ CHECKSUMS nokogiri (1.19.4-x86_64-linux-musl) sha256=17dfb7c1fa194ae02fbf7c51a7afc8d278045ab3fdacfd86f91d02d7b274470b oaken (1.0.0) sha256=e89249bc4f6cd3ab9b5e3abdc06c377fa772e6bcb3005cc09ff044bb8d3f1dc2 oj (3.17.6) sha256=f511257e01a12030c3ceb8023cd2e96fca924ea9c741f57ea366816f1fd8e68c + opentelemetry-api (1.11.1) sha256=d07f9987ea6240b3fd5ed5dd7450f93c58e403e0553e804069c23ae18ba61ff7 + opentelemetry-common (0.25.2) sha256=c035d44987789356fe8feeeb870708b44fe7564ffc032dec7200c39da92a911d + opentelemetry-exporter-otlp (0.37.0) sha256=edc8bda657dce5f4d527ac735f75d44043bb5924de6df0e3f01260108aa76136 + opentelemetry-exporter-otlp-common (0.3.0) sha256=855d10d788a7272dfe5b14efa229507619abbef29085ef4a8acfd2c0d3af6171 + opentelemetry-instrumentation-action_pack (0.18.1) sha256=0b827d703baa2951458e36f2bdfa4a2cdb17a0130cfb0e1b6fe5d67fadc85c2d + opentelemetry-instrumentation-active_job (0.13.0) sha256=42da156924bbabdaca6268be1e46963a04829bd0b2b82fc6e368597a3525cde2 + opentelemetry-instrumentation-base (0.27.0) sha256=e55edb82cd0b018e82e746be61fa36cb05b7b68b2fa37f0e4a96b923ad3d7735 + opentelemetry-instrumentation-net_http (0.29.2) sha256=4256105f058404eb0bd43e4a987e48f9e48fcade06b40a388092c380b2e946eb + opentelemetry-instrumentation-rack (0.31.2) sha256=4a183066110455711a0c77754e3f46652f646221d7704d968a48d155b47f8309 + opentelemetry-logs-api (0.6.0) sha256=b889ac61d90326bffadb83bf7278d246d1086e2349c9052aeb072ed375ce73d0 + opentelemetry-metrics-api (0.9.0) sha256=e11a12055113e11421313f71c758e5e10e4662e9e5c3b91786218a692e81206d + opentelemetry-registry (0.6.3) sha256=568fa8eaea94394f29f4fce3f0339ab10c3bf6d6c8beed6b3cfee6fd37fcc608 + opentelemetry-sdk (1.13.1) sha256=91c9cc315ac7ef30243549eeff8b27cd0d39bd02bdf9c9ae481ab96a1cec6754 + opentelemetry-semantic_conventions (1.44.0) sha256=b09b8793422f13b01c08f7a6f29b5f511f47f37bcdb57c0513465a0464168513 ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912 parallel (2.2.0) sha256=e1059c5fd7b649558a0aec38a769f06a42942bdb40503d005a59c352fe011cd8 parser (3.3.12.0) sha256=21a6d7f755d5a24dfbdc6e6b772e4e879a52e7631a88bc5a3a134606052c9828 diff --git a/hub/config/application.rb b/hub/config/application.rb index ac8fcab..b48369a 100644 --- a/hub/config/application.rb +++ b/hub/config/application.rb @@ -85,4 +85,7 @@ class Application < Rails::Application # only to replica-learning and never stores it. See docs/copilot.md. config.x.copilot_token = ENV["REPLICA_COPILOT_TOKEN"].presence end + + require_relative "../lib/hub_tracing" + HubTracing.start! end diff --git a/hub/docs/README.md b/hub/docs/README.md index 91add69..9637091 100644 --- a/hub/docs/README.md +++ b/hub/docs/README.md @@ -4,7 +4,7 @@ Start with [Deployment](deployment.md) to run Replica Hub or [Development](devel The [shared contributor guides](../../docs/README.md) apply across Replica. This index covers Hub-specific behavior and operations. -- **Operate Hub:** [Deployment](deployment.md), [GitHub App setup](github-app-setup.md), [security](security.md), and [backups](backups-and-restore.md). +- **Operate Hub:** [Deployment](deployment.md), [server-side tracing](deployment.md#optional-server-side-tracing), [GitHub App setup](github-app-setup.md), [security](security.md), and [backups](backups-and-restore.md). - **GitHub and Builds:** [App setup](github-app-setup.md), [access](authentication-and-repository-authorization.md), [discovery](build-discovery.md), [Actions](github-actions-builds.md), and [storage](artifact-storage.md). - **Review:** [Gallery](gallery.md), [previews](preview-serving.md), [collaboration](review-collaboration.md), and [share links](share-links.md). - **Analytics:** [Repository activity events](analytics.md). diff --git a/hub/docs/deployment.md b/hub/docs/deployment.md index f6851d9..70b91e2 100644 --- a/hub/docs/deployment.md +++ b/hub/docs/deployment.md @@ -13,6 +13,78 @@ The recommended deployment needs: The default deployment runs one Replica Hub container and stores its SQLite databases and Build files in one persistent Docker volume. +## Optional server-side tracing + +Hub can send server-side traces to an OpenTelemetry Protocol (OTLP) endpoint. Traces describe Rails requests, Active Job enqueueing and execution, instrumented Active Record operations, and outbound Net::HTTP calls. The published image includes OpenTelemetry instrumentation and its HTTP/protobuf exporter, not vendor-specific APM SDKs. + +Tracing is off by default. To enable it, add these settings to the private deployment `.env` or your container platform's environment: + +```dotenv +OTEL_TRACES_EXPORTER=otlp +OTEL_SERVICE_NAME=replica-hub +OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=production +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel-collector:4318/v1/traces +OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf +``` + +This example assumes a Collector named `otel-collector` is reachable from Hub on a trusted private container network. Hub does not install or start that service. Use HTTPS for endpoints outside a trusted private network. + +Restart every Hub web and job container after changing these settings. No image rebuild is needed. Docker Compose already passes the private `.env` to Hub; other container platforms must pass the variables to every process themselves. + +| Setting | Purpose | +| --- | --- | +| `OTEL_TRACES_EXPORTER` | `otlp` enables export; `none` disables tracing and is Hub's default. Other exporters are not included. | +| `OTEL_SERVICE_NAME` | Service identity; defaults to `replica-hub`. | +| `OTEL_RESOURCE_ATTRIBUTES` | Comma-separated deployment metadata, such as `deployment.environment.name=production,service.version=1.0.0`. Do not put secrets or personal data here. | +| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Complete HTTP(S) trace endpoint, including its required path. Credentials, query strings, and fragments are not accepted in the URL. | +| `OTEL_EXPORTER_OTLP_TRACES_HEADERS` | Authentication headers required by the recipient, in the exporter's comma-separated `key=value` format. Keep this value private. | +| `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` | Only `http/protobuf` is supported. A gRPC-only endpoint needs a Collector or gateway that accepts HTTP/protobuf. | +| `OTEL_TRACES_SAMPLER` | Standard SDK sampling policy, for example `parentbased_traceidratio`. The SDK default is parent-based, with all new root traces sampled. | +| `OTEL_TRACES_SAMPLER_ARG` | Sampling-policy argument; for `parentbased_traceidratio`, a fraction from `0` to `1`, such as `0.1`. | +| `OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE` | Optional path inside the container to a trusted CA certificate for a private endpoint. Mount the certificate when needed; keep TLS verification enabled. | +| `OTEL_SDK_DISABLED` | Set to `true` to disable tracing regardless of the exporter setting. | +| `REPLICA_OTEL_INCLUDE_SENSITIVE_DATA` | Set to `true` to include structured job-argument identity fields listed below. Defaults to `false`; credential-keyed fields remain filtered. | +| `REPLICA_OTEL_TRUST_INCOMING_CONTEXT` | Set to `true` only behind a trusted ingress that removes client-supplied trace headers and supplies its own. Defaults to `false`. | +| `REPLICA_OTEL_UNTRACED_HOSTS` | Optional comma-separated exact hostnames to exclude from outbound Net::HTTP tracing and trace-header injection. Subdomains are not included automatically. Empty entries are rejected. | + +The exporter also accepts the common `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, and `OTEL_EXPORTER_OTLP_PROTOCOL` settings. Trace-specific values take precedence. The common endpoint is a base URL to which the exporter appends `/v1/traces`; the trace-specific endpoint is used as supplied. Enabling export requires an explicit endpoint. Hub rejects unsupported exporters, protocols, and malformed endpoint URLs at startup. + +### Choose a recipient + +Send directly to an APM provider that accepts OTLP over HTTP/protobuf, using its documented trace endpoint and authentication headers. Alternatively, send to a company-managed OpenTelemetry Collector. The Collector can filter, sample, translate, and forward traces to one or more APMs without changing Hub's image. Choose a Collector distribution that includes the exporters your providers need. + +Sentry documents a direct OTLP trace endpoint. Datadog provides a Collector exporter. These paths do not guarantee the same features as each provider's native SDK; check the provider's supported signals, release status, and deployment version. + +The deployment operator controls the destination and credentials. Restrict recipients and network access through your deployment's outbound-network policy. Hub's repository analytics destination settings do not configure or restrict tracing. + +For mutual TLS, the exporter accepts `OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE` and `OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY`, pointing to mounted client certificate and key files. Keep certificate verification enabled. + +### Trace-context boundaries + +Public HTTP requests start new traces by default. Hub ignores their incoming `traceparent`, so a client cannot override the deployment's root sampling decision by supplying a sampled parent. Enable `REPLICA_OTEL_TRUST_INCOMING_CONTEXT` only when all traffic passes through an ingress that sanitizes client-supplied trace headers. This setting trusts every incoming request; it does not authenticate a proxy or enforce a network restriction. + +Hub discards incoming `tracestate` and baggage even with trusted ingress. Jobs continue the context serialized by Hub through its trusted job queue. Protect queue storage and access independently. + +Outbound Net::HTTP calls propagate trace context by default. Use `REPLICA_OTEL_UNTRACED_HOSTS` to exclude specific destinations; those calls also lose their HTTP spans. This setting does not block the requests or restrict telemetry export destinations. Hub fixes its propagation policy in code; `OTEL_PROPAGATORS` does not override these boundaries. + +### Data and delivery limits + +Tracing includes SQL query text, serialized job arguments, and exception type, message, and stack trace by default. Job spans retain queue, job ID, adapter, provider job ID, and priority metadata when the instrumentation supplies them. Hub uses explicit attribute lists matching its bundled instrumentation, including legacy HTTP and code keys that those versions still emit. + +Hub filters structured job arguments with Rails' configured parameter filters. By default it also filters the keys `email`, `name`, `username`, `user_name`, `first_name`, `last_name`, `display_name`, `given_name`, `family_name`, and `login`. Set `REPLICA_OTEL_INCLUDE_SENSITIVE_DATA=true` to include those identity fields. This relaxes only Hub's additional identity filter and Rails' baseline literal `email` rule; other configured filters still apply, including password, token, credential, and authorization rules. Cookie-keyed values remain filtered in either mode. The flag does not attach signed-in user records or enable request bodies or headers. + +Filtering is key-based, not a guarantee that exported data contains no personal information or secrets. SQL text can contain literal values, exception messages and stack traces can contain application data, and positional or free-text job arguments lack field names to filter. SQL bind values are not attached separately. Serialized model arguments may include Global IDs. Configure your APM's access and retention policies accordingly, and apply additional Collector filtering if needed. Operator-supplied resource attributes are exported as configured. + +The deployment operator controls the identity flag through environment configuration. Restart every web and job process when enabling or disabling it. The new setting applies to diagnostics collected by those processes; data already sent remains subject to the recipient's retention policy. + +Hub excludes raw HTTP URL paths and queries, request and response headers and bodies, arbitrary span events, span links, and error status descriptions from export. Exception events retain only `exception.type`, `exception.message`, and `exception.stacktrace`. HTTP span names use methods and route templates instead of concrete URLs. + +Active Job execution continues the enqueueing trace when the job's serialized context is available. The SDK's bounded background queue exports batches outside the request path and resets its worker after process forks. Hub allows up to five seconds to flush at normal process exit. Collector or provider outages are logged and may drop traces; delivery is not durable and does not gate successful Hub requests. Forced termination can lose pending spans. + +Health checks at `/up` are excluded. Asset compilation with `SECRET_KEY_BASE_DUMMY` does not initialize tracing. This integration does not enable browser monitoring, metric or log export, or vendor-native error reporting. Providers that discard OTLP span events may not display the exported exceptions. + +For exporter settings and their formats, see the [Ruby OTLP exporter](https://github.com/open-telemetry/opentelemetry-ruby/tree/main/exporter/otlp) and the [Ruby SDK documentation](https://opentelemetry.io/docs/languages/ruby/). Provider setup examples are available for [Sentry OTLP](https://docs.sentry.io/concepts/otlp/direct/traces/) and the [Datadog Collector exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/datadogexporter). + ## Choose an isolated domain Hub displays prototypes inside its review pages. It requires each preview to use a separate HTTPS subdomain, such as `b123.preview.hub.example.com` under `hub.example.com`. This lets preview cookies work while the prototype is embedded. Hub keeps its own sign-in cookie separate. diff --git a/hub/lib/hub_tracing.rb b/hub/lib/hub_tracing.rb new file mode 100644 index 0000000..654c162 --- /dev/null +++ b/hub/lib/hub_tracing.rb @@ -0,0 +1,107 @@ +require "uri" + +module HubTracing + def self.start! + return if ENV["SECRET_KEY_BASE_DUMMY"] || ENV["OTEL_SDK_DISABLED"]&.casecmp?("true") + + exporter = ENV.fetch("OTEL_TRACES_EXPORTER", "none") + unless %w[none otlp].include?(exporter) + raise ArgumentError, "OTEL_TRACES_EXPORTER must be none or otlp" + end + return if exporter == "none" + + include_sensitive_data = boolean_setting("REPLICA_OTEL_INCLUDE_SENSITIVE_DATA") + trust_incoming_context = boolean_setting("REPLICA_OTEL_TRUST_INCOMING_CONTEXT") + untraced_hosts = ENV.fetch("REPLICA_OTEL_UNTRACED_HOSTS", "").split(",", -1).map(&:strip) + untraced_hosts = [] unless ENV.key?("REPLICA_OTEL_UNTRACED_HOSTS") + if untraced_hosts.any?(&:empty?) + raise ArgumentError, "REPLICA_OTEL_UNTRACED_HOSTS must contain nonempty comma-separated hostnames" + end + + protocol = ENV["OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"] || + ENV.fetch("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + unless protocol == "http/protobuf" + raise ArgumentError, "Hub tracing supports only the http/protobuf OTLP protocol" + end + endpoint = ENV["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] || ENV["OTEL_EXPORTER_OTLP_ENDPOINT"] + begin + uri = URI.parse(endpoint.to_s) + rescue URI::InvalidURIError + raise ArgumentError, "OTLP endpoint must be an absolute HTTP(S) URL" + end + unless %w[http https].include?(uri.scheme) && uri.host && + !uri.userinfo && !uri.query && !uri.fragment + raise ArgumentError, "OTLP endpoint must be an absolute HTTP(S) URL without credentials, query, or fragment" + end + + require "opentelemetry/sdk" + require "opentelemetry/exporter/otlp" + require "opentelemetry/instrumentation/action_pack" + require "opentelemetry/instrumentation/active_job" + require "opentelemetry/instrumentation/net/http" + require_relative "hub_tracing/exporter" + require_relative "hub_tracing/data_filter" + require_relative "hub_tracing/propagator" + + processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new \ + Exporter.new(OpenTelemetry::Exporter::OTLP::Exporter.new) + OpenTelemetry::SDK.configure do |config| + config.service_name = ENV.fetch("OTEL_SERVICE_NAME", "replica-hub") + config.propagators = [ Propagator.new(trust_incoming_context:) ] + config.add_span_processor(processor) + config.use "OpenTelemetry::Instrumentation::Rack", untraced_endpoints: [ "/up" ] + config.use "OpenTelemetry::Instrumentation::ActionPack", span_naming: :class + config.use "OpenTelemetry::Instrumentation::ActiveJob", + propagation_style: :child, span_naming: :job_class + config.use "OpenTelemetry::Instrumentation::Net::HTTP", untraced_hosts: + end + + # Snapshot the original rules before Rails compiles them into combined regular expressions. + Rails.application.config.after_initialize do + data_filter = DataFilter.new(include_sensitive_data:) + ActiveSupport.on_load(:active_job) do + record_arguments = proc do |job| + span = OpenTelemetry::Trace.current_span + next unless span.recording? + + begin + span.set_attribute("replica.job.arguments", data_filter.job_arguments(job.arguments)) + rescue ActiveJob::SerializationError, JSON::GeneratorError + Rails.logger.warn("OpenTelemetry could not serialize job arguments; diagnostic arguments omitted") + end + end + before_enqueue(&record_arguments) + before_perform(&record_arguments) + end + end + + # ORM instrumentation does not cover Hub's direct SQLite calls. + tracer = OpenTelemetry.tracer_provider.tracer("replica-hub") + ActiveSupport::Notifications.subscribe("sql.active_record") do |_, started, finished, _, payload| + next if payload[:name] == "SCHEMA" || payload[:cached] + + span = tracer.start_span \ + "database query", + kind: :client, + start_timestamp: started, + attributes: { "db.system.name" => "sqlite", "db.query.text" => payload[:sql] } + span.status = OpenTelemetry::Trace::Status.error if payload[:exception] + span.record_exception(payload[:exception_object]) if payload[:exception_object] + span.finish(end_timestamp: finished) + end + at_exit do + Timeout.timeout(5) { OpenTelemetry.tracer_provider.shutdown(timeout: 5) } + rescue Timeout::Error + OpenTelemetry.handle_error(message: "OpenTelemetry shutdown exceeded its deadline; pending traces dropped") + end + end + + def self.boolean_setting(name) + value = ENV.fetch(name, "false") + unless %w[true false].include?(value.downcase) + raise ArgumentError, "#{name} must be true or false" + end + + value.casecmp?("true") + end +end diff --git a/hub/lib/hub_tracing/data_filter.rb b/hub/lib/hub_tracing/data_filter.rb new file mode 100644 index 0000000..49d4b32 --- /dev/null +++ b/hub/lib/hub_tracing/data_filter.rb @@ -0,0 +1,21 @@ +module HubTracing + class DataFilter + IDENTITY_KEYS = /\A(?:email|name|username|user_name|first_name|last_name|display_name|given_name|family_name|login)\z/i + + def initialize(include_sensitive_data: false) + filters = Rails.application.config.filter_parameters + if include_sensitive_data + # Only the baseline email rule is relaxed; deployment-added filters still apply. + filters = filters.reject { |filter| filter == :email || filter == "email" } + else + filters = filters + [ IDENTITY_KEYS ] + end + @filter = ActiveSupport::ParameterFilter.new(filters + [ :cookie ]) + end + + def job_arguments(arguments) + serialized = ActiveJob::Arguments.serialize(arguments) + JSON.generate(@filter.filter("arguments" => serialized).fetch("arguments")) + end + end +end diff --git a/hub/lib/hub_tracing/exporter.rb b/hub/lib/hub_tracing/exporter.rb new file mode 100644 index 0000000..986cec5 --- /dev/null +++ b/hub/lib/hub_tracing/exporter.rb @@ -0,0 +1,66 @@ +require "timeout" + +module HubTracing + class Exporter + # Keep this list aligned with the bundled instrumentation and Hub's diagnostic collectors. + ATTRIBUTES = %w[ + http.request.method http.response.status_code http.method http.status_code http.route + url.scheme http.scheme server.address server.port net.peer.name net.peer.port + db.system.name db.query.text + messaging.system messaging.destination messaging.message.id messaging.active_job.adapter.name + messaging.active_job.message.provider_job_id messaging.active_job.message.priority + code.namespace code.function replica.job.arguments + ].freeze + EXCEPTION_ATTRIBUTES = %w[exception.type exception.message exception.stacktrace].freeze + + def initialize(exporter) + @exporter = exporter + end + + def export(spans, timeout: nil) + raise Timeout::Error if timeout && timeout <= 0 + + # The bundled OTLP exporter's retry sleeps can outlast its request timeout. + Timeout.timeout(timeout) do + @exporter.export(spans.map { |span| sanitized(span) }, timeout:) + end + rescue Timeout::Error + OpenTelemetry.handle_error(message: "OTLP export exceeded its deadline; trace batch dropped") + OpenTelemetry::SDK::Trace::Export::TIMEOUT + end + + def shutdown(timeout: nil) + @exporter.shutdown(timeout:) + end + + def force_flush(timeout: nil) + @exporter.force_flush(timeout:) + end + + private + + def sanitized(span) + result = span.dup + result.attributes = (span.attributes || {}).slice(*ATTRIBUTES) + result.events = (span.events || []).filter_map do |event| + next unless event.name == "exception" + + OpenTelemetry::SDK::Trace::Event.new \ + event.name, + (event.attributes || {}).slice(*EXCEPTION_ATTRIBUTES), + event.timestamp + end + result.links = [] + result.status = case span.status.code + when OpenTelemetry::Trace::Status::ERROR then OpenTelemetry::Trace::Status.error + when OpenTelemetry::Trace::Status::OK then OpenTelemetry::Trace::Status.ok + else OpenTelemetry::Trace::Status.unset + end + method = result.attributes["http.request.method"] || result.attributes["http.method"] + if span.kind == :server || method + result.name = [ method || "HTTP", result.attributes["http.route"] ].compact.join(" ") + end + result + end + end +end diff --git a/hub/lib/hub_tracing/propagator.rb b/hub/lib/hub_tracing/propagator.rb new file mode 100644 index 0000000..8bdb82b --- /dev/null +++ b/hub/lib/hub_tracing/propagator.rb @@ -0,0 +1,29 @@ +module HubTracing + class Propagator + def initialize(trust_incoming_context: false) + @trust_incoming_context = trust_incoming_context + @trace_context = OpenTelemetry::Trace::Propagation::TraceContext.text_map_propagator + end + + def extract(carrier, context: OpenTelemetry::Context.current, + getter: OpenTelemetry::Context::Propagation.text_map_getter) + if getter.equal?(OpenTelemetry::Common::Propagation.rack_env_getter) && !@trust_incoming_context + return OpenTelemetry::Trace.context_with_span \ + OpenTelemetry::Trace::Span::INVALID, + parent_context: context + end + + # Vendor state and baggage never cross a Hub boundary, even with trusted ingress. + @trace_context.extract({ "traceparent" => getter.get(carrier, "traceparent") }, context:) + end + + def inject(carrier, context: OpenTelemetry::Context.current, + setter: OpenTelemetry::Context::Propagation.text_map_setter) + @trace_context.inject(carrier, context:, setter:) + end + + def fields + [ "traceparent" ] + end + end +end diff --git a/hub/test/integration/hub_tracing_test.rb b/hub/test/integration/hub_tracing_test.rb new file mode 100644 index 0000000..f35db2d --- /dev/null +++ b/hub/test/integration/hub_tracing_test.rb @@ -0,0 +1,393 @@ +require "test_helper" +require "open3" +require "base64" +require "opentelemetry/exporter/otlp" + +class HubTracingIntegrationTest < ActiveSupport::TestCase + self.use_transactional_tests = false + + test "normal boot isolates inbound context and traces serialized child jobs SQL and HTTP diagnostics" do + result = run_traced_request + + assert_equal 200, result.fetch("status") + assert_equal "completed", result.fetch("body") + assert result.fetch("job_performed") + assert_equal 200, result.fetch("health_status") + refute result.fetch("baggage_propagated"), "incoming baggage reached the outbound service" + refute result.fetch("tracestate_propagated"), "incoming tracestate reached the outbound service" + assert result.fetch("traceparent_propagated"), "ordinary outbound HTTP lost context injection" + payloads = result.fetch("payloads").map { |payload| Base64.strict_decode64(payload) } + assert_not_empty payloads, result.fetch("diagnostics") + payloads.each do |payload| + %w[request-private-canary identity-email-canary identity-name-canary password-canary token-canary].each do |private_value| + refute_includes payload, private_value + end + end + spans = decoded_spans(result) + server = spans.find { |span| span.kind == :SPAN_KIND_SERVER } + assert server, "Rails boot did not produce a server span" + assert_equal "POST /tracing-probe", server.name + refute_equal "1234567890abcdef1234567890abcdef", server.trace_id.unpack1("H*") + assert_empty server.parent_span_id + assert_equal 1, spans.count { |span| span.kind == :SPAN_KIND_SERVER }, "health check was traced" + jobs = spans.select { |span| span.name.include?("TracingProbeJob") } + assert_operator jobs.size, :>=, 2, "enqueue and serialized execution must both be traced" + assert jobs.all? { |span| span.trace_id == server.trace_id }, "job execution lost its request trace" + jobs.each do |job| + assert spans.any? { |parent| parent.span_id == job.parent_span_id }, "job has no exported parent" + end + metadata = spans.flat_map(&:attributes) + assert metadata.any? { |attribute| attribute.key == "db.system.name" } + assert metadata.any? { |attribute| attribute.key == "db.query.text" && attribute.value.string_value.include?("sql-diagnostic-canary") } + jobs.each do |job| + attributes = span_attributes(job) + assert_equal "default", attributes.fetch("messaging.destination").string_value + assert attributes.fetch("messaging.message.id").string_value.present? + assert_equal "test", attributes.fetch("messaging.active_job.adapter.name").string_value + assert_equal "3", attributes.fetch("messaging.active_job.message.priority").string_value + arguments = JSON.parse(attributes.fetch("replica.job.arguments").string_value) + assert_equal "argument-diagnostic-canary", arguments.first + assert_equal "[FILTERED]", arguments.last.fetch("email") + assert_equal "[FILTERED]", arguments.last.fetch("name") + assert_equal "[FILTERED]", arguments.last.fetch("password") + assert_equal "[FILTERED]", arguments.last.fetch("token") + assert_equal "ActiveJob::Serializers::DateSerializer", arguments.last.fetch("on").fetch("_aj_serialized") + end + assert spans.any? { |span| span.kind == :SPAN_KIND_CLIENT }, "outbound HTTP was not traced" + spans.each do |span| + assert_empty span.events + assert_empty span.status.message if span.has_status? + end + end + + test "trusted inbound context retains sampled parent but discards tracestate and baggage" do + result = run_traced_request(configuration: { + "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "TrUe", + "OTEL_TRACES_SAMPLER" => "parentbased_traceidratio", + "OTEL_TRACES_SAMPLER_ARG" => "0" + }) + + server = decoded_spans(result).find { |span| span.kind == :SPAN_KIND_SERVER } + assert server, result.fetch("diagnostics") + assert_equal "1234567890abcdef1234567890abcdef", server.trace_id.unpack1("H*") + assert_equal "1234567890abcdef", server.parent_span_id.unpack1("H*") + refute result.fetch("baggage_propagated") + refute result.fetch("tracestate_propagated") + assert result.fetch("traceparent_propagated") + assert decoded_spans(result).all? { |span| span.trace_state.empty? } + end + + test "public inbound sampled context cannot override zero sampling" do + result = run_traced_request(configuration: { + "OTEL_TRACES_SAMPLER" => "parentbased_traceidratio", + "OTEL_TRACES_SAMPLER_ARG" => "0" + }) + + assert_equal 200, result.fetch("status") + assert result.fetch("job_performed") + assert_empty decoded_spans(result), "an untrusted sampled parent bypassed the configured sampler" + end + + test "explicit false keeps identity filtering and incoming context isolation" do + result = run_traced_request(configuration: { + "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "FaLsE", + "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "FaLsE" + }) + + server = decoded_spans(result).find { |span| span.kind == :SPAN_KIND_SERVER } + assert server, result.fetch("diagnostics") + refute_equal "1234567890abcdef1234567890abcdef", server.trace_id.unpack1("H*") + refute_includes result.fetch("payloads").map { |payload| Base64.strict_decode64(payload) }.join, "identity-email-canary" + end + + test "sensitive opt in permits structured identities but not credentials" do + result = run_traced_request(configuration: { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "TrUe" }) + + jobs = decoded_spans(result).select { |span| span.name.include?("TracingProbeJob") } + assert_operator jobs.size, :>=, 2 + jobs.each do |job| + arguments = JSON.parse(span_attributes(job).fetch("replica.job.arguments").string_value).last + assert_equal "identity-email-canary@example.test", arguments.fetch("email"), "#{job.name} did not honor identity opt in" + assert_equal "identity-name-canary", arguments.fetch("name") + assert_equal "[FILTERED]", arguments.fetch("password") + assert_equal "[FILTERED]", arguments.fetch("token") + assert_equal "[FILTERED]", arguments.fetch("customer_secret") + end + assert result.fetch("job_performed"), "diagnostic filtering changed the arguments delivered to the job" + end + + test "sensitive collection setting remains fixed until restart" do + [ [ "true", "false", "identity-email-canary@example.test" ], [ "false", "true", "[FILTERED]" ] ].each do |startup, changed, expected| + result = run_traced_request( + configuration: { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => startup }, + change_sensitive_setting: changed + ) + jobs = decoded_spans(result).select { |span| span.name.include?("TracingProbeJob") } + assert_operator jobs.size, :>=, 2 + jobs.each do |job| + arguments = JSON.parse(span_attributes(job).fetch("replica.job.arguments").string_value).last + assert_equal expected, arguments.fetch("email"), "runtime ENV change overrode startup value #{startup}" + end + end + end + + test "untraced exact hosts suppress HTTP spans and propagation without suppressing other hosts or jobs" do + result = run_traced_request(configuration: { + "REPLICA_OTEL_UNTRACED_HOSTS" => " upstream.example, other.example " + }) + + refute result.fetch("traceparent_propagated"), "untraced host received trace context" + refute result.fetch("tracestate_propagated") + refute result.fetch("baggage_propagated") + assert result.fetch("other_traceparent_propagated"), "hostname suffix incorrectly suppressed another host" + spans = decoded_spans(result) + http = spans.select { |span| span_attributes(span).key?("http.method") || span_attributes(span).key?("http.request.method") } + refute http.any? { |span| span.kind == :SPAN_KIND_CLIENT && span_attributes(span).values.any? { |value| value.string_value == "upstream.example" } } + assert http.any? { |span| span.kind == :SPAN_KIND_CLIENT && span_attributes(span).values.any? { |value| value.string_value == "not-upstream.example" } } + server = spans.find { |span| span.kind == :SPAN_KIND_SERVER } + jobs = spans.select { |span| span.name.include?("TracingProbeJob") } + assert_operator jobs.size, :>=, 2 + assert jobs.all? { |span| span.trace_id == server.trace_id } + end + + test "job and SQL failures retain useful exception and query diagnostics" do + result = run_traced_request(failing_job: true) + + assert_equal 200, result.fetch("status") + spans = decoded_spans(result) + job = spans.find { |span| span.name.include?("TracingProbeJob") && span.events.any? } + assert job, "job failure has no exported exception event" + assert_equal :STATUS_CODE_ERROR, job.status.code + assert_empty job.status.message + exception = job.events.find { |event| event.name == "exception" } + assert exception, "job failure did not preserve the exception event" + attributes = exception.attributes.to_h { |attribute| [ attribute.key, attribute.value.string_value ] } + assert_equal "RuntimeError", attributes.fetch("exception.type") + assert_includes attributes.fetch("exception.message"), "job-failure-diagnostic-canary" + assert attributes.fetch("exception.stacktrace").present? + assert_equal %w[exception.message exception.stacktrace exception.type], attributes.keys.sort + query = spans.find do |span| + span_attributes(span)["db.query.text"]&.string_value&.include?("missing_diagnostic_table") + end + assert query, "failed SQL query was not exported" + assert_equal :STATUS_CODE_ERROR, query.status.code + assert spans.flat_map(&:attributes).none? { |attribute| attribute.key.match?(/bind|parameter/) } + assert spans.flat_map(&:events).all? { |event| event.name == "exception" } + end + + test "an unavailable collector does not interrupt a request or serialized job and flush is bounded" do + result = run_traced_request(collector_failure: true) + + assert_equal 200, result.fetch("status") + assert_equal "completed", result.fetch("body") + assert result.fetch("job_performed") + assert_operator result.fetch("export_attempts"), :>, 0 + assert_operator result.fetch("request_seconds"), :<, 5 + assert_operator result.fetch("flush_seconds"), :<, 5 + end + + test "a forked worker exports new job spans without replaying its parents pending request" do + result = run_traced_request(fork_worker: true) + + assert_equal 0, result.fetch("worker_exitstatus") + assert result.fetch("worker_performed") + spans = result.fetch("worker_payloads").flat_map do |payload| + request = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode \ + Base64.strict_decode64(payload) + request.resource_spans.flat_map(&:scope_spans).flat_map(&:spans) + end + assert spans.any? { |span| span.name.include?("TracingProbeJob") }, "forked worker did not export" + refute spans.any? { |span| span.kind == :SPAN_KIND_SERVER }, "child re-exported its parent's request" + spans.each { |span| assert_empty span.events } + end + + private + + def decoded_spans(result) + result.fetch("payloads").flat_map do |payload| + request = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode \ + Base64.strict_decode64(payload) + request.resource_spans.flat_map(&:scope_spans).flat_map(&:spans) + end + end + + def span_attributes(span) + span.attributes.to_h { |attribute| [ attribute.key, attribute.value ] } + end + + def run_traced_request(collector_failure: false, fork_worker: false, configuration: {}, failing_job: false, change_sensitive_setting: nil) + environment = ENV.keys.grep(/\A(?:OTEL_|REPLICA_OTEL_)/).to_h { |key| [ key, nil ] } + environment.merge! \ + "SECRET_KEY_BASE_DUMMY" => nil, + "RAILS_ENV" => "test", + "OTEL_TRACES_EXPORTER" => "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT" => "http://collector.example/v1/traces", + "OTEL_EXPORTER_OTLP_TIMEOUT" => "1", + "OTEL_BSP_SCHEDULE_DELAY" => "60000", + "OTEL_BSP_EXPORT_TIMEOUT" => "1000" + environment.merge!(configuration) + script = <<~RUBY + require "webmock" + require "json" + require "base64" + WebMock.enable! + WebMock.disable_net_connect! + require_relative "config/environment" + Rails.application.config.filter_parameters += [ :customer_secret ] + ENV["REPLICA_OTEL_INCLUDE_SENSITIVE_DATA"] = #{change_sensitive_setting.inspect} unless #{change_sensitive_setting.nil?} + payloads = [] + export_attempts = 0 + baggage_propagated = false + tracestate_propagated = false + traceparent_propagated = false + other_traceparent_propagated = false + intake = WebMock.stub_request(:post, "http://collector.example/v1/traces") + if #{collector_failure} + intake.to_return do + export_attempts += 1 + raise Errno::ECONNREFUSED + end + else + intake.to_return do |request| + export_attempts += 1 + payloads << Base64.strict_encode64(request.body) + { status: 200, body: "" } + end + end + WebMock.stub_request(:get, "http://upstream.example/request-private-canary?token=request-private-canary") + .to_return do |request| + baggage_propagated ||= request.headers.key?("Baggage") + tracestate_propagated ||= request.headers.key?("Tracestate") + traceparent_propagated ||= request.headers.key?("Traceparent") + { status: 200, body: "request-private-canary" } + end + WebMock.stub_request(:get, "http://not-upstream.example/") + .to_return do |request| + other_traceparent_propagated ||= request.headers.key?("Traceparent") + { status: 200, body: "" } + end + class TracingProbeJob < ActiveJob::Base + def perform(diagnostic, values) + raise "job argument changed" unless diagnostic == "argument-diagnostic-canary" && + values.fetch("email") == "identity-email-canary@example.test" && + values.fetch("name") == "identity-name-canary" && + values.fetch("password") == "password-canary" && + values.fetch("token") == "token-canary" && + values.fetch("on") == Date.new(2026, 10, 19) + raise "job-failure-diagnostic-canary" if #{failing_job} + self.class.performed = true + end + class_attribute :performed, default: false + end + TracingProbeJob.queue_adapter = :test + class TracingProbeController < ActionController::Base + def create + ActiveRecord::Base.connection.select_value("SELECT 'sql-diagnostic-canary'") + if #{failing_job} + begin + ActiveRecord::Base.connection.select_value("SELECT * FROM missing_diagnostic_table") + rescue ActiveRecord::StatementInvalid + end + end + Net::HTTP.get(URI("http://upstream.example/request-private-canary?token=request-private-canary")) + Net::HTTP.get(URI("http://not-upstream.example/")) + TracingProbeJob.set(priority: 3).perform_later("argument-diagnostic-canary", { + "email" => "identity-email-canary@example.test", + "name" => "identity-name-canary", + "password" => "password-canary", + "token" => "token-canary", + "customer_secret" => "app-secret-canary", + "on" => Date.new(2026, 10, 19) + }) + queued = TracingProbeJob.queue_adapter.enqueued_jobs.shift.select { |key, _| key.is_a?(String) } + serialized = JSON.parse(JSON.generate(queued)) + begin + ActiveJob::Base.execute(serialized) + rescue RuntimeError => error + raise unless #{failing_job} && error.message == "job-failure-diagnostic-canary" + end + render plain: "completed" + end + end + Rails.application.routes.draw do + post "/tracing-probe", to: "tracing_probe#create" + get "/up", to: "rails/health#show" + end + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + request = Rack::MockRequest.env_for \ + "/tracing-probe?token=request-private-canary", + method: "POST", + "HTTP_AUTHORIZATION" => "Bearer private-value-canary", + "HTTP_BAGGAGE" => "secret=request-private-canary", + "HTTP_TRACEPARENT" => "00-1234567890abcdef1234567890abcdef-1234567890abcdef-01", + "HTTP_TRACESTATE" => "vendor=request-private-canary", + input: "request-private-canary", + "CONTENT_TYPE" => "text/plain" + status, headers, body = Rails.application.call(request) + response_body = body.each.to_a.join + body.close if body.respond_to?(:close) + request_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + health_status, _, health_body = Rails.application.call(Rack::MockRequest.env_for("/up")) + health_body.each { |_part| } + health_body.close if health_body.respond_to?(:close) + worker = {} + if #{fork_worker} + reader, writer = IO.pipe + pid = fork do + reader.close + payloads.clear + TracingProbeJob.performed = false + TracingProbeJob.perform_now("argument-diagnostic-canary", { + "email" => "identity-email-canary@example.test", + "name" => "identity-name-canary", + "password" => "password-canary", + "token" => "token-canary", + "on" => Date.new(2026, 10, 19) + }) + OpenTelemetry.tracer_provider.force_flush(timeout: 1) + OpenTelemetry.tracer_provider.shutdown(timeout: 1) + writer.write(JSON.generate(payloads: payloads, performed: TracingProbeJob.performed)) + writer.close + exit! 0 + end + writer.close + worker = JSON.parse(reader.read) + reader.close + _, worker_status = Process.wait2(pid) + worker["exitstatus"] = worker_status.exitstatus + end + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + OpenTelemetry.tracer_provider.force_flush(timeout: 1) + flush_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + result = JSON.generate \ + status: status, + body: response_body, + job_performed: TracingProbeJob.performed, + health_status: health_status, + baggage_propagated: baggage_propagated, + tracestate_propagated: tracestate_propagated, + traceparent_propagated: traceparent_propagated, + other_traceparent_propagated: other_traceparent_propagated, + payloads: payloads, + export_attempts: export_attempts, + request_seconds: request_seconds, + flush_seconds: flush_seconds, + worker_payloads: worker["payloads"], + worker_performed: worker["performed"], + worker_exitstatus: worker["exitstatus"] + puts "HUB_TRACING_RESULT=" + result + RUBY + output, error, status = Open3.capture3 \ + environment, + RbConfig.ruby, + "-rbundler/setup", + "-e", + script, + chdir: Rails.root + assert status.success?, "Tracing subprocess failed:\n#{output}\n#{error}" + result = output.lines.find { |line| line.start_with?("HUB_TRACING_RESULT=") } + assert result, "Tracing subprocess returned no result:\n#{output}\n#{error}" + JSON.parse(result.delete_prefix("HUB_TRACING_RESULT=")) + .merge("diagnostics" => "#{output}\n#{error}") + end +end diff --git a/hub/test/lib/hub_tracing_data_filter_test.rb b/hub/test/lib/hub_tracing_data_filter_test.rb new file mode 100644 index 0000000..a90754e --- /dev/null +++ b/hub/test/lib/hub_tracing_data_filter_test.rb @@ -0,0 +1,109 @@ +require "test_helper" +require "json" +require_relative "../../lib/hub_tracing/data_filter" + +class HubTracingDataFilterTest < ActiveSupport::TestCase + test "default argument filtering recursively removes structured identity and credential values without mutation" do + identities = %w[email name username user_name first_name last_name display_name given_name family_name login] + identity_values = identities.to_h { |key| [ key, "#{key}-identity-canary" ] } + arguments = [ + { + "nested" => [ identity_values ], + "password" => "password-canary", + "access_token" => "token-canary", + "operation" => "publish" + }, + "positional-diagnostic-canary" + ] + original = Marshal.load(Marshal.dump(arguments)) + + result = JSON.parse(HubTracing::DataFilter.new.job_arguments(arguments)) + + assert_equal original, arguments + identities.each { |key| assert_equal "[FILTERED]", result.first.fetch("nested").first.fetch(key) } + assert_equal "[FILTERED]", result.first.fetch("password") + assert_equal "[FILTERED]", result.first.fetch("access_token") + assert_equal "publish", result.first.fetch("operation") + assert_equal "positional-diagnostic-canary", result.last + end + + test "sensitive opt in retains identities while preserving additional Rails literal regex and proc filters" do + original_filters = Rails.application.config.filter_parameters + Rails.application.config.filter_parameters = [ + :email, "email", :password, :token, :customer_secret, /private_email/, + ->(key, value) { value.replace("[APP FILTERED]") if key == "internal_note" } + ] + filter = HubTracing::DataFilter.new(include_sensitive_data: true) + arguments = [ + { + "email" => "person@example.test", + "name" => "Example Person", + "login" => "example-person", + "password" => "password-canary", + "token" => "token-canary", + "customer_secret" => "app-secret-canary", + "private_email" => "private@example.test", + "internal_note" => "note-canary" + } + ] + original = Marshal.load(Marshal.dump(arguments)) + + result = JSON.parse(filter.job_arguments(arguments)).first + + assert_equal "person@example.test", result.fetch("email") + assert_equal "Example Person", result.fetch("name") + assert_equal "example-person", result.fetch("login") + %w[password token customer_secret private_email].each do |key| + assert_equal "[FILTERED]", result.fetch(key) + end + assert_equal "[APP FILTERED]", result.fetch("internal_note") + assert_equal original, arguments + ensure + Rails.application.config.filter_parameters = original_filters + end + + test "constructed filters retain identity policy and credentials after Rails replaces its source with precompiled rules" do + original_filters = Rails.application.config.filter_parameters + Rails.application.config.filter_parameters = [ :email, :password, :token, :customer_secret ] + default_filter = HubTracing::DataFilter.new + opted_in_filter = HubTracing::DataFilter.new(include_sensitive_data: true) + source = Rails.application.config.filter_parameters + source.replace(ActiveSupport::ParameterFilter.precompile_filters(source)) + arguments = [ + { + "email" => "person@example.test", + "name" => "Example Person", + "password" => "password-canary", + "token" => "token-canary", + "customer_secret" => "app-secret-canary" + } + ] + + default = JSON.parse(default_filter.job_arguments(arguments)).first + opted_in = JSON.parse(opted_in_filter.job_arguments(arguments)).first + + assert_equal "[FILTERED]", default.fetch("email") + assert_equal "[FILTERED]", default.fetch("name") + assert_equal "person@example.test", opted_in.fetch("email") + assert_equal "Example Person", opted_in.fetch("name") + [ default, opted_in ].each do |result| + %w[password token customer_secret].each { |key| assert_equal "[FILTERED]", result.fetch(key) } + end + ensure + Rails.application.config.filter_parameters = original_filters + end + + test "sensitive opt in does not remove an application rule for another identity field" do + original_filters = Rails.application.config.filter_parameters + Rails.application.config.filter_parameters = [ :email, :name, /email/ ] + arguments = [ { "email" => "person@example.test", "name" => "Example Person", "login" => "example-person" } ] + + result = JSON.parse(HubTracing::DataFilter.new(include_sensitive_data: true).job_arguments(arguments)).first + + assert_equal "[FILTERED]", result.fetch("email") + assert_equal "[FILTERED]", result.fetch("name") + assert_equal "example-person", result.fetch("login") + ensure + Rails.application.config.filter_parameters = original_filters + end +end diff --git a/hub/test/lib/hub_tracing_test.rb b/hub/test/lib/hub_tracing_test.rb new file mode 100644 index 0000000..46339a6 --- /dev/null +++ b/hub/test/lib/hub_tracing_test.rb @@ -0,0 +1,234 @@ +require "test_helper" +require "open3" +require "opentelemetry/sdk" +require "opentelemetry/exporter/otlp" +require "opentelemetry/sdk/trace/export/in_memory_span_exporter" +require_relative "../../lib/hub_tracing/exporter" + +class HubTracingTest < ActiveSupport::TestCase + self.use_transactional_tests = false + + test "default disabled and dummy builds do not load the tracing SDK" do + [ + {}, + { "OTEL_TRACES_EXPORTER" => "none" }, + { "OTEL_TRACES_EXPORTER" => "otlp", "OTEL_SDK_DISABLED" => "true" }, + { "OTEL_TRACES_EXPORTER" => "otlp", "SECRET_KEY_BASE_DUMMY" => "1" }, + { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "invalid", "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "invalid" } + ].each do |configuration| + output, error, status = run_startup(configuration, <<~RUBY) + HubTracing.start! + abort "SDK initialized while disabled" if defined?(OpenTelemetry::SDK) + puts "disabled" + RUBY + + assert status.success?, error + assert_equal "disabled", output.strip + end + end + + test "invalid enabled configuration fails without printing endpoint credentials" do + [ + { "OTEL_EXPORTER_OTLP_ENDPOINT" => nil }, + { "OTEL_EXPORTER_OTLP_ENDPOINT" => "https://user:credential-canary@collector.example" }, + { "OTEL_EXPORTER_OTLP_ENDPOINT" => "https://collector.example?token=credential-canary" }, + { "OTEL_EXPORTER_OTLP_ENDPOINT" => "https://collector.example/#credential-canary" }, + { "OTEL_EXPORTER_OTLP_ENDPOINT" => "https://[credential-canary" }, + { "OTEL_EXPORTER_OTLP_ENDPOINT" => "file:///credential-canary" }, + { "OTEL_EXPORTER_OTLP_PROTOCOL" => "grpc" }, + { "OTEL_TRACES_EXPORTER" => "console,otlp" }, + { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "credential-canary" }, + { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "" }, + { "REPLICA_OTEL_INCLUDE_SENSITIVE_DATA" => "1" }, + { "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "credential-canary" }, + { "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "" }, + { "REPLICA_OTEL_TRUST_INCOMING_CONTEXT" => "yes" }, + { "REPLICA_OTEL_UNTRACED_HOSTS" => "upstream.example," }, + { "REPLICA_OTEL_UNTRACED_HOSTS" => ",upstream.example" }, + { "REPLICA_OTEL_UNTRACED_HOSTS" => "upstream.example, ,other.example" } + ].each do |invalid| + configuration = { + "OTEL_TRACES_EXPORTER" => "otlp", + "OTEL_EXPORTER_OTLP_ENDPOINT" => "http://collector.example" + }.merge(invalid) + output, error, status = run_startup(configuration, <<~RUBY) + begin + HubTracing.start! + rescue ArgumentError => error + puts error.message + exit 42 + end + RUBY + + assert_equal 42, status.exitstatus, "#{configuration.keys}: #{output} #{error}" + refute_includes output + error, "credential-canary" + end + end + + test "export deadline interrupts a sleeping exporter even when it rescues ordinary errors" do + delegate = Object.new + entered = false + completed = false + delegate.define_singleton_method(:export) do |_, timeout:| + entered = true + sleep 0.25 + completed = true + OpenTelemetry::SDK::Trace::Export::SUCCESS + rescue StandardError + OpenTelemetry::SDK::Trace::Export::SUCCESS + end + exporter = HubTracing::Exporter.new(delegate) + + assert_equal OpenTelemetry::SDK::Trace::Export::TIMEOUT, exporter.export([], timeout: 0.01) + assert entered, "delegate was never called" + refute completed, "delegate continued beyond the export deadline" + end + + test "expired export deadlines do not send spans" do + delegate = Object.new + calls = 0 + delegate.define_singleton_method(:export) do |_, timeout:| + calls += 1 + OpenTelemetry::SDK::Trace::Export::SUCCESS + end + exporter = HubTracing::Exporter.new(delegate) + + [ 0, -1 ].each do |timeout| + assert_equal OpenTelemetry::SDK::Trace::Export::TIMEOUT, exporter.export([], timeout:) + end + assert_equal 0, calls + end + + test "serialized OTLP preserves emitted metadata and exception diagnostics but removes unrelated private content" do + memory = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new + provider = OpenTelemetry::SDK::Trace::TracerProvider.new + provider.add_span_processor(OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(memory)) + tracer = provider.tracer("hub-tracing-test") + sensitive = "private-value-canary" + attributes = { + "http.request.method" => "POST", + "http.response.status_code" => 500, + "http.route" => "/builds/:id", + "http.method" => "POST", + "http.status_code" => 500, + "url.scheme" => "https", + "http.scheme" => "https", + "server.address" => "example.com", + "server.port" => 443, + "net.peer.name" => "example.com", + "net.peer.port" => 443, + "db.system.name" => "sqlite", + "db.query.text" => "SELECT 'sql-diagnostic-canary'", + "messaging.system" => "active_job", + "messaging.destination" => "default", + "messaging.message.id" => "job-id", + "messaging.active_job.adapter.name" => "ActiveJob::QueueAdapters::TestAdapter", + "messaging.active_job.message.provider_job_id" => "provider-id", + "messaging.active_job.message.priority" => "3", + "code.namespace" => "TracingProbeJob", + "code.function" => "perform", + "replica.job.arguments" => '["argument-diagnostic-canary"]', + "http.request.header.authorization" => "Bearer #{sensitive}", + "http.request.body" => sensitive, + "url.full" => "https://user:#{sensitive}@example.com/#{sensitive}?token=#{sensitive}", + "db.statement" => "SELECT '#{sensitive}'", + "db.query.parameter.0" => sensitive, + "job.arguments" => sensitive, + "db.system" => sensitive, + "db.operation" => sensitive, + "db.operation.name" => sensitive, + "messaging.operation" => sensitive, + "messaging.operation.name" => sensitive, + "messaging.operation.type" => sensitive, + "messaging.destination.name" => sensitive, + "messaging.destination_kind" => sensitive + } + link = OpenTelemetry::Trace::Link.new \ + OpenTelemetry::Trace::SpanContext.new, + { "secret" => sensitive } + span = tracer.start_span("POST /#{sensitive}", kind: :server, attributes:, links: [ link ]) + span.add_event(sensitive, attributes: { "exception.message" => sensitive }) + span.add_event \ + "exception", + attributes: { + "exception.type" => "RuntimeError", + "exception.message" => "exception-diagnostic-canary", + "exception.stacktrace" => "probe.rb:1:in `perform'", + "exception.escaped" => true, + "request.body" => sensitive + } + span.status = OpenTelemetry::Trace::Status.error(sensitive) + span.finish + client = tracer.start_span \ + "GET https://example.com/#{sensitive}", + kind: :client, + attributes: attributes.merge("http.request.method" => "GET") + client.finish + originals = memory.finished_spans + requests = [] + intake = stub_request(:post, "http://collector.example/v1/traces").with do |request| + requests << request + true + end.to_return(status: 200, body: "") + exporter = HubTracing::Exporter.new \ + OpenTelemetry::Exporter::OTLP::Exporter.new(endpoint: "http://collector.example/v1/traces") + + assert_equal OpenTelemetry::SDK::Trace::Export::SUCCESS, exporter.export(originals, timeout: 1) + + assert_requested intake, times: 1 + assert_equal "application/x-protobuf", requests.first.headers["Content-Type"] + payload = requests.first.body + refute_includes payload, sensitive + decoded = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(payload) + exported = decoded.resource_spans.flat_map(&:scope_spans).flat_map(&:spans) + assert_equal [ "POST /builds/:id", "GET /builds/:id" ], exported.map(&:name) + exported.zip(originals).each do |actual, original| + assert_equal original.trace_id, actual.trace_id + assert_equal original.span_id, actual.span_id + assert_equal original.start_timestamp, actual.start_time_unix_nano + assert_equal original.end_timestamp, actual.end_time_unix_nano + assert_empty actual.links + metadata = actual.attributes.to_h { |attribute| [ attribute.key, attribute.value ] } + attributes.reject { |_, value| value.to_s.include?(sensitive) } + .merge("http.request.method" => original.attributes["http.request.method"]).each do |key, value| + assert metadata.key?(key), "missing emitted metadata #{key}" + assert_equal value, metadata.fetch(key).public_send(value.is_a?(Integer) ? :int_value : :string_value) + end + end + assert_equal [ "exception" ], exported.first.events.map(&:name) + event = exported.first.events.first + assert_equal originals.first.events.last.timestamp, event.time_unix_nano + expected_exception = { + "exception.type" => "RuntimeError", + "exception.message" => "exception-diagnostic-canary", + "exception.stacktrace" => "probe.rb:1:in `perform'" + } + assert_equal expected_exception, event.attributes.to_h { |attribute| [ attribute.key, attribute.value.string_value ] } + assert_empty exported.last.events + assert_equal :STATUS_CODE_ERROR, exported.first.status.code + assert_empty exported.first.status.message + assert_equal "POST /#{sensitive}", originals.first.name + assert_equal sensitive, originals.first.events.first.name + assert_equal sensitive, originals.first.status.description + ensure + exporter&.shutdown(timeout: 1) + provider&.shutdown(timeout: 1) + end + + private + + def run_startup(configuration, script) + environment = ENV.keys.grep(/\A(?:OTEL_|REPLICA_OTEL_)/).to_h { |key| [ key, nil ] } + .merge("SECRET_KEY_BASE_DUMMY" => nil) + .merge(configuration) + Open3.capture3 \ + environment, + RbConfig.ruby, + "-rbundler/setup", + "-Ilib", + "-rhub_tracing", + "-e", + script, + chdir: Rails.root + end +end