diff --git a/docs/execute/build.md b/docs/execute/build.md index 28017a63bab..4e4f4684ea9 100644 --- a/docs/execute/build.md +++ b/docs/execute/build.md @@ -33,6 +33,7 @@ Build images used for system tests. * `php` * `python` * `ruby` +* `rust` `cpp` is not available for `build.sh` because only parametric tests are runnable for `dd-trace-cpp`. @@ -46,6 +47,7 @@ Build images used for system tests. * For `php`: `apache-mod-8.1`, `apache-mod-8.0` (default), `apache-mod-7.4`, `apache-mod-7.3`, `apache-mod-7.2`, `apache-mod-7.1`, `apache-mod-7.0`, `apache-mod-8.1-zts`, `apache-mod-8.0-zts`, `apache-mod-7.4-zts`, `apache-mod-7.3-zts`, `apache-mod-7.2-zts`, `apache-mod-7.1-zts`, `apache-mod-7.0-zts`, `php-fpm-8.1`, `php-fpm-8.0`, `php-fpm-7.4`, `php-fpm-7.3`, `php-fpm-7.2`, `php-fpm-7.1`, `php-fpm-7.0` * For `python`: `flask-poc` (default), `fastapi`, `uwsgi-poc`, `django-poc`, `python3.12` * For `ruby`: `rails70` (default), `rack`, `sinatra21`, and lot of other sinatra/rails versions +* For `rust`: `axum` (default) ## Real life examples diff --git a/manifests/php.yml b/manifests/php.yml index af3b3083c57..b29cb80aef5 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -697,7 +697,7 @@ manifest: tests/k8s_lib_injection/test_k8s_lib_injection_profiling.py::TestK8sLibInjectioProfilingClusterOverride: v1.9.0 tests/k8s_lib_injection/test_k8s_lib_injection_profiling.py::TestK8sLibInjectioProfilingDisabledByDefault: v1.9.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: incomplete_test_app (endpoint not implemented) - tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: v1.21.0-dev tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 7aca11f90ff..bcbf76e3fcd 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2048,7 +2048,7 @@ manifest: "*": incomplete_test_app (endpoint not implemented) rails72: v2.0.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api::test_propagation_extract: incomplete_test_app (Ruby extract seems to fail even though it should be supported) - tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: v2.36.0-dev tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/manifests/rust.yml b/manifests/rust.yml index 7380f086c6a..750314cc46a 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -54,7 +54,7 @@ manifest: tests/integrations/test_mongo.py::Test_Mongo: missing_feature (Endpoint is not implemented on weblog) tests/integrations/test_service_overrides.py::Test_SqlServiceNameSource: irrelevant (Only implemented for Java) tests/integrations/test_sql.py::Test_Sql: missing_feature (Endpoint is not implemented on weblog) - tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: v0.3.4-dev tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/utils/build/docker/rust/axum.Dockerfile b/utils/build/docker/rust/axum.Dockerfile new file mode 100644 index 00000000000..dadf619667b --- /dev/null +++ b/utils/build/docker/rust/axum.Dockerfile @@ -0,0 +1,37 @@ +FROM rust:1.87-slim-bookworm AS builder +WORKDIR /usr/app + +# Bring in the weblog sources. +COPY utils/build/docker/rust/weblog . + +# Bring in the binaries folder, which may contain a dd-trace-rs checkout and/or a rust-load-from-git file. +COPY binaries/ /binaries/ + +RUN apt-get update && apt-get install -y --no-install-recommends openssh-client git jq build-essential perl libssl-dev pkg-config curl + +# Resolves the dd-trace-rs source into /binaries/dd-trace-rs and stamps a -dev version (trace-only, +# no gRPC features) so this builds against the feature branch on the rust:1.87 toolchain. +RUN bash ./install_ddtrace.sh + +RUN \ + --mount=type=cache,target=/usr/app/target/ \ + --mount=type=cache,target=/usr/local/cargo/registry/ \ + cargo build --release && cp ./target/release/weblog /usr/app/weblog + +# Resolve the tracer version from the lockfile so the weblog can report it on /healthcheck. +RUN ./system_tests_library_version.sh > /usr/app/SYSTEM_TESTS_LIBRARY_VERSION + +FROM debian:bookworm-slim AS final +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /usr/app/weblog /usr/app/weblog +COPY --from=builder /usr/app/Cargo.lock /usr/app/Cargo.lock +COPY --from=builder /usr/app/SYSTEM_TESTS_LIBRARY_VERSION /usr/app/SYSTEM_TESTS_LIBRARY_VERSION + +WORKDIR /usr/app + +# The harness invokes ./app.sh; export the resolved tracer version so /healthcheck can return it. +RUN printf '#!/bin/bash\nexport SYSTEM_TESTS_LIBRARY_VERSION="$(cat /usr/app/SYSTEM_TESTS_LIBRARY_VERSION)"\nexec ./weblog\n' > app.sh \ + && chmod +x app.sh + +CMD ["./app.sh"] diff --git a/utils/build/docker/rust/weblog/.gitignore b/utils/build/docker/rust/weblog/.gitignore new file mode 100644 index 00000000000..eb5a316cbd1 --- /dev/null +++ b/utils/build/docker/rust/weblog/.gitignore @@ -0,0 +1 @@ +target diff --git a/utils/build/docker/rust/weblog/Cargo.toml b/utils/build/docker/rust/weblog/Cargo.toml new file mode 100644 index 00000000000..045424dfc64 --- /dev/null +++ b/utils/build/docker/rust/weblog/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "weblog" +version = "0.0.1" +description = "Dockerized Rust/Axum based HTTP weblog for end-to-end System tests" +edition = "2021" +license = "Apache-2.0" +publish = false + +rust-version = "1.87" # should match Dockerfile rustc version + +[[bin]] +name = "weblog" +path = "src/main.rs" +doc = false +bench = false + +[dependencies] +anyhow = { version = "1.0" } +axum = { version = "0.8.4", features = [ "http1", "json", "query" ] } +hyper = { version = "1.0", features = ["client", "http1"] } +hyper-util = { version = "0.1", features = ["tokio", "client", "client-legacy", "http1"] } +http-body-util = { version = "0.1" } +reqwest = { version = "0.12", default-features = false, features = ["json"] } +serde = { version = "1.0", features = [ "derive" ] } +serde_json = { version = "1.0" } +tokio = { version = "1", features = [ "macros", "rt-multi-thread", "signal", "net" ] } +tracing = { version = "0.1", default-features = false } +tracing-subscriber = { version = "0.3", default-features = false, features = [ "env-filter", "fmt" ] } +opentelemetry = { version = "0.31.0" } +opentelemetry_sdk = { version = "0.31.0" } +opentelemetry-http = { version = "0.31.0" } +opentelemetry-semantic-conventions = { version = "0.31.0", features = [ "semconv_experimental" ] } + +# Trace-only OTLP export: default features (no metrics/logs gRPC), which keeps the dependency tree +# off tonic and lets this build on the rust:1.87 base image. The path points at the dd-trace-rs +# checkout placed in /binaries by the weblog build (see axum.Dockerfile). +datadog-opentelemetry = { path = "/binaries/dd-trace-rs/datadog-opentelemetry" } diff --git a/utils/build/docker/rust/weblog/install_ddtrace.sh b/utils/build/docker/rust/weblog/install_ddtrace.sh new file mode 100755 index 00000000000..cd0505ccd87 --- /dev/null +++ b/utils/build/docker/rust/weblog/install_ddtrace.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Prepares the dd-trace-rs (datadog-opentelemetry) source the weblog depends on. +# +# Unlike the parametric install_ddtrace.sh, this script does NOT enable the metrics/logs gRPC +# features: the OTLP trace-export weblog only needs trace export over http/json, and pulling in +# tonic via the gRPC features raises the minimum rustc above the base image's toolchain. +# +# Resolution order: +# 1. /binaries/rust-load-from-git present -> clone that branch/ref of dd-trace-rs. +# 2. /binaries/dd-trace-rs present -> use the pre-supplied checkout as-is. +# 3. otherwise -> swap the path dependency for the crates.io release. + +set -eu + +cd /usr/app + +REPO_URL=https://github.com/DataDog/dd-trace-rs + +if [ -e /binaries/rust-load-from-git ] && [ ! -e /binaries/dd-trace-rs ]; then + rev_or_branch=$(&2 + exit 1 + fi + new_version="${major}.${minor}.$((patch + 1))" + + if [ -e /binaries/dd-trace-rs/.git ]; then + dev_version="${new_version}-dev+$(git -C /binaries/dd-trace-rs rev-parse --short HEAD)" + else + dev_version="${new_version}-dev" + fi + + echo "generating dev version $dev_version from $current_version" + sed -i "s/^version = \"${current_version}\"/version = \"${dev_version}\"/" /binaries/dd-trace-rs/Cargo.toml + + cd /usr/app +else + echo "No local dd-trace-rs checkout; using crates.io release" + cargo remove datadog-opentelemetry || true + cargo add datadog-opentelemetry +fi diff --git a/utils/build/docker/rust/weblog/src/main.rs b/utils/build/docker/rust/weblog/src/main.rs new file mode 100644 index 00000000000..4b39df63e09 --- /dev/null +++ b/utils/build/docker/rust/weblog/src/main.rs @@ -0,0 +1,232 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +//! Minimal end-to-end weblog for the Datadog Rust tracer (dd-trace-rs). +//! +//! It exposes the endpoints the system-tests end-to-end scenarios hit: +//! - `GET /` -> a single server span +//! - `GET /make_distant_call` -> a server span + an outbound HTTP client span +//! - `GET /healthcheck` -> readiness probe consumed by the test harness +//! +//! OTLP export behavior is entirely env-driven by the scenario (OTEL_TRACES_EXPORTER, +//! OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, ...), so the app only has to initialize the +//! tracer normally and produce the expected spans. + +use std::collections::HashMap; +use std::env; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use axum::{ + extract::Query, + http::{HeaderMap, StatusCode}, + response::IntoResponse, + routing::get, + Json, Router, +}; +use opentelemetry::{ + global, + trace::{Span, SpanKind, Status, TraceContextExt, Tracer}, + Context, KeyValue, +}; +use opentelemetry_http::HeaderInjector; +use opentelemetry_sdk::trace::SdkTracerProvider; +use serde::Serialize; +use serde_json::json; +use tokio::net::TcpListener; +use tokio::signal::unix::{signal, SignalKind}; + +// Datadog-tracer attribute conventions. The tracer maps these OTel semantic-convention +// keys onto the Datadog span fields the OTLP tests assert against: +// http.request.method -> http.method +// http.response.status_code -> http.status_code +// http.request.headers.user-agent (the harness correlates the span to the request via this attr) +const ATTR_HTTP_REQUEST_METHOD: &str = "http.request.method"; +const ATTR_HTTP_RESPONSE_STATUS_CODE: &str = "http.response.status_code"; +const ATTR_URL_PATH: &str = "url.path"; +// `get_otel_spans` matches the injected `rid/` token on the RAW OTLP payload, which it reads from +// `http.request.headers.user-agent` (or `http.useragent`) — NOT the OTel `user_agent.original` key. +const ATTR_USER_AGENT: &str = "http.request.headers.user-agent"; + +fn user_agent(headers: &HeaderMap) -> String { + headers + .get(axum::http::header::USER_AGENT) + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string() +} + +/// Build the common HTTP server-span attributes. Setting `http.request.headers.user-agent` is what +/// lets the system-tests harness correlate the exported span back to the originating request: it +/// injects a `rid/` token into the User-Agent header, and `get_otel_spans` looks that token up in +/// the raw OTLP `http.request.headers.user-agent` / `http.useragent` attribute. +fn server_span_attributes(method: &str, path: &str, headers: &HeaderMap, status: i64) -> Vec { + vec![ + KeyValue::new(ATTR_HTTP_REQUEST_METHOD, method.to_string()), + KeyValue::new(ATTR_URL_PATH, path.to_string()), + KeyValue::new(ATTR_HTTP_RESPONSE_STATUS_CODE, status), + KeyValue::new(ATTR_USER_AGENT, user_agent(headers)), + ] +} + +async fn healthcheck() -> impl IntoResponse { + let version = env::var("SYSTEM_TESTS_LIBRARY_VERSION").unwrap_or_default(); + Json(json!({ + "status": "ok", + "library": { + "name": "rust", + "version": version, + } + })) +} + +async fn home(headers: HeaderMap) -> impl IntoResponse { + let tracer = global::tracer("weblog"); + let mut span = tracer + .span_builder("server.request") + .with_kind(SpanKind::Server) + .start(&tracer); + span.set_status(Status::Unset); + for kv in server_span_attributes("GET", "/", &headers, 200) { + span.set_attribute(kv); + } + span.end(); + + (StatusCode::OK, "Hello world!\n") +} + +#[derive(Serialize)] +struct DistantCallResponse { + url: String, + status_code: u16, + request_headers: HashMap, + response_headers: HashMap, +} + +async fn make_distant_call(headers: HeaderMap, Query(params): Query>) -> impl IntoResponse { + let tracer = global::tracer("weblog"); + let mut server_span = tracer + .span_builder("server.request") + .with_kind(SpanKind::Server) + .start(&tracer); + server_span.set_status(Status::Unset); + for kv in server_span_attributes("GET", "/make_distant_call", &headers, 200) { + server_span.set_attribute(kv); + } + + // Make the server span the active context so the outbound client span is a child of it, + // producing the multi-span trace the 128-bit trace-id test needs. + let cx = Context::current_with_span(server_span); + + let Some(url) = params.get("url").filter(|u| !u.is_empty()).cloned() else { + // Match the contract of the other weblogs: no url => plain "OK". + cx.span().end(); + return (StatusCode::OK, "OK").into_response(); + }; + + let result = perform_distant_call(&cx, &url).await; + + cx.span().end(); + + match result { + Ok(body) => (StatusCode::OK, Json(body)).into_response(), + Err(err) => { + tracing::error!("make_distant_call failed: {err:#}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": err.to_string() })), + ) + .into_response() + } + } +} + +async fn perform_distant_call(parent_cx: &Context, url: &str) -> anyhow::Result { + // Use the global (boxed) tracer so the produced span is Send + Sync and can be carried in a + // Context across the await point below. + let tracer = global::tracer("weblog"); + let mut client_span = tracer + .span_builder("http.request") + .with_kind(SpanKind::Client) + .start_with_context(&tracer, parent_cx); + client_span.set_attribute(KeyValue::new(ATTR_HTTP_REQUEST_METHOD, "GET")); + + let client_cx = parent_cx.with_span(client_span); + + // Inject the trace context into the outbound request headers so propagation is exercised. + let mut injected_headers = reqwest::header::HeaderMap::new(); + global::get_text_map_propagator(|propagator| { + propagator.inject_context(&client_cx, &mut HeaderInjector(&mut injected_headers)); + }); + + let client = reqwest::Client::new(); + let response = client.get(url).headers(injected_headers.clone()).send().await?; + let status = response.status(); + + let request_headers = injected_headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) + .collect(); + let response_headers = response + .headers() + .iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) + .collect(); + + client_cx + .span() + .set_attribute(KeyValue::new(ATTR_HTTP_RESPONSE_STATUS_CODE, status.as_u16() as i64)); + client_cx.span().end(); + + Ok(DistantCallResponse { + url: url.to_string(), + status_code: status.as_u16(), + request_headers, + response_headers, + }) +} + +fn init_tracer() -> SdkTracerProvider { + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .try_init(); + + // All Datadog/OTLP configuration is supplied through environment variables by the scenario. + datadog_opentelemetry::tracing() + .with_config(datadog_opentelemetry::configuration::Config::builder().build()) + .init() +} + +#[tokio::main] +async fn main() { + let provider = init_tracer(); + + let app = Router::new() + .route("/", get(home)) + .route("/make_distant_call", get(make_distant_call)) + .route("/healthcheck", get(healthcheck)); + + let port: u16 = env::var("WEBLOG_PORT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(7777); + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port); + + let listener = TcpListener::bind(addr).await.expect("bind weblog listener"); + tracing::info!("weblog listening on {addr}"); + + axum::serve(listener, app) + .with_graceful_shutdown(shutdown_signal()) + .await + .expect("run weblog server"); + + let _ = provider.shutdown(); +} + +async fn shutdown_signal() { + let mut term = signal(SignalKind::terminate()).expect("install SIGTERM handler"); + term.recv().await; +} diff --git a/utils/build/docker/rust/weblog/system_tests_library_version.sh b/utils/build/docker/rust/weblog/system_tests_library_version.sh new file mode 100755 index 00000000000..97403df98a0 --- /dev/null +++ b/utils/build/docker/rust/weblog/system_tests_library_version.sh @@ -0,0 +1,2 @@ +#!/bin/bash +grep 'name = "datadog-opentelemetry"' ./Cargo.lock -A 1 | grep -Po 'version = "\K.*?(?=")'