From cec23b93aaaec7ca520b71e520b6f73e606280ac Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 18 Apr 2023 14:13:10 +0200 Subject: [PATCH] Better tracing attributes in the HTTP client --- crates/cli/src/main.rs | 1 + crates/http/src/layers/client.rs | 93 +++++++++++++++++++++++++++++--- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index a472f3e6d..e05411df0 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -98,6 +98,7 @@ async fn try_main() -> anyhow::Result<()> { tracing_opentelemetry::layer() .with_tracer(tracer) .with_tracked_inactivity(false) + .with_exception_fields(true) .with_filter(LevelFilter::INFO) }); diff --git a/crates/http/src/layers/client.rs b/crates/http/src/layers/client.rs index 637d6ae03..a44eff7e5 100644 --- a/crates/http/src/layers/client.rs +++ b/crates/http/src/layers/client.rs @@ -14,8 +14,12 @@ use std::{sync::Arc, time::Duration}; -use http::{header::USER_AGENT, HeaderValue, Request}; -use mas_tower::{MakeSpan, TraceContextLayer, TraceContextService, TraceLayer, TraceService}; +use headers::{ContentLength, HeaderMapExt, Host, UserAgent}; +use http::{header::USER_AGENT, HeaderValue, Request, Response}; +use hyper::client::connect::HttpInfo; +use mas_tower::{ + EnrichSpan, MakeSpan, TraceContextLayer, TraceContextService, TraceLayer, TraceService, +}; use tokio::sync::Semaphore; use tower::{ limit::{ConcurrencyLimit, GlobalConcurrencyLimitLayer}, @@ -26,10 +30,18 @@ use tower_http::{ set_header::{SetRequestHeader, SetRequestHeaderLayer}, timeout::{Timeout, TimeoutLayer}, }; +use tracing::Span; pub type ClientService = SetRequestHeader< ConcurrencyLimit< - FollowRedirect>, MakeSpanForRequest>>, + FollowRedirect< + TraceService< + TraceContextService>, + MakeSpanForRequest, + EnrichSpanOnResponse, + EnrichSpanOnError, + >, + >, >, HeaderValue, >; @@ -38,22 +50,85 @@ pub type ClientService = SetRequestHeader< pub struct MakeSpanForRequest; impl MakeSpan> for MakeSpanForRequest { - fn make_span(&self, request: &Request) -> tracing::Span { - // TODO: better attributes + fn make_span(&self, request: &Request) -> Span { + let headers = request.headers(); + let host = headers.typed_get::().map(tracing::field::display); + let user_agent = headers + .typed_get::() + .map(tracing::field::display); + let content_length = headers.typed_get().map(|ContentLength(len)| len); + let net_sock_peer_name = request.uri().host(); + tracing::info_span!( "http.client.request", + "otel.kind" = "client", + "otel.status_code" = tracing::field::Empty, "http.method" = %request.method(), - "http.uri" = %request.uri(), + "http.url" = %request.uri(), + "http.status_code" = tracing::field::Empty, + "http.host" = host, + "http.request_content_length" = content_length, + "http.response_content_length" = tracing::field::Empty, + "net.transport" = "ip_tcp", + "net.sock.family" = tracing::field::Empty, + "net.sock.peer.name" = net_sock_peer_name, + "net.sock.peer.addr" = tracing::field::Empty, + "net.sock.peer.port" = tracing::field::Empty, + "net.sock.host.addr" = tracing::field::Empty, + "net.sock.host.port" = tracing::field::Empty, + "user_agent.original" = user_agent, + "rust.error" = tracing::field::Empty, ) } } +#[derive(Debug, Clone)] +pub struct EnrichSpanOnResponse; + +impl EnrichSpan> for EnrichSpanOnResponse { + fn enrich_span(&self, span: &Span, response: &Response) { + span.record("otel.status_code", "OK"); + span.record("http.status_code", response.status().as_u16()); + + if let Some(ContentLength(content_length)) = response.headers().typed_get() { + span.record("http.response_content_length", content_length); + } + + if let Some(http_info) = response.extensions().get::() { + let local = http_info.local_addr(); + let remote = http_info.remote_addr(); + + let family = if local.is_ipv4() { "inet" } else { "inet6" }; + span.record("net.sock.family", family); + span.record("net.sock.peer.addr", remote.ip().to_string()); + span.record("net.sock.peer.port", remote.port()); + span.record("net.sock.host.addr", local.ip().to_string()); + span.record("net.sock.host.port", local.port()); + } else { + tracing::warn!("No HttpInfo injected in response extensions"); + } + } +} + +#[derive(Debug, Clone)] +pub struct EnrichSpanOnError; + +impl EnrichSpan for EnrichSpanOnError +where + E: std::error::Error + 'static, +{ + fn enrich_span(&self, span: &Span, error: &E) { + span.record("otel.status_code", "ERROR"); + span.record("rust.error", error as &dyn std::error::Error); + } +} + #[derive(Debug, Clone)] pub struct ClientLayer { user_agent_layer: SetRequestHeaderLayer, concurrency_limit_layer: GlobalConcurrencyLimitLayer, follow_redirect_layer: FollowRedirectLayer, - trace_layer: TraceLayer, + trace_layer: TraceLayer, trace_context_layer: TraceContextLayer, timeout_layer: TimeoutLayer, } @@ -80,7 +155,9 @@ impl ClientLayer { ), concurrency_limit_layer: GlobalConcurrencyLimitLayer::with_semaphore(semaphore), follow_redirect_layer: FollowRedirectLayer::new(), - trace_layer: TraceLayer::new(MakeSpanForRequest), + trace_layer: TraceLayer::new(MakeSpanForRequest) + .on_response(EnrichSpanOnResponse) + .on_error(EnrichSpanOnError), trace_context_layer: TraceContextLayer::new(), timeout_layer: TimeoutLayer::new(Duration::from_secs(10)), }