feat(tracing): support for distributed tracing

Author: solnic

lib/sentry/opentelemetry/propagator.ex

@@ -0,0 +1,135 @@
+if Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
+  defmodule Sentry.OpenTelemetry.Propagator do
+    @moduledoc """
+    OpenTelemetry propagator for Sentry distributed tracing.
+
+    This propagator implements the `sentry-trace` and `sentry-baggage` header propagation
+    to enable distributed tracing across service boundaries. It follows the W3C Trace Context.
+    """
+
+    import Bitwise
+
+    require Record
+    require OpenTelemetry.Tracer, as: Tracer
+
+    @behaviour :otel_propagator_text_map
+
+    @fields Record.extract(:span_ctx, from_lib: "opentelemetry_api/include/opentelemetry.hrl")
+    Record.defrecordp(:span_ctx, @fields)
+
+    @sentry_trace_key "sentry-trace"
+    @sentry_baggage_key "baggage"
+    @sentry_trace_ctx_key :"sentry-trace"
+    @sentry_baggage_ctx_key :"sentry-baggage"
+
+    @impl true
+    def fields(_opts) do
+      [@sentry_trace_key, @sentry_baggage_key]
+    end
+
+    @impl true
+    def inject(ctx, carrier, setter, _opts) do
+      case Tracer.current_span_ctx(ctx) do
+        span_ctx(trace_id: tid, span_id: sid, trace_flags: flags) when tid != 0 and sid != 0 ->
+          sentry_trace_header = encode_sentry_trace({tid, sid, flags})
+          carrier = setter.(@sentry_trace_key, sentry_trace_header, carrier)
+
+          baggage_value = :otel_ctx.get_value(ctx, @sentry_baggage_ctx_key, :not_found)
+
+          if is_binary(baggage_value) and baggage_value != :not_found do
+            setter.(@sentry_baggage_key, baggage_value, carrier)
+          else
+            carrier
+          end
+
+        _ ->
+          carrier
+      end
+    end
+
+    @impl true
+    def extract(ctx, carrier, _keys_fun, getter, _opts) do
+      case getter.(@sentry_trace_key, carrier) do
+        :undefined ->
+          ctx
+
+        header when is_binary(header) ->
+          case decode_sentry_trace(header) do
+            {:ok, {trace_hex, span_hex, sampled}} ->
+              ctx =
+                ctx
+                |> :otel_ctx.set_value(@sentry_trace_ctx_key, {trace_hex, span_hex, sampled})
+                |> maybe_set_baggage(getter.(@sentry_baggage_key, carrier))
+
+              trace_id = hex_to_int(trace_hex)
+              span_id = hex_to_int(span_hex)
+
+              # Create a remote, sampled parent span in the OTEL context.
+              # We will set to "always sample" because Sentry will decide real sampling
+              remote_span_ctx = :otel_tracer.from_remote_span(trace_id, span_id, 1)
+
+              Tracer.set_current_span(ctx, remote_span_ctx)
+
+            {:error, _reason} ->
+              ctx
+          end
+
+        _ ->
+          ctx
+      end
+    end
+
+    # Encode trace ID, span ID, and sampled flag to sentry-trace header format
+    # Format: {trace_id}-{span_id}-{sampled}
+    defp encode_sentry_trace({trace_id_int, span_id_int, trace_flags}) do
+      sampled = if (trace_flags &&& 1) == 1, do: "1", else: "0"
+      int_to_hex(trace_id_int, 16) <> "-" <> int_to_hex(span_id_int, 8) <> "-" <> sampled
+    end
+
+    # Decode sentry-trace header
+    # Format: {trace_id}-{span_id}-{sampled} or {trace_id}-{span_id}
+    defp decode_sentry_trace(
+           <<trace_hex::binary-size(32), "-", span_hex::binary-size(16), "-",
+             sampled::binary-size(1)>>
+         ) do
+      {:ok, {trace_hex, span_hex, sampled == "1"}}
+    end
+
+    defp decode_sentry_trace(<<trace_hex::binary-size(32), "-", span_hex::binary-size(16)>>) do
+      {:ok, {trace_hex, span_hex, false}}
+    end
+
+    defp decode_sentry_trace(_invalid) do
+      {:error, :invalid_format}
+    end
+
+    defp maybe_set_baggage(ctx, :undefined), do: ctx
+    defp maybe_set_baggage(ctx, ""), do: ctx
+    defp maybe_set_baggage(ctx, nil), do: ctx
+
+    defp maybe_set_baggage(ctx, baggage) when is_binary(baggage) do
+      :otel_ctx.set_value(ctx, @sentry_baggage_ctx_key, baggage)
+    end
+
+    # Convert hex string to integer
+    defp hex_to_int(hex) do
+      hex
+      |> Base.decode16!(case: :mixed)
+      |> :binary.decode_unsigned()
+    end
+
+    # Convert integer to hex string with padding
+    defp int_to_hex(value, num_bytes) do
+      value
+      |> :binary.encode_unsigned()
+      |> bin_pad_left(num_bytes)
+      |> Base.encode16(case: :lower)
+    end
+
+    # Pad binary to specified number of bytes
+    defp bin_pad_left(bin, total_bytes) do
+      missing = total_bytes - byte_size(bin)
+      if missing > 0, do: :binary.copy(<<0>>, missing) <> bin, else: bin
+    end
+  end
+end

lib/sentry/opentelemetry/span_processor.ex

@@ -11,11 +11,15 @@ if Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
     require OpenTelemetry.SemConv.Incubating.MessagingAttributes, as: MessagingAttributes
 
     require Logger
+    require Record
 
     alias Sentry.{Transaction, OpenTelemetry.SpanStorage, OpenTelemetry.SpanRecord}
     alias Sentry.Interfaces.Span
 
-    # This can be a no-op since we can postpone inserting the span into storage until on_end
+    # Extract span record fields to access parent_span_id in on_start
+    @span_fields Record.extract(:span, from_lib: "opentelemetry/include/otel_span.hrl")
+    Record.defrecordp(:span, @span_fields)
+
     @impl :otel_span_processor
     def on_start(_ctx, otel_span, _config) do
       otel_span
@@ -24,42 +28,93 @@ if Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
     @impl :otel_span_processor
     def on_end(otel_span, _config) do
       span_record = SpanRecord.new(otel_span)
+      process_span(span_record)
+    end
 
+    defp process_span(span_record) do
       SpanStorage.store_span(span_record)
 
-      if span_record.parent_span_id == nil do
-        child_span_records = SpanStorage.get_child_spans(span_record.span_id)
-        transaction = build_transaction(span_record, child_span_records)
+      # Check if this is a root span (no parent) or a transaction root
+      #
+      # A span should be a transaction root if:
+      # 1. It has no parent (true root span)
+      # 2. OR it's a server span with only a REMOTE parent (distributed tracing)
+      #
+      # A span should NOT be a transaction root if:
+      # - It has a LOCAL parent (parent span exists in our SpanStorage)
+      is_transaction_root =
+        cond do
+          # No parent = definitely a root
+          span_record.parent_span_id == nil ->
+            true
+
+          # Has a parent - check if it's local or remote
+          true ->
+            has_local_parent = has_local_parent_span?(span_record.parent_span_id)
+
+            if has_local_parent do
+              # Parent exists locally - this is a child span, not a transaction root
+              false
+            else
+              # Parent is remote (distributed tracing) - treat server spans as transaction roots
+              is_server_span?(span_record)
+            end
+        end
 
-        result =
-          case Sentry.send_transaction(transaction) do
-            {:ok, _id} ->
-              true
+      if is_transaction_root do
+        build_and_send_transaction(span_record)
+      else
+        true
+      end
+    end
 
-            :ignored ->
-              true
+    defp build_and_send_transaction(span_record) do
+      child_span_records = SpanStorage.get_child_spans(span_record.span_id)
+      transaction = build_transaction(span_record, child_span_records)
 
-            :excluded ->
-              true
+      result =
+        case Sentry.send_transaction(transaction) do
+          {:ok, _id} ->
+            true
 
-            {:error, error} ->
-              Logger.warning("Failed to send transaction to Sentry: #{inspect(error)}")
-              {:error, :invalid_span}
-          end
+          :ignored ->
+            true
 
-        :ok = SpanStorage.remove_root_span(span_record.span_id)
+          :excluded ->
+            true
 
-        result
-      else
-        true
-      end
+          {:error, error} ->
+            Logger.warning("Failed to send transaction to Sentry: #{inspect(error)}")
+            {:error, :invalid_span}
+        end
+
+      # Clean up: remove the transaction root span and all its children
+      :ok = SpanStorage.remove_root_span(span_record.span_id)
+
+      result
     end
 
     @impl :otel_span_processor
     def force_flush(_config) do
       :ok
     end
 
+    # Checks if a parent span exists in our local SpanStorage
+    # This helps distinguish between:
+    # - Local parents: span exists in storage (same service)
+    # - Remote parents: span doesn't exist in storage (distributed tracing from another service)
+    defp has_local_parent_span?(parent_span_id) do
+      SpanStorage.span_exists?(parent_span_id)
+    end
+
+    # Helper function to detect if a span is a server span that should be
+    # treated as a transaction root for distributed tracing.
+    defp is_server_span?(%{kind: :server, attributes: attributes}) do
+      Map.has_key?(attributes, to_string(HTTPAttributes.http_request_method()))
+    end
+
+    defp is_server_span?(_), do: false
+
     defp build_transaction(root_span_record, child_span_records) do
       root_span = build_span(root_span_record)
       child_spans = Enum.map(child_span_records, &build_span(&1))
@@ -114,10 +169,19 @@ if Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
 
       url_path = Map.get(span_record.attributes, to_string(URLAttributes.url_path()))
 
+      # Build description with method and path
       description =
-        to_string(http_request_method) <>
-          ((client_address && " from #{client_address}") || "") <>
-          ((url_path && " #{url_path}") || "")
+        case url_path do
+          nil -> to_string(http_request_method)
+          path -> "#{http_request_method} #{path}"
+        end
+
+      description =
+        if client_address do
+          "#{description} from #{client_address}"
+        else
+          description
+        end
 
       {op, description}
     end

lib/sentry/opentelemetry/span_storage.ex

@@ -37,6 +37,22 @@ if Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
       {:noreply, state}
     end
 
+    @spec span_exists?(String.t(), keyword()) :: boolean()
+    def span_exists?(span_id, opts \\ []) do
+      table_name = Keyword.get(opts, :table_name, default_table_name())
+
+      case :ets.lookup(table_name, {:root_span, span_id}) do
+        [{{:root_span, ^span_id}, _span, _stored_at}] ->
+          true
+
+        [] ->
+          case :ets.match_object(table_name, {{:child_span, :_, span_id}, :_, :_}) do
+            [] -> false
+            _ -> true
+          end
+      end
+    end
+
     @spec store_span(SpanRecord.t(), keyword()) :: true
     def store_span(span_data, opts \\ []) do
       table_name = Keyword.get(opts, :table_name, default_table_name())