diff --git a/CHANGELOG.md b/CHANGELOG.md index 09aa9af85e..b745d9b415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ All notable changes to this project will be documented in this file. From versio - Log host, port and pg version of listener database connection by @mkleczek in #4617 #4618 - Optimize requests with `Prefer: count=exact` that do not use ranges or `db-max-rows` by @laurenceisla in #3957 + Removed unnecessary double count when building the `Content-Range`. +- Introduced producing OpenTelemetry traces by @develop7 in #3140 + + Requires a new `server-otel-enabled` config parameter to be enabled first. ### Fixed diff --git a/docs/integrations/opentelemetry.rst b/docs/integrations/opentelemetry.rst new file mode 100644 index 0000000000..0239d306da --- /dev/null +++ b/docs/integrations/opentelemetry.rst @@ -0,0 +1,26 @@ +.. _opentelemetry: + +OpenTelemetry +------------- + +PostgREST is able to act as OpenTelemetry traces producer. OpenTelemetry is configured +using ``OTEL_*`` environment variables, per the `OpenTelemetry specification`_. + +Example configuration: + +.. code-block:: shell + + OTEL_EXPORTER_OTLP_ENDPOINT='https://api.honeycomb.io/' \ + OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=" \ + OTEL_SERVICE_NAME='PostgREST'\ + OTEL_LOG_LEVEL='debug'\ + OTEL_TRACES_SAMPLER='always_on' \ + postgrest + +Since current OpenTelemetry implementation incurs a small (~6% in our "Loadtest (mixed)" suite) +performance hit, it is gated behind the :ref:`server-otel-enabled` configuration option, disabled by default. + +.. _hs-opentelemetry: https://github.com/iand675/hs-opentelemetry/ + +.. _`OpenTelemetry specification`: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/ + diff --git a/docs/postgrest.dict b/docs/postgrest.dict index a890f85f8e..4fcc20648f 100644 --- a/docs/postgrest.dict +++ b/docs/postgrest.dict @@ -56,6 +56,7 @@ HMAC htmx Htmx Homebrew +hs hstore HTTP HTTPS @@ -83,6 +84,7 @@ localhost login lookups Logins +Loadtest LIBPQ logins lon @@ -106,6 +108,10 @@ Observability Okta OpenAPI openapi +OpenTelemetry +opentelemetry +otel +OTLP ov parametrized passphrase diff --git a/docs/references/configuration.rst b/docs/references/configuration.rst index 82800f36f9..4ae741511f 100644 --- a/docs/references/configuration.rst +++ b/docs/references/configuration.rst @@ -888,6 +888,21 @@ server-timing-enabled Enables the `Server-Timing `_ header. See :ref:`server-timing_header`. +.. _server-otel-enabled: + +server-otel-enabled +------------------- + + =============== ================================= + **Type** Boolean + **Default** False + **Reloadable** N + **Environment** PGRST_SERVER_OTEL_ENABLED + **In-Database** `n/a` + =============== ================================= + + When this is set to :code:`true`, OpenTelemetry tracing is enabled. See :ref:`opentelemetry` for details and settings. + .. _server-unix-socket: server-unix-socket diff --git a/nix/tools/loadtest.nix b/nix/tools/loadtest.nix index b25c435916..78da68aba4 100644 --- a/nix/tools/loadtest.nix +++ b/nix/tools/loadtest.nix @@ -68,6 +68,8 @@ let # TODO clean once PGRST_JWT_CACHE_MAX_ENTRIES merged and released export PGRST_JWT_CACHE_MAX_LIFETIME="86400" + export OTEL_SDK_DISABLED="true" + mkdir -p "$(dirname "$_arg_output")" abs_output="$(realpath "$_arg_output")" diff --git a/postgrest.cabal b/postgrest.cabal index fd5f3ec28b..c96d57d6ef 100644 --- a/postgrest.cabal +++ b/postgrest.cabal @@ -78,6 +78,7 @@ library PostgREST.Query.QueryBuilder PostgREST.Query.SqlFragment PostgREST.Query.Statements + PostgREST.OpenTelemetry PostgREST.Plan PostgREST.Plan.CallPlan PostgREST.Plan.MutatePlan @@ -111,6 +112,7 @@ library , cookie >= 0.4.2 && < 0.6 , directory >= 1.2.6 && < 1.4 , either >= 4.4.1 && < 5.1 + , exceptions >= 0.10 && < 0.12 , extra >= 1.7.0 && < 2.0 , fuzzyset >= 0.2.4 && < 0.3 , hasql >= 1.6.1.1 && < 1.7 @@ -120,6 +122,12 @@ library , hasql-transaction >= 1.0.1 && < 1.2 , http-client >= 0.7.19 && < 0.8 , http-types >= 0.12.2 && < 0.13 + , hs-opentelemetry-sdk >= 0.1.0.0 && < 0.2.0.0 + , hs-opentelemetry-instrumentation-wai + , hs-opentelemetry-api + -- ^ this is due to hs-otel-sdk is not reexporting getTracerTracerProvider + -- needed to initialize OpenTelemetry.middleware + , hs-opentelemetry-utils-exceptions , insert-ordered-containers >= 0.2.2 && < 0.3 , jose-jwt >= 0.9.6 && < 0.11 , lens >= 4.14 && < 5.4 @@ -269,6 +277,8 @@ test-suite spec , hasql-pool >= 1.0.1 && < 1.1 , hasql-transaction >= 1.0.1 && < 1.2 , heredoc >= 0.2 && < 0.3 + , hs-opentelemetry-sdk >= 0.1.0.0 && < 0.2.0.0 + , hs-opentelemetry-instrumentation-hspec , hspec >= 2.3 && < 2.12 , hspec-expectations >= 0.8.4 && < 0.9 , hspec-wai >= 0.10 && < 0.12 diff --git a/src/PostgREST/App.hs b/src/PostgREST/App.hs index 411694fd1e..6e543926a0 100644 --- a/src/PostgREST/App.hs +++ b/src/PostgREST/App.hs @@ -27,22 +27,23 @@ import qualified Data.Text.Encoding as T import qualified Network.Wai as Wai import qualified Network.Wai.Handler.Warp as Warp -import qualified PostgREST.Admin as Admin -import qualified PostgREST.ApiRequest as ApiRequest -import qualified PostgREST.AppState as AppState -import qualified PostgREST.Auth as Auth -import qualified PostgREST.Cors as Cors -import qualified PostgREST.Error as Error -import qualified PostgREST.Listener as Listener -import qualified PostgREST.Logger as Logger -import qualified PostgREST.MainTx as MainTx -import qualified PostgREST.Plan as Plan -import qualified PostgREST.Query as Query -import qualified PostgREST.Response as Response -import qualified PostgREST.Unix as Unix (installSignalHandlers) +import qualified PostgREST.Admin as Admin +import qualified PostgREST.ApiRequest as ApiRequest +import qualified PostgREST.AppState as AppState +import qualified PostgREST.Auth as Auth +import qualified PostgREST.Cors as Cors +import qualified PostgREST.Error as Error +import qualified PostgREST.Listener as Listener +import qualified PostgREST.Logger as Logger +import qualified PostgREST.MainTx as MainTx +import qualified PostgREST.OpenTelemetry as OTel +import qualified PostgREST.Plan as Plan +import qualified PostgREST.Query as Query +import qualified PostgREST.Response as Response +import qualified PostgREST.Unix as Unix (installSignalHandlers) import PostgREST.ApiRequest (ApiRequest (..)) -import PostgREST.AppState (AppState) +import PostgREST.AppState (AppState, getOTelTracer) import PostgREST.Auth.Types (AuthResult (..)) import PostgREST.Config (AppConfig (..), LogLevel (..)) import PostgREST.Error (Error) @@ -57,11 +58,12 @@ import PostgREST.Version (docsVersion, prettyVersion) import qualified Data.ByteString.Char8 as BS import qualified Data.List as L import qualified Network.HTTP.Types as HTTP +import OpenTelemetry.Trace (defaultSpanArguments) import Protolude hiding (Handler) type Handler = ExceptT Error -run :: AppState -> IO () +run :: HasCallStack => AppState -> IO () run appState = do let observer = AppState.getObserver appState conf@AppConfig{..} <- AppState.getConfig appState @@ -89,41 +91,44 @@ serverSettings AppConfig{..} = & setServerName ("postgrest/" <> prettyVersion) -- | PostgREST application -postgrest :: LogLevel -> AppState.AppState -> IO () -> Wai.Application +postgrest :: HasCallStack => LogLevel -> AppState.AppState -> IO () -> Wai.Application postgrest logLevel appState connWorker = + OTel.middleware appState . traceHeaderMiddleware appState . Cors.middleware appState . Auth.middleware appState . Logger.middleware logLevel Auth.getRole $ -- fromJust can be used, because the auth middleware will **always** add -- some AuthResult to the vault. - \req respond -> case fromJust $ Auth.getResult req of - Left err -> respond $ Error.errorResponseFor err - Right authResult -> do - appConf <- AppState.getConfig appState -- the config must be read again because it can reload - maybeSchemaCache <- AppState.getSchemaCache appState - - let - eitherResponse :: IO (Either Error Wai.Response) - eitherResponse = - runExceptT $ postgrestResponse appState appConf maybeSchemaCache authResult req - - response <- either Error.errorResponseFor identity <$> eitherResponse - -- Launch the connWorker when the connection is down. The postgrest - -- function can respond successfully (with a stale schema cache) before - -- the connWorker is done. However, when there's an empty schema cache - -- postgrest responds with the error `PGRST002`; this means that the schema - -- cache is still loading, so we don't launch the connWorker here because - -- it would duplicate the loading process, e.g. https://github.com/PostgREST/postgrest/issues/3704 - -- TODO: this process may be unnecessary when the Listener is enabled. Revisit once https://github.com/PostgREST/postgrest/issues/1766 is done - when (isServiceUnavailable response && isJust maybeSchemaCache) connWorker - resp <- do - delay <- AppState.getNextDelay appState - return $ addRetryHint delay response - respond resp + \req respond -> OTel.inSpanM (getOTelTracer appState) "request" defaultSpanArguments $ + case fromJust $ Auth.getResult req of + Left err -> respond $ Error.errorResponseFor err + Right authResult -> do + appConf <- AppState.getConfig appState -- the config must be read again because it can reload + maybeSchemaCache <- AppState.getSchemaCache appState + + let + eitherResponse :: IO (Either Error Wai.Response) + eitherResponse = + runExceptT $ postgrestResponse appState appConf maybeSchemaCache authResult req + + response <- either Error.errorResponseFor identity <$> eitherResponse + -- Launch the connWorker when the connection is down. The postgrest + -- function can respond successfully (with a stale schema cache) before + -- the connWorker is done. However, when there's an empty schema cache + -- postgrest responds with the error `PGRST002`; this means that the schema + -- cache is still loading, so we don't launch the connWorker here because + -- it would duplicate the loading process, e.g. https://github.com/PostgREST/postgrest/issues/3704 + -- TODO: this process may be unnecessary when the Listener is enabled. Revisit once https://github.com/PostgREST/postgrest/issues/1766 is done + when (isServiceUnavailable response && isJust maybeSchemaCache) connWorker + resp <- do + delay <- AppState.getNextDelay appState + return $ addRetryHint delay response + respond resp postgrestResponse - :: AppState.AppState + :: HasCallStack + => AppState.AppState -> AppConfig -> Maybe SchemaCache -> AuthResult @@ -146,14 +151,14 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache authResult@AuthRe timezones = dbTimezones sCache prefs = ApiRequest.userPreferences conf req timezones - (parseTime, apiReq@ApiRequest{..}) <- withTiming $ liftEither . mapLeft Error.ApiRequestError $ ApiRequest.userApiRequest conf prefs req body - (planTime, plan) <- withTiming $ liftEither $ Plan.actionPlan iAction conf apiReq sCache + (parseTime, apiReq@ApiRequest{..}) <- withOTel "parse" $ withTiming $ liftEither . mapLeft Error.ApiRequestError $ ApiRequest.userApiRequest conf prefs req body + (planTime, plan) <- withOTel "plan" $ withTiming $ liftEither $ Plan.actionPlan iAction conf apiReq sCache let mainQ = Query.mainQuery plan conf apiReq authResult configDbPreRequest tx = MainTx.mainTx mainQ conf authResult apiReq plan sCache obsQuery s = when configLogQuery $ observer $ QueryObs mainQ s - (txTime, txResult) <- withTiming $ do + (txTime, txResult) <- withOTel "query" $ withTiming $ do case tx of MainTx.NoDbTx r -> pure r MainTx.DbTx{..} -> do @@ -166,7 +171,7 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache authResult@AuthRe lift $ whenLeft eitherResp $ obsQuery . Error.status liftEither eitherResp - (respTime, resp) <- withTiming $ do + (respTime, resp) <- withOTel "response" $ withTiming $ do let response = Response.actionResponse txResult apiReq (T.decodeUtf8 prettyVersion, docsVersion) conf sCache iSchema iNegotiatedByProfile status' = either Error.status Response.pgrstStatus response @@ -177,10 +182,10 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache authResult@AuthRe return $ toWaiResponse (ServerTiming jwtTime parseTime planTime txTime respTime) resp where - toWaiResponse :: ServerTiming -> Response.PgrstResponse -> Wai.Response + toWaiResponse :: HasCallStack => ServerTiming -> Response.PgrstResponse -> Wai.Response toWaiResponse timing (Response.PgrstResponse st hdrs bod) = Wai.responseLBS st (hdrs ++ ([serverTimingHeader timing | configServerTimingEnabled])) bod - withTiming :: Handler IO a -> Handler IO (Maybe Double, a) + withTiming :: HasCallStack => Handler IO a -> Handler IO (Maybe Double, a) withTiming f = if configServerTimingEnabled then do (t, r) <- timeItT f @@ -189,6 +194,10 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache authResult@AuthRe r <- f pure (Nothing, r) + withOTel :: HasCallStack => Text -> Handler IO a -> Handler IO a + withOTel label = do + OTel.inSpanM (getOTelTracer appState) label defaultSpanArguments + traceHeaderMiddleware :: AppState -> Wai.Middleware traceHeaderMiddleware appState app req respond = do conf <- AppState.getConfig appState diff --git a/src/PostgREST/AppState.hs b/src/PostgREST/AppState.hs index ed2eef3fc5..1f05457727 100644 --- a/src/PostgREST/AppState.hs +++ b/src/PostgREST/AppState.hs @@ -15,6 +15,7 @@ module PostgREST.AppState , getJwtCacheState , getSocketREST , getSocketAdmin + , getOTelTracer , init , initSockets , initWithPool @@ -74,6 +75,7 @@ import PostgREST.Unix (createAndBindDomainSocket) import Data.Streaming.Network (bindPortTCP, bindRandomPortTCP) import Data.String (IsString (..)) +import OpenTelemetry.Trace (Tracer) import Protolude data AppState = AppState @@ -109,6 +111,10 @@ data AppState = AppState , stateJwtCache :: JwtCache.JwtCacheState , stateLogger :: Logger.LoggerState , stateMetrics :: Metrics.MetricsState + -- | OpenTelemetry tracer. @Nothing@ represents disabled OTel SDK. + -- It's a workaround for now, as @hs-opentelemetry-api@ doesn't have @Tracer.tracerIsEnabled@ released yet. + -- Tracking issue: https://github.com/iand675/hs-opentelemetry/issues/212 + , stateOTelTracer :: Maybe Tracer } -- | Schema cache status @@ -119,8 +125,8 @@ data SchemaCacheStatus type AppSockets = (NS.Socket, Maybe NS.Socket) -init :: AppConfig -> IO AppState -init conf@AppConfig{configLogLevel, configDbPoolSize} = do +init :: AppConfig -> Maybe Tracer -> IO AppState +init conf@AppConfig{configLogLevel, configDbPoolSize} tracer = do loggerState <- Logger.init metricsState <- Metrics.init configDbPoolSize let observer = liftA2 (>>) (Logger.observationLogger loggerState configLogLevel) (Metrics.observationMetrics metricsState) @@ -129,11 +135,11 @@ init conf@AppConfig{configLogLevel, configDbPoolSize} = do pool <- initPool conf observer (sock, adminSock) <- initSockets conf - state' <- initWithPool (sock, adminSock) pool conf loggerState metricsState observer + state' <- initWithPool (sock, adminSock) pool conf loggerState metricsState tracer observer pure state' { stateSocketREST = sock, stateSocketAdmin = adminSock} -initWithPool :: AppSockets -> SQL.Pool -> AppConfig -> Logger.LoggerState -> Metrics.MetricsState -> ObservationHandler -> IO AppState -initWithPool (sock, adminSock) pool conf loggerState metricsState observer = do +initWithPool :: AppSockets -> SQL.Pool -> AppConfig -> Logger.LoggerState -> Metrics.MetricsState -> Maybe Tracer -> ObservationHandler -> IO AppState +initWithPool (sock, adminSock) pool conf loggerState metricsState tracer observer = do appState <- AppState pool <$> newIORef minimumPgVersion -- assume we're in a supported version when starting, this will be corrected on a later step @@ -152,6 +158,7 @@ initWithPool (sock, adminSock) pool conf loggerState metricsState observer = do <*> JwtCache.init conf observer <*> pure loggerState <*> pure metricsState + <*> pure tracer deb <- let decisecond = 100000 in @@ -319,6 +326,9 @@ getSocketREST = stateSocketREST getSocketAdmin :: AppState -> Maybe NS.Socket getSocketAdmin = stateSocketAdmin +getOTelTracer :: AppState -> Maybe Tracer +getOTelTracer = stateOTelTracer + getMainThreadId :: AppState -> ThreadId getMainThreadId = stateMainThreadId diff --git a/src/PostgREST/CLI.hs b/src/PostgREST/CLI.hs index a635aa0814..8b948fcd40 100644 --- a/src/PostgREST/CLI.hs +++ b/src/PostgREST/CLI.hs @@ -13,11 +13,12 @@ import qualified Data.ByteString.Lazy as LBS import qualified Hasql.Transaction.Sessions as SQL import qualified Options.Applicative as O -import PostgREST.AppState (AppState) -import PostgREST.Config (AppConfig (..)) -import PostgREST.Observation (Observation (..)) -import PostgREST.SchemaCache (querySchemaCache) -import PostgREST.Version (prettyVersion) +import PostgREST.AppState (AppState) +import PostgREST.Config (AppConfig (..)) +import PostgREST.Observation (Observation (..)) +import PostgREST.OpenTelemetry (Tracer, withTracer) +import PostgREST.SchemaCache (querySchemaCache) +import PostgREST.Version (prettyVersion) import qualified PostgREST.App as App import qualified PostgREST.AppState as AppState @@ -26,27 +27,31 @@ import qualified PostgREST.Config as Config import Protolude +main :: HasCallStack => CLI -> IO () +main CLI{cliCommand, cliPath} = withTracer $ \tracer -> do -main :: CLI -> IO () -main CLI{cliCommand, cliPath} = do conf <- either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty mempty + let tracer' = if configServerOtelEnabled conf + then Just tracer + else Nothing + case cliCommand of Client adminCmd -> runClientCommand conf adminCmd - Run runCmd -> runAppCommand conf runCmd + Run runCmd -> runAppCommand tracer' conf runCmd -- | Run command using http-client to communicate with an already running postgrest runClientCommand :: AppConfig -> ClientCommand -> IO () runClientCommand conf CmdReady = Client.ready conf -- | Run postgrest with command -runAppCommand :: AppConfig -> RunCommand -> IO () -runAppCommand conf@AppConfig{..} runCmd = do +runAppCommand :: Maybe Tracer -> AppConfig -> RunCommand -> IO () +runAppCommand tracer conf@AppConfig{..} runCmd = do -- Per https://github.com/PostgREST/postgrest/issues/268, we want to -- explicitly close the connections to PostgreSQL on shutdown. -- 'AppState.destroy' takes care of that. bracket - (AppState.init conf) + (AppState.init conf tracer) AppState.destroy (\appState -> case runCmd of CmdDumpConfig -> do diff --git a/src/PostgREST/Config.hs b/src/PostgREST/Config.hs index a5706a74d8..b10bde9fd2 100644 --- a/src/PostgREST/Config.hs +++ b/src/PostgREST/Config.hs @@ -121,6 +121,7 @@ data AppConfig = AppConfig , configInternalSCQuerySleep :: Maybe Int32 , configInternalSCLoadSleep :: Maybe Int32 , configInternalSCRelLoadSleep :: Maybe Int32 + , configServerOtelEnabled :: Bool } data LogLevel = LogCrit | LogError | LogWarn | LogInfo | LogDebug @@ -186,6 +187,7 @@ toText conf = ,("server-port", show . configServerPort) ,("server-trace-header", q . T.decodeUtf8 . maybe mempty CI.original . configServerTraceHeader) ,("server-timing-enabled", T.toLower . show . configServerTimingEnabled) + ,("server-otel-enabled", T.toLower . show . configServerOtelEnabled) ,("server-unix-socket", q . maybe mempty T.pack . configServerUnixSocket) ,("server-unix-socket-mode", q . T.pack . showSocketMode) ,("admin-server-host", q . configAdminServerHost) @@ -309,6 +311,7 @@ parser optPath env dbSettings roleSettings roleIsolationLvl = <*> optInt "internal-schema-cache-query-sleep" <*> optInt "internal-schema-cache-load-sleep" <*> optInt "internal-schema-cache-relationship-load-sleep" + <*> (fromMaybe False <$> optBool "server-otel-enabled") where parseAppSettings :: C.Key -> C.Parser C.Config [(Text, Text)] parseAppSettings key = addFromEnv . fmap (fmap coerceText) <$> C.subassocs key C.value @@ -543,6 +546,8 @@ readDbUriFile maybeDbUri conf = type Environment = M.Map [Char] Text -- | Read environment variables that start with PGRST_ +-- Note: `OTEL_*` environment variables, while being recornized by OpenTelemetry +-- subsystem, are specifically ignored here readPGRSTEnvironment :: IO Environment readPGRSTEnvironment = M.map T.pack . M.fromList . filter (isPrefixOf "PGRST_" . fst) <$> getEnvironment diff --git a/src/PostgREST/OpenTelemetry.hs b/src/PostgREST/OpenTelemetry.hs new file mode 100644 index 0000000000..34de4dff34 --- /dev/null +++ b/src/PostgREST/OpenTelemetry.hs @@ -0,0 +1,70 @@ +{- | +Module : PostgREST.OpenTelemetry +Description : OpenTelemetry integration +Maintains the OpenTelemetry Tracer and provides a function to run +PostgREST with it. + +Basically, you want to use `withTracer` in your main function, and then +use `getOTelTracer` in your application code to get the tracer and +create spans with `inSpanM`. + +At this moment trace spans have to be explicit, by wrapping the code in `inSpanM` calls. +In order produced spans to have correct code locations, all the functions across the call stack up to the +`inSpanM` call must have `HasCallStack` constraint, because +[GHC is never inferring it](https://downloads.haskell.org/ghc/9.8.4/docs/users_guide/exts/callstack.html) for us. +-} +module PostgREST.OpenTelemetry (Tracer, withTracer, middleware, inSpanM) where + +import Control.Monad.Catch (MonadMask) +import Network.Wai (Middleware) +import OpenTelemetry.Attributes (emptyAttributes) +import OpenTelemetry.Instrumentation.Wai (newOpenTelemetryWaiMiddleware') +import OpenTelemetry.Trace (InstrumentationLibrary (..), + SpanArguments, Tracer, + initializeGlobalTracerProvider, + makeTracer, + shutdownTracerProvider, + tracerOptions) +import OpenTelemetry.Trace.Core (getTracerTracerProvider) +import OpenTelemetry.Utils.Exceptions (inSpanM'') +import PostgREST.AppState (AppState, getOTelTracer) +import PostgREST.Version (prettyVersion) +import Protolude + +{- | Wrap user's code with OpenTelemetry Tracer, initializing it with sensible defaults -} +withTracer :: (Tracer -> IO c) -> IO c +withTracer f = bracket + initializeGlobalTracerProvider + shutdownTracerProvider + (\tracerProvider -> f $ makeTracer tracerProvider instrumentationLibrary tracerOptions) + where + instrumentationLibrary = + InstrumentationLibrary + { libraryName = "PostgREST" + , libraryVersion = decodeUtf8 prettyVersion + , librarySchemaUrl = "" + , libraryAttributes = emptyAttributes} + +middleware :: AppState -> Network.Wai.Middleware +middleware s = case getOTelTracer s of + Just t -> newOpenTelemetryWaiMiddleware' $ getTracerTracerProvider t + -- Make sure OTel code is actually noop + Nothing -> identity + +-- | The simplest function for annotating code with trace information. +-- In case the tracer is @Nothing@, i.e. disabled, the function is noop. +inSpanM + :: (MonadIO m, MonadMask m, HasCallStack) + => Maybe Tracer + -> Text + -- ^ The name of the span. This may be updated later via 'updateName' + -> SpanArguments + -- ^ Additional options for creating the span, such as 'SpanKind', + -- span links, starting attributes, etc. + -> m a + -- ^ The action to perform. 'inSpan' will record the time spent on the + -- action without forcing strict evaluation of the result. Any uncaught + -- exceptions will be recorded and rethrown. + -> m a +inSpanM (Just t) n args m = inSpanM'' t callStack n args (const m) +inSpanM Nothing _ _ m = m diff --git a/stack.yaml b/stack.yaml index c88035cee7..758a35fee7 100644 --- a/stack.yaml +++ b/stack.yaml @@ -16,5 +16,18 @@ extra-deps: - postgresql-libpq-0.10.1.0 - streaming-commons-0.2.3.1 + - hs-opentelemetry-sdk-0.1.0.0@sha256:2642851866f11a494c99f15202d4bd9e75d4a5e1a7f3f172742a0676a33c664f,4059 + - hs-opentelemetry-api-0.2.0.0@sha256:bbdbe7e212e99f17a7e68d09b94c1a6613e50ce88b3cb1b68979bbb0221291ae,4051 + - hs-opentelemetry-exporter-otlp-0.1.0.0@sha256:4c908a7e2e5053879687b7a7ee6e40a8eb22868e1a0808cd0cfd6ac9905057b8,1526 + - hs-opentelemetry-instrumentation-wai-0.1.1.0@sha256:d97b4cb3870217e64e95da3f51db814eca62eb57484ee0a6f747366da5940bc2,1371 + - hs-opentelemetry-propagator-b3-0.0.1.2@sha256:8815dd74f27a908b5be0729cc09a3bf9f3049481c982252bbd6c3f6b908ecfcd,1340 + - hs-opentelemetry-propagator-datadog-0.0.1.0@sha256:c85de95e3c33b3ffcf980f560166e960cab0888e0741315f487288b3653c007c,2950 + - hs-opentelemetry-propagator-w3c-0.0.1.4@sha256:251428754454fbaf71d9b6acbbea473014b1ab50bdcda8bc8fe1532e63193374,1382 + - hs-opentelemetry-utils-exceptions-0.2.0.1@sha256:b32c3109b896dbab67c74c28e8ffcfe6e7f86aa29454fc6a31c06a671246e78b,1477 + - hs-opentelemetry-otlp-0.1.0.0@sha256:5cd096b15f26f51ffae4c18f6a26794daef801acc9e13033db8b21a7606336d4,2533 + - hs-opentelemetry-instrumentation-hspec-0.0.1.2@sha256:cba36dc9a8fed4288a1b9d071b869f5b3382451fe37821e369628e2761834fb4,1191 + - thread-utils-context-0.3.0.4@sha256:e763da1c6cab3b6d378fb670ca74aa9bf03c9b61b6fcf7628c56363fb0e3e71e,1671 + - thread-utils-finalizers-0.1.1.0@sha256:24944b71d9f1d01695a5908b4a3b44838fab870883114a323336d537995e0a5b,1381 + # fix build with GCC 15-ish; https://github.com/gregorycollins/hashtables/issues/98 for details - hashtables-1.4.2@sha256:4940cab94a15d469845ccf5225f9cb3d354c15e8127ebb58425c8b681f7721d9,10386 diff --git a/stack.yaml.lock b/stack.yaml.lock index ae2cbfd7ce..0f875245c9 100644 --- a/stack.yaml.lock +++ b/stack.yaml.lock @@ -46,6 +46,90 @@ packages: size: 2374 original: hackage: streaming-commons-0.2.3.1 +- completed: + hackage: hs-opentelemetry-sdk-0.1.0.0@sha256:2642851866f11a494c99f15202d4bd9e75d4a5e1a7f3f172742a0676a33c664f,4059 + pantry-tree: + sha256: c0868a6eb3d6add84df1ad32cdb0ebdbebe41205897e16ae8b30e96f205a8fe0 + size: 1934 + original: + hackage: hs-opentelemetry-sdk-0.1.0.0@sha256:2642851866f11a494c99f15202d4bd9e75d4a5e1a7f3f172742a0676a33c664f,4059 +- completed: + hackage: hs-opentelemetry-api-0.2.0.0@sha256:bbdbe7e212e99f17a7e68d09b94c1a6613e50ce88b3cb1b68979bbb0221291ae,4051 + pantry-tree: + sha256: fcb11b19fa633afb8c34e002e6b8e8927d20fc2332d4234cc10a0b6e3dbe6022 + size: 4396 + original: + hackage: hs-opentelemetry-api-0.2.0.0@sha256:bbdbe7e212e99f17a7e68d09b94c1a6613e50ce88b3cb1b68979bbb0221291ae,4051 +- completed: + hackage: hs-opentelemetry-exporter-otlp-0.1.0.0@sha256:4c908a7e2e5053879687b7a7ee6e40a8eb22868e1a0808cd0cfd6ac9905057b8,1526 + pantry-tree: + sha256: dd22c915f65b1ca76c6130cfb39ce666376d4813c267e12dd59be61a914bb264 + size: 511 + original: + hackage: hs-opentelemetry-exporter-otlp-0.1.0.0@sha256:4c908a7e2e5053879687b7a7ee6e40a8eb22868e1a0808cd0cfd6ac9905057b8,1526 +- completed: + hackage: hs-opentelemetry-instrumentation-wai-0.1.1.0@sha256:d97b4cb3870217e64e95da3f51db814eca62eb57484ee0a6f747366da5940bc2,1371 + pantry-tree: + sha256: 23bbd4e58ba48b0ec3541a494d02e08e6b934d7173523be8aab04c6b2c7bb98b + size: 360 + original: + hackage: hs-opentelemetry-instrumentation-wai-0.1.1.0@sha256:d97b4cb3870217e64e95da3f51db814eca62eb57484ee0a6f747366da5940bc2,1371 +- completed: + hackage: hs-opentelemetry-propagator-b3-0.0.1.2@sha256:8815dd74f27a908b5be0729cc09a3bf9f3049481c982252bbd6c3f6b908ecfcd,1340 + pantry-tree: + sha256: fc71f8b7dc25625af6b81c1b3c1c5d808b682e2a7c1daf8e23f2af45ab9dc123 + size: 431 + original: + hackage: hs-opentelemetry-propagator-b3-0.0.1.2@sha256:8815dd74f27a908b5be0729cc09a3bf9f3049481c982252bbd6c3f6b908ecfcd,1340 +- completed: + hackage: hs-opentelemetry-propagator-datadog-0.0.1.0@sha256:c85de95e3c33b3ffcf980f560166e960cab0888e0741315f487288b3653c007c,2950 + pantry-tree: + sha256: 04c10d8901e506c8c7662c8ce549a152118303fd2d0354a887bede4e73f0a8ee + size: 730 + original: + hackage: hs-opentelemetry-propagator-datadog-0.0.1.0@sha256:c85de95e3c33b3ffcf980f560166e960cab0888e0741315f487288b3653c007c,2950 +- completed: + hackage: hs-opentelemetry-propagator-w3c-0.0.1.4@sha256:251428754454fbaf71d9b6acbbea473014b1ab50bdcda8bc8fe1532e63193374,1382 + pantry-tree: + sha256: 5f7ff3fd37b7f720064193f02c84e8af6b554f8a7a2b7702a4bcd34fe576f721 + size: 445 + original: + hackage: hs-opentelemetry-propagator-w3c-0.0.1.4@sha256:251428754454fbaf71d9b6acbbea473014b1ab50bdcda8bc8fe1532e63193374,1382 +- completed: + hackage: hs-opentelemetry-utils-exceptions-0.2.0.1@sha256:b32c3109b896dbab67c74c28e8ffcfe6e7f86aa29454fc6a31c06a671246e78b,1477 + pantry-tree: + sha256: 7829e2f06282a2ca913ab46ac98a3dd5b0b89b1189d5eb071f250b641115e548 + size: 406 + original: + hackage: hs-opentelemetry-utils-exceptions-0.2.0.1@sha256:b32c3109b896dbab67c74c28e8ffcfe6e7f86aa29454fc6a31c06a671246e78b,1477 +- completed: + hackage: hs-opentelemetry-otlp-0.1.0.0@sha256:5cd096b15f26f51ffae4c18f6a26794daef801acc9e13033db8b21a7606336d4,2533 + pantry-tree: + sha256: 618a513764a7ae9995fc4f8b8ee5cec731a8759ac8c5df8e9553171abd3ff97d + size: 2585 + original: + hackage: hs-opentelemetry-otlp-0.1.0.0@sha256:5cd096b15f26f51ffae4c18f6a26794daef801acc9e13033db8b21a7606336d4,2533 +- completed: + hackage: hs-opentelemetry-instrumentation-hspec-0.0.1.2@sha256:cba36dc9a8fed4288a1b9d071b869f5b3382451fe37821e369628e2761834fb4,1191 + pantry-tree: + sha256: 04ca1dab12e22a1a43f491568674470e49ad29a4d2796af87c41a6e354c49293 + size: 365 + original: + hackage: hs-opentelemetry-instrumentation-hspec-0.0.1.2@sha256:cba36dc9a8fed4288a1b9d071b869f5b3382451fe37821e369628e2761834fb4,1191 +- completed: + hackage: thread-utils-context-0.3.0.4@sha256:e763da1c6cab3b6d378fb670ca74aa9bf03c9b61b6fcf7628c56363fb0e3e71e,1671 + pantry-tree: + sha256: 57d909a991b5e0b4c7a28121cb52ee9c2db6c09e0419b89af6c82fae52be88d4 + size: 397 + original: + hackage: thread-utils-context-0.3.0.4@sha256:e763da1c6cab3b6d378fb670ca74aa9bf03c9b61b6fcf7628c56363fb0e3e71e,1671 +- completed: + hackage: thread-utils-finalizers-0.1.1.0@sha256:24944b71d9f1d01695a5908b4a3b44838fab870883114a323336d537995e0a5b,1381 + pantry-tree: + sha256: 8c2c2e2e22c20bf3696ee6f30b50b3a9eeae187a22beb536441eefb0a3f9c549 + size: 400 + original: + hackage: thread-utils-finalizers-0.1.1.0@sha256:24944b71d9f1d01695a5908b4a3b44838fab870883114a323336d537995e0a5b,1381 - completed: hackage: hashtables-1.4.2@sha256:4940cab94a15d469845ccf5225f9cb3d354c15e8127ebb58425c8b681f7721d9,10386 pantry-tree: diff --git a/test/io/configs/expected/aliases.config b/test/io/configs/expected/aliases.config index 0655e5c4b3..492f37b917 100644 --- a/test/io/configs/expected/aliases.config +++ b/test/io/configs/expected/aliases.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/expected/boolean-numeric.config b/test/io/configs/expected/boolean-numeric.config index 53a13a7b81..f09684846a 100644 --- a/test/io/configs/expected/boolean-numeric.config +++ b/test/io/configs/expected/boolean-numeric.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/expected/boolean-string.config b/test/io/configs/expected/boolean-string.config index 53a13a7b81..f09684846a 100644 --- a/test/io/configs/expected/boolean-string.config +++ b/test/io/configs/expected/boolean-string.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/expected/defaults.config b/test/io/configs/expected/defaults.config index 87909425e8..dffdd15f71 100644 --- a/test/io/configs/expected/defaults.config +++ b/test/io/configs/expected/defaults.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/expected/jwt-role-claim-key1.config b/test/io/configs/expected/jwt-role-claim-key1.config index 319a4932c5..4ad318be0b 100644 --- a/test/io/configs/expected/jwt-role-claim-key1.config +++ b/test/io/configs/expected/jwt-role-claim-key1.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/expected/jwt-role-claim-key2.config b/test/io/configs/expected/jwt-role-claim-key2.config index 535dc9e248..9584590cc5 100644 --- a/test/io/configs/expected/jwt-role-claim-key2.config +++ b/test/io/configs/expected/jwt-role-claim-key2.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/expected/jwt-role-claim-key3.config b/test/io/configs/expected/jwt-role-claim-key3.config index c052f2dfc3..2f13a835da 100644 --- a/test/io/configs/expected/jwt-role-claim-key3.config +++ b/test/io/configs/expected/jwt-role-claim-key3.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/expected/jwt-role-claim-key4.config b/test/io/configs/expected/jwt-role-claim-key4.config index a3f8e8df5d..c02c306450 100644 --- a/test/io/configs/expected/jwt-role-claim-key4.config +++ b/test/io/configs/expected/jwt-role-claim-key4.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/expected/jwt-role-claim-key5.config b/test/io/configs/expected/jwt-role-claim-key5.config index 0cfcd55da9..c3263534fb 100644 --- a/test/io/configs/expected/jwt-role-claim-key5.config +++ b/test/io/configs/expected/jwt-role-claim-key5.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/expected/no-defaults-with-db-other-authenticator.config b/test/io/configs/expected/no-defaults-with-db-other-authenticator.config index ebb54e0ea6..37d66da61b 100644 --- a/test/io/configs/expected/no-defaults-with-db-other-authenticator.config +++ b/test/io/configs/expected/no-defaults-with-db-other-authenticator.config @@ -34,6 +34,7 @@ server-host = "0.0.0.0" server-port = 80 server-trace-header = "traceparent" server-timing-enabled = true +server-otel-enabled = true server-unix-socket = "/tmp/pgrst_io_test.sock" server-unix-socket-mode = "777" admin-server-host = "127.0.0.1" diff --git a/test/io/configs/expected/no-defaults-with-db.config b/test/io/configs/expected/no-defaults-with-db.config index 9077fdbde0..94498d572e 100644 --- a/test/io/configs/expected/no-defaults-with-db.config +++ b/test/io/configs/expected/no-defaults-with-db.config @@ -34,6 +34,7 @@ server-host = "0.0.0.0" server-port = 80 server-trace-header = "CF-Ray" server-timing-enabled = false +server-otel-enabled = true server-unix-socket = "/tmp/pgrst_io_test.sock" server-unix-socket-mode = "777" admin-server-host = "127.0.0.1" diff --git a/test/io/configs/expected/no-defaults.config b/test/io/configs/expected/no-defaults.config index 2b0ab43a96..77f0cfab80 100644 --- a/test/io/configs/expected/no-defaults.config +++ b/test/io/configs/expected/no-defaults.config @@ -34,6 +34,7 @@ server-host = "0.0.0.0" server-port = 80 server-trace-header = "X-Request-Id" server-timing-enabled = true +server-otel-enabled = true server-unix-socket = "/tmp/pgrst_io_test.sock" server-unix-socket-mode = "777" admin-server-host = "127.0.0.1" diff --git a/test/io/configs/expected/types.config b/test/io/configs/expected/types.config index cb474bcddd..73444b51d9 100644 --- a/test/io/configs/expected/types.config +++ b/test/io/configs/expected/types.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/expected/utf-8.config b/test/io/configs/expected/utf-8.config index dc8226011d..fdf0a2951c 100644 --- a/test/io/configs/expected/utf-8.config +++ b/test/io/configs/expected/utf-8.config @@ -34,6 +34,7 @@ server-host = "!4" server-port = 3000 server-trace-header = "" server-timing-enabled = false +server-otel-enabled = false server-unix-socket = "" server-unix-socket-mode = "660" admin-server-host = "!4" diff --git a/test/io/configs/no-defaults-env.yaml b/test/io/configs/no-defaults-env.yaml index bda8e9b281..c42736be96 100644 --- a/test/io/configs/no-defaults-env.yaml +++ b/test/io/configs/no-defaults-env.yaml @@ -37,6 +37,7 @@ PGRST_SERVER_HOST: 0.0.0.0 PGRST_SERVER_PORT: 80 PGRST_SERVER_TRACE_HEADER: X-Request-Id PGRST_SERVER_TIMING_ENABLED: true +PGRST_SERVER_OTEL_ENABLED: true PGRST_SERVER_UNIX_SOCKET: /tmp/pgrst_io_test.sock PGRST_SERVER_UNIX_SOCKET_MODE: 777 PGRST_ADMIN_SERVER_HOST: 127.0.0.1 diff --git a/test/io/configs/no-defaults.config b/test/io/configs/no-defaults.config index ceeb5dbdb4..fed0f1f5cb 100644 --- a/test/io/configs/no-defaults.config +++ b/test/io/configs/no-defaults.config @@ -34,6 +34,7 @@ server-host = "0.0.0.0" server-port = 80 server-trace-header = "X-Request-Id" server-timing-enabled = true +server-otel-enabled = true server-unix-socket = "/tmp/pgrst_io_test.sock" server-unix-socket-mode = "777" admin-server-port = 3001 diff --git a/test/io/test_cli.py b/test/io/test_cli.py index 8135fcce21..2f2d0a249a 100644 --- a/test/io/test_cli.py +++ b/test/io/test_cli.py @@ -405,3 +405,11 @@ def test_cli_ready_flag_fail_when_no_admin_server(defaultenv): "ERROR: Admin server is not running. Please check admin-server-port config." in output ) + + +# def test_cli_ready_flag_success_server_otel_enabled(defaultenv): +# """Test PostgREST ready when server-otel-enabled=true""" +# +# env = {**defaultenv, "PGRST_SERVER_OTEL_ENABLED": "true"} +# with run(env) as postgrest: +# diff --git a/test/spec/Main.hs b/test/spec/Main.hs index e847926b6f..aa859e612d 100644 --- a/test/spec/Main.hs +++ b/test/spec/Main.hs @@ -6,11 +6,14 @@ import qualified Hasql.Transaction.Sessions as HT import Data.Function (id) +import OpenTelemetry.Context.ThreadLocal (getContext) +import OpenTelemetry.Instrumentation.Hspec import Test.Hspec import PostgREST.App (postgrest) import PostgREST.Config (AppConfig (..)) import PostgREST.Config.Database (queryPgVersion) +import PostgREST.OpenTelemetry (withTracer) import PostgREST.SchemaCache (querySchemaCache) import Protolude hiding (toList, toS) import SpecHelper @@ -71,7 +74,7 @@ import qualified Feature.RpcPreRequestGucsSpec main :: IO () -main = do +main = withTracer $ \tracer -> do pool <- P.acquire $ P.settings [ P.size 3 , P.acquisitionTimeout 10 @@ -90,7 +93,10 @@ main = do let initApp sCache st config = do - appState <- AppState.initWithPool sockets pool config loggerState metricsState (Metrics.observationMetrics metricsState) + let tracer' = if configServerOtelEnabled config + then Just tracer + else Nothing + appState <- AppState.initWithPool sockets pool config loggerState metricsState tracer' (Metrics.observationMetrics metricsState) AppState.putPgVersion appState actualPgVersion AppState.putSchemaCache appState (Just sCache) return (st, postgrest (configLogLevel config) appState (pure ())) @@ -166,7 +172,8 @@ main = do , ("Feature.Query.UpsertSpec" , Feature.Query.UpsertSpec.spec) ] - hspec $ do + ctxt <- getContext + hspec $ instrumentSpec tracer ctxt $ do mapM_ (parallel . before withApp) specs -- we analyze to get accurate results from EXPLAIN diff --git a/test/spec/SpecHelper.hs b/test/spec/SpecHelper.hs index 3c48a1134d..9fe3ed0a99 100644 --- a/test/spec/SpecHelper.hs +++ b/test/spec/SpecHelper.hs @@ -161,6 +161,7 @@ baseCfg = let secret = encodeUtf8 "reallyreallyreallyreallyverysafe" in , configInternalSCLoadSleep = Nothing , configInternalSCRelLoadSleep = Nothing , configServerTimingEnabled = True + , configServerOtelEnabled = False } testCfg :: AppConfig