Add configurable backtrace_additional_directories option

CHANGELOG.markdown

@@ -1,5 +1,8 @@
+# Unreleased
+- Add `backtrace_additional_directories` config option to allow additional directories to be included in the backtrace parser
+
 # 6.0.2
-- Fix `endpoint_sample_rate` and `job_sample_rate` to support float values 
+- Fix `endpoint_sample_rate` and `job_sample_rate` to support float values
 
 # 6.0.1
 - Fix capturing of job params for non ActiveJob
@@ -17,7 +20,7 @@ sampled at exactly 1%, you must now set `0.01` instead of `1`.**
 - Add HTTPX instrumentation (#588)
 - Add ability to automatically capture Sidekiq job args as context
   - `job_params_capture` - Set to true to enable job argument capturing
-  - `job_params_filter`  - A list of arguments to filter (automatically includes Rails filtered_parameters) 
+  - `job_params_filter`  - A list of arguments to filter (automatically includes Rails filtered_parameters)
 - Fix user error context being incorrectly flattened (#581)
 - Handle Delayed Job PerformableMethod jobs for error tracking (#584)
 - Require 'httpclient' library on instrumentation install (#586)

lib/scout_apm/config.rb

@@ -120,6 +120,7 @@ class Config
         'use_prepend',
         'alias_method_instruments',
         'prepend_instruments',
+        'backtrace_additional_directories',
 
         # Error Service Related Configuration
         'errors_enabled',
@@ -267,6 +268,7 @@ def coerce(val)
       'errors_ignored_exceptions' => JsonCoercion.new,
       'errors_filtered_params' => JsonCoercion.new,
       'errors_env_capture' => JsonCoercion.new,
+      'backtrace_additional_directories' => JsonCoercion.new,
     }
 
 
@@ -404,6 +406,7 @@ class ConfigDefaults
         'errors_filtered_params'               => %w(password s3-key),
         'errors_env_capture'                   => %w(),
         'errors_host'                          => 'https://errors.scoutapm.com',
+        'backtrace_additional_directories'     => [],
       }.freeze
 
       def value(key)

lib/scout_apm/utils/backtrace_parser.rb

@@ -11,16 +11,20 @@ class BacktraceParser
 
       attr_reader :call_stack
 
-      # call_stack - an +Array+ of calls, typically generated via the +caller+ method. 
-      # Example single line: 
+      # call_stack - an +Array+ of calls, typically generated via the +caller+ method.
+      # Example single line:
       # "/Users/dlite/.rvm/rubies/ruby-2.4.5/lib/ruby/2.4.0/irb/workspace.rb:87:in `eval'"
-      def initialize(call_stack, root=ScoutApm::Agent.instance.context.environment.root)
+      # additional_dirs - an +Array+ of additional directory names to include beyond lib/, app/, and config/
+      def initialize(call_stack, root=ScoutApm::Agent.instance.context.environment.root, additional_dirs=ScoutApm::Agent.instance.context.config.value('backtrace_additional_directories'))
         @call_stack = call_stack
         # We can't use a constant as it'd be too early to fetch environment info
         #
         # This regex looks for files under the app root, inside lib/, app/, and
-        # config/ dirs, and captures the path under root.
-        @@app_dir_regex = %r[#{root}/((?:lib/|app/|config/).*)]
+        # config/ dirs (plus any additional configured directories), and captures the path under root.
+        base_dirs = %w[lib app config]
+        all_dirs = base_dirs + Array(additional_dirs)
+        dir_pattern = all_dirs.map { |d| Regexp.escape(d) + "/" }.join("|")
+        @@app_dir_regex = %r[#{root}/((?:#{dir_pattern}).*)]
       end
 
       def call

test/unit/utils/backtrace_parser_test.rb

@@ -68,4 +68,116 @@ def test_excludes_vendor_paths
     assert_equal false, (result[0] =~ %r|app/controllers/users_controller.rb|).nil?
     assert_equal false, (result[1] =~ %r|config/initializers/inject_something.rb|).nil?
   end
+
+  ################################################################################
+  # Additional directories tests
+
+  def test_with_empty_additional_directories
+    raw_backtrace = [
+      "#{root}/app/controllers/users_controller.rb",
+      "#{root}/engines/my_engine/app/models/thing.rb",
+    ]
+    result = ScoutApm::Utils::BacktraceParser.new(raw_backtrace, root, []).call
+
+    assert_equal 1, result.length
+    assert_equal false, (result[0] =~ %r|app/controllers/users_controller.rb|).nil?
+  end
+
+  def test_with_single_additional_directory
+    raw_backtrace = [
+      "#{root}/engines/my_engine/app/models/thing.rb",
+      "#{root}/app/controllers/users_controller.rb",
+    ]
+    result = ScoutApm::Utils::BacktraceParser.new(raw_backtrace, root, ['engines']).call
+
+    assert_equal 2, result.length
+    assert_equal false, (result[0] =~ %r|engines/my_engine/app/models/thing.rb|).nil?
+    assert_equal false, (result[1] =~ %r|app/controllers/users_controller.rb|).nil?
+  end
+
+  def test_with_multiple_additional_directories
+    raw_backtrace = [
+      "#{root}/foo/something.rb",
+      "#{root}/bar/something_else.rb",
+      "#{root}/app/controllers/users_controller.rb",
+    ]
+    result = ScoutApm::Utils::BacktraceParser.new(raw_backtrace, root, ['foo', 'bar']).call
+
+    assert_equal 3, result.length
+    assert_equal false, (result[0] =~ %r|foo/something.rb|).nil?
+    assert_equal false, (result[1] =~ %r|bar/something_else.rb|).nil?
+    assert_equal false, (result[2] =~ %r|app/controllers/users_controller.rb|).nil?
+  end
+
+  def test_default_directories_still_work_with_additional_dirs
+    raw_backtrace = [
+      "#{root}/lib/utilities.rb",
+      "#{root}/app/models/user.rb",
+      "#{root}/config/initializers/setup.rb",
+      "#{root}/engines/core/lib/core.rb",
+    ]
+    result = ScoutApm::Utils::BacktraceParser.new(raw_backtrace, root, ['engines']).call
+
+    assert_equal 4, result.length
+    assert_equal false, (result[0] =~ %r|lib/utilities.rb|).nil?
+    assert_equal false, (result[1] =~ %r|app/models/user.rb|).nil?
+    assert_equal false, (result[2] =~ %r|config/initializers/setup.rb|).nil?
+    assert_equal false, (result[3] =~ %r|engines/core/lib/core.rb|).nil?
+  end
+
+  def test_additional_directory_with_special_regex_characters
+    raw_backtrace = [
+      "#{root}/my.engine/lib/something.rb",
+      "#{root}/app/controllers/users_controller.rb",
+    ]
+    result = ScoutApm::Utils::BacktraceParser.new(raw_backtrace, root, ['my.engine']).call
+
+    assert_equal 2, result.length
+    assert_equal false, (result[0] =~ %r|my\.engine/lib/something.rb|).nil?
+    assert_equal false, (result[1] =~ %r|app/controllers/users_controller.rb|).nil?
+  end
+
+  def test_additional_directory_does_not_match_similar_names
+    # Ensure "my.engine" doesn't match "myXengine" (the dot should be escaped)
+    raw_backtrace = [
+      "#{root}/myXengine/lib/something.rb",
+      "#{root}/app/controllers/users_controller.rb",
+    ]
+    result = ScoutApm::Utils::BacktraceParser.new(raw_backtrace, root, ['my.engine']).call
+
+    assert_equal 1, result.length
+    assert_equal false, (result[0] =~ %r|app/controllers/users_controller.rb|).nil?
+  end
+
+  def test_backtrace_entirely_within_single_additional_directory
+    # Backtrace originates entirely from an additional directory with no default dirs
+    raw_backtrace = [
+      "#{root}/engines/core/lib/core/base.rb",
+      "#{root}/engines/core/app/models/engine_model.rb",
+      "#{root}/engines/auth/lib/auth/strategy.rb",
+      "#{root}/vendor/bundle/gems/some_gem.rb",
+    ]
+    result = ScoutApm::Utils::BacktraceParser.new(raw_backtrace, root, ['engines']).call
+
+    assert_equal 3, result.length
+    assert_equal false, (result[0] =~ %r|engines/core/lib/core/base.rb|).nil?
+    assert_equal false, (result[1] =~ %r|engines/core/app/models/engine_model.rb|).nil?
+    assert_equal false, (result[2] =~ %r|engines/auth/lib/auth/strategy.rb|).nil?
+  end
+
+  def test_backtrace_entirely_within_multiple_additional_directories
+    # Backtrace spans multiple additional directories with no default dirs
+    raw_backtrace = [
+      "#{root}/engines/billing/lib/invoice.rb",
+      "#{root}/components/shared/helpers.rb",
+      "#{root}/plugins/analytics/tracker.rb",
+      "#{root}/vendor/bundle/gems/external.rb",
+    ]
+    result = ScoutApm::Utils::BacktraceParser.new(raw_backtrace, root, ['engines', 'components', 'plugins']).call
+
+    assert_equal 3, result.length
+    assert_equal false, (result[0] =~ %r|engines/billing/lib/invoice.rb|).nil?
+    assert_equal false, (result[1] =~ %r|components/shared/helpers.rb|).nil?
+    assert_equal false, (result[2] =~ %r|plugins/analytics/tracker.rb|).nil?
+  end
 end