diff --git a/.buildscript/e2e.sh b/.buildscript/e2e.sh new file mode 100755 index 00000000..a634eda1 --- /dev/null +++ b/.buildscript/e2e.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +set -ex + +if [ "$RUN_E2E_TESTS" != "true" ]; then + echo "Skipping end to end tests." +else + echo "Running end to end tests..." + wget -q https://github.com/segmentio/library-e2e-tester/releases/download/0.4.0/tester_linux_amd64 -O tester + chmod +x tester + ./tester -path='./bin/analytics' + echo "End to end tests completed!" +fi diff --git a/.github/workflows/create_jira.yml b/.github/workflows/create_jira.yml new file mode 100644 index 00000000..8180ac0f --- /dev/null +++ b/.github/workflows/create_jira.yml @@ -0,0 +1,39 @@ +name: Create Jira Ticket + +on: + issues: + types: + - opened + +jobs: + create_jira: + name: Create Jira Ticket + runs-on: ubuntu-latest + environment: IssueTracker + steps: + - name: Checkout + uses: actions/checkout@master + - name: Login + uses: atlassian/gajira-login@master + env: + JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} + JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} + JIRA_API_TOKEN: ${{ secrets.JIRA_TOKEN }} + JIRA_EPIC_KEY: ${{ secrets.JIRA_EPIC_KEY }} + JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }} + + - name: Create + id: create + uses: atlassian/gajira-create@master + with: + project: ${{ secrets.JIRA_PROJECT }} + issuetype: Bug + summary: | + [${{ github.event.repository.name }}] (${{ github.event.issue.number }}): ${{ github.event.issue.title }} + description: | + Github Link: ${{ github.event.issue.html_url }} + ${{ github.event.issue.body }} + fields: '{"parent": {"key": "${{ secrets.JIRA_EPIC_KEY }}"}}' + + - name: Log created issue + run: echo "Issue ${{ steps.create.outputs.issue }} was created" \ No newline at end of file diff --git a/.github/workflows/gem-push.yml b/.github/workflows/gem-push.yml new file mode 100644 index 00000000..5deca817 --- /dev/null +++ b/.github/workflows/gem-push.yml @@ -0,0 +1,30 @@ +name: Ruby Gem + +on: + workflow_dispatch: + +jobs: + build: + name: Build + Publish + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v4 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.7.7 + + - name: Publish to RubyGems + run: | + mkdir -p $HOME/.gem + touch $HOME/.gem/credentials + chmod 0600 $HOME/.gem/credentials + printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials + gem build *.gemspec + gem push *.gem + env: + GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}" diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml new file mode 100644 index 00000000..99831ccd --- /dev/null +++ b/.github/workflows/ruby.yml @@ -0,0 +1,36 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake +# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby + +name: Ruby + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: ['2.4', '2.5', '2.6', '2.7', '3.0', '3.1', '3.2'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Run tests + run: bundle exec rake + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.2.0 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 83f5868f..ab09e0c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.gem Gemfile.lock .ruby-version +coverage/ diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 00000000..ecabcc43 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,84 @@ +inherit_from: .rubocop_todo.yml + +AllCops: + TargetRubyVersion: '2.0' + SuggestExtensions: false + NewCops: disable + +Layout/FirstHashElementIndentation: + EnforcedStyle: consistent + +Metrics/AbcSize: + Exclude: + - "spec/**/*.rb" + +Metrics/BlockLength: + Exclude: + - "spec/**/*.rb" + +Metrics/ClassLength: + Exclude: + - "spec/**/*.rb" + +Metrics/CyclomaticComplexity: + Exclude: + - "spec/**/*.rb" + +Layout/LineLength: + Exclude: + - "spec/**/*.rb" + +Metrics/MethodLength: + Exclude: + - "spec/**/*.rb" + +Metrics/PerceivedComplexity: + Exclude: + - "spec/**/*.rb" + +Naming/FileName: + Exclude: + - lib/analytics-ruby.rb # Gem name, added for easier Gemfile usage + +Naming/PredicateName: + AllowedMethods: + - is_requesting? # Can't be renamed, backwards compatibility + +Style/BlockDelimiters: + Exclude: + - 'spec/**/*' + +Style/DateTime: + Exclude: + - 'spec/**/*.rb' + +Style/Documentation: + Enabled: false + +Style/FormatString: + EnforcedStyle: percent + +# Allow one-liner functions to be wrapped in conditionals rather +# than forcing a guard clause +Style/GuardClause: + MinBodyLength: 2 + +Style/HashSyntax: + EnforcedStyle: hash_rockets + Exclude: + - 'spec/**/*.rb' + +Style/ModuleFunction: + Enabled: false + +Style/MutableConstant: + Enabled: false + +Style/NumericLiterals: + MinDigits: 6 + +Style/ParallelAssignment: + Enabled: false + +Style/PreferredHashMethods: + EnforcedStyle: verbose diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml new file mode 100644 index 00000000..c02e1294 --- /dev/null +++ b/.rubocop_todo.yml @@ -0,0 +1,170 @@ +# This configuration was generated by +# `rubocop --auto-gen-config` +# on 2023-01-24 09:12:04 UTC using RuboCop version 1.44.0. +# The point is for the user to remove these configuration records +# one by one as the offenses are removed from the code base. +# Note that changes in the inspected code, or installation of new +# versions of RuboCop, may require this file to be generated again. + +# Offense count: 2 +# This cop supports safe autocorrection (--autocorrect). +# Configuration parameters: TreatCommentsAsGroupSeparators, ConsiderPunctuation, Include. +# Include: **/*.gemspec +Gemspec/OrderedDependencies: + Exclude: + - 'analytics-ruby.gemspec' + +# Offense count: 1 +# Configuration parameters: Severity, Include. +# Include: **/*.gemspec +Gemspec/RubyVersionGlobalsUsage: + Exclude: + - 'analytics-ruby.gemspec' + +# Offense count: 4 +# This cop supports safe autocorrection (--autocorrect). +Layout/EmptyLineAfterGuardClause: + Exclude: + - 'lib/segment/analytics/client.rb' + - 'spec/spec_helper.rb' + +# Offense count: 3 +# This cop supports safe autocorrection (--autocorrect). +# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle. +# SupportedHashRocketStyles: key, separator, table +# SupportedColonStyles: key, separator, table +# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit +Layout/HashAlignment: + Exclude: + - 'spec/segment/analytics/client_spec.rb' + - 'spec/spec_helper.rb' + +# Offense count: 1 +# This cop supports safe autocorrection (--autocorrect). +Layout/SpaceAfterComma: + Exclude: + - 'Rakefile' + +# Offense count: 1 +# Configuration parameters: AllowedMethods. +# AllowedMethods: enums +Lint/ConstantDefinitionInBlock: + Exclude: + - 'bin/analytics' + +# Offense count: 2 +# This cop supports safe autocorrection (--autocorrect). +# Configuration parameters: IgnoreEmptyBlocks, AllowUnusedKeywordArguments. +Lint/UnusedBlockArgument: + Exclude: + - 'bin/analytics' + +# Offense count: 3 +# Configuration parameters: AllowedMethods, AllowedPatterns, IgnoredMethods, CountRepeatedAttributes. +Metrics/AbcSize: + Max: 25 + +# Offense count: 3 +# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, AllowedMethods, AllowedPatterns, IgnoredMethods. +# AllowedMethods: refine +Metrics/BlockLength: + Max: 76 + +# Offense count: 1 +# Configuration parameters: CountComments, CountAsOne. +Metrics/ClassLength: + Max: 115 + +# Offense count: 2 +# Configuration parameters: AllowedMethods, AllowedPatterns, IgnoredMethods. +Metrics/CyclomaticComplexity: + Max: 8 + +# Offense count: 11 +# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, AllowedMethods, AllowedPatterns, IgnoredMethods. +Metrics/MethodLength: + Max: 16 + +# Offense count: 1 +# This cop supports safe autocorrection (--autocorrect). +# Configuration parameters: PreferredName. +Naming/RescuedExceptionsVariableName: + Exclude: + - 'spec/spec_helper.rb' + +# Offense count: 1 +# This cop supports safe autocorrection (--autocorrect). +Style/ExpandPathArguments: + Exclude: + - 'analytics-ruby.gemspec' + +# Offense count: 6 +# This cop supports safe autocorrection (--autocorrect). +# Configuration parameters: MaxUnannotatedPlaceholdersAllowed, AllowedMethods, AllowedPatterns, IgnoredMethods. +# SupportedStyles: annotated, template, unannotated +Style/FormatStringToken: + EnforcedStyle: unannotated + +# Offense count: 1 +# This cop supports unsafe autocorrection (--autocorrect-all). +Style/GlobalStdStream: + Exclude: + - 'lib/segment/analytics/logging.rb' + +# Offense count: 4 +# This cop supports safe autocorrection (--autocorrect). +Style/IfUnlessModifier: + Exclude: + - 'analytics-ruby.gemspec' + - 'bin/analytics' + - 'lib/segment/analytics/client.rb' + +# Offense count: 1 +Style/MixinUsage: + Exclude: + - 'spec/spec_helper.rb' + +# Offense count: 2 +# Configuration parameters: AllowedMethods. +# AllowedMethods: respond_to_missing? +Style/OptionalBooleanParameter: + Exclude: + - 'lib/segment/analytics/utils.rb' + +# Offense count: 1 +# This cop supports safe autocorrection (--autocorrect). +Style/Proc: + Exclude: + - 'bin/analytics' + +# Offense count: 1 +# This cop supports safe autocorrection (--autocorrect). +# Configuration parameters: AllowMultipleReturnValues. +Style/RedundantReturn: + Exclude: + - 'bin/analytics' + +# Offense count: 8 +# This cop supports safe autocorrection (--autocorrect). +# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline. +# SupportedStyles: single_quotes, double_quotes +Style/StringLiterals: + Exclude: + - 'Rakefile' + - 'analytics-ruby.gemspec' + - 'bin/analytics' + +# Offense count: 1 +# This cop supports safe autocorrection (--autocorrect). +# Configuration parameters: EnforcedStyleForMultiline. +# SupportedStylesForMultiline: comma, consistent_comma, no_comma +Style/TrailingCommaInArrayLiteral: + Exclude: + - 'Rakefile' + +# Offense count: 2 +# This cop supports safe autocorrection (--autocorrect). +# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, IgnoredPatterns. +# URISchemes: http, https +Layout/LineLength: + Max: 147 diff --git a/.travis.yml b/.travis.yml index 1a02c9f7..d015dd25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,20 @@ language: ruby rvm: - - jruby-19mode - - 1.9.3 - - 2.0.0 - - 2.1.0 + - 2.4.3 + - 2.5.0 + - 2.6.0 + - 2.7.0 # Performs deploys. Change condition below when changing this. + +script: + - make check + +# Deploy tagged commits to Rubygems +# See https://docs.travis-ci.com/user/deployment/rubygems/ for more details +deploy: + provider: rubygems + on: + tags: true + condition: "$TRAVIS_RUBY_VERSION == 2.7.0" + api_key: + secure: Ceq6J4aBpsoqRfSiC7z+/J4moOXNjcPMFb2Bfm5qE51cIZzeyuOIOc6zhrad9tUgoX6uTRRxLxkybyu4wNYSluMA3IXW20CJyXZeJEHIaTYIDTWFAIYyerBJyMujJycSo7XueWb0faKBENrBQKx1K1tS0EiXpA2rMhdA6RM3DOY= diff --git a/Gemfile b/Gemfile index 817f62a8..d54fb9a2 100644 --- a/Gemfile +++ b/Gemfile @@ -1,2 +1,5 @@ source 'http://rubygems.org' gemspec + +gem 'simplecov' +gem 'simplecov-cobertura' diff --git a/History.md b/History.md index c188aed2..d07c50a3 100644 --- a/History.md +++ b/History.md @@ -1,3 +1,104 @@ +2.5.0 / 2024-07-17 +================== + +* Fix silent failures (https://github.com/segmentio/analytics-ruby/pull/269) +* Update to Ruby 3.2 (https://github.com/segmentio/analytics-ruby/pull/262) +* Rename Segment namespace to SegmentIO (https://github.com/segmentio/analytics-ruby/pull/259) +* Lower allocated and retained strings (https://github.com/segmentio/analytics-ruby/pull/258) +* Modify timestamp to have 3 fractional digits (https://github.com/segmentio/analytics-ruby/pull/250 && https://github.com/segmentio/analytics-ruby/pull/251) +* Fix for empty user_id or anonymous_id (https://github.com/segmentio/analytics-ruby/pull/245) +* Not enqueuing test action in real queue (https://github.com/segmentio/analytics-ruby/pull/237) +* Fix test queue reset! documentation (https://github.com/segmentio/analytics-ruby/pull/235) + + +2.4.0 / 2021-05-05 +================== + +* Enable overriding transport Pass options when initializing Transport () + + +2.3.1 / 2021-04-13 +================== + + * Add test option for easier testing (https://github.com/segmentio/analytics-ruby/pull/222) + + +2.3.0 / 2021-03-26 +================== + + * [Improvement](https://github.com/segmentio/analytics-ruby/pull/225): Update timestamp for sub-millisecond reporting + * Update supported Ruby versions (2.4, 2.5, 2.6, 2.7), remove unsupported Ruby versions (2.0, 2.1, 2.2, 2.3) + +2.2.8 / 2020-02-10 +================== + + * Promoted pre-release version to stable. + +2.2.8.pre / 2019-11-29 +================== + + * [Fix](https://github.com/segmentio/analytics-ruby/pull/212): Fix log message + for stubbed requests + * [Deprecate](https://github.com/segmentio/analytics-ruby/pull/209): Deprecate + Ruby <2.0 support + +2.2.7 / 2019-05-09 +================== + + * [Fix](https://github.com/segmentio/analytics-ruby/pull/188): Allow `anonymous_id` + in `#alias` and `#group`. + +2.2.6 / 2018-06-11 +================== + + * Promote pre-release version to stable. + * [Fix](https://github.com/segmentio/analytics-ruby/pull/187): Don't assume + all errors are 'ConnectionError's + +2.2.6.pre / 2018-06-27 +================== + + * [Fix](https://github.com/segmentio/analytics-ruby/pull/168): Revert 'reuse + TCP connections' to fix EMFILE errors + * [Fix](https://github.com/segmentio/analytics-ruby/pull/166): Fix oj/rails + conflict + * [Fix](https://github.com/segmentio/analytics-ruby/pull/162): Add missing + 'Forwardable' requirement + * [Improvement](https://github.com/segmentio/analytics-ruby/pull/163): Better + logging + +2.2.5 / 2018-05-01 +================== + + * [Fix](https://github.com/segmentio/analytics-ruby/pull/158): Require `version` module first. + +2.2.4 / 2018-04-30 +================== + + * Promote pre-release version to stable. + +2.2.4.pre / 2018-02-04 +====================== + + * [Fix](https://github.com/segmentio/analytics-ruby/pull/147): Prevent 'batch + size exceeded' errors by automatically batching + items according to size + * [Performance](https://github.com/segmentio/analytics-ruby/pull/149): Reuse + TCP connections + * [Improvement](https://github.com/segmentio/analytics-ruby/pull/145): Emit logs + when in-memory queue is full + * [Improvement](https://github.com/segmentio/analytics-ruby/pull/143): Emit logs + when messages exceed maximum allowed size + * [Improvement](https://github.com/segmentio/analytics-ruby/pull/134): Add + exponential backoff to retries + * [Improvement](https://github.com/segmentio/analytics-ruby/pull/132): Handle + HTTP status code failure appropriately + +2.2.3.pre / 2017-09-14 +================== + + * [Fix](https://github.com/segmentio/analytics-ruby/pull/120): Override `respond_to_missing` instead of `respond_to?` to facilitate mock the library in tests. + 2.2.2 / 2016-08-03 ================== diff --git a/Makefile b/Makefile index 50825291..44b6c43b 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,22 @@ - -test: - rake spec - -build: - gem build ./analytics-ruby.gemspec - -.PHONY: test build + +# Install any tools required to build this library, e.g. Ruby, Bundler etc. +bootstrap: + brew install ruby + gem install bundler + +# Install any library dependencies. +dependencies: + bundle install --verbose + +# Run all tests and checks (including linters). +check: install # Installation required for testing binary + bundle exec rake + sh .buildscript/e2e.sh + +# Compile the code and produce any binaries where applicable. +build: + rm -f analytics-ruby-*.gem + gem build ./analytics-ruby.gemspec + +install: build + gem install analytics-ruby-*.gem diff --git a/README.md b/README.md index 7e1e1c4f..ab3eeb49 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,58 @@ analytics-ruby ============== -[![Build Status](https://travis-ci.org/segmentio/analytics-ruby.png?branch=master)](https://travis-ci.org/segmentio/analytics-ruby) - analytics-ruby is a ruby client for [Segment](https://segment.com) +### ⚠️ Maintenance ⚠️ +This library is in maintenance mode. It will send data as intended, but receive no new feature support and only critical maintenance updates from Segment. + +
+ +

You can't fix what you can't measure

+
+ +Analytics helps you measure your users, product, and business. It unlocks insights into your app's funnel, core business metrics, and whether you have product-market fit. + +## How to get started +1. **Collect analytics data** from your app(s). + - The top 200 Segment companies collect data from 5+ source types (web, mobile, server, CRM, etc.). +2. **Send the data to analytics tools** (for example, Google Analytics, Amplitude, Mixpanel). + - Over 250+ Segment companies send data to eight categories of destinations such as analytics tools, warehouses, email marketing and remarketing systems, session recording, and more. +3. **Explore your data** by creating metrics (for example, new signups, retention cohorts, and revenue generation). + - The best Segment companies use retention cohorts to measure product market fit. Netflix has 70% paid retention after 12 months, 30% after 7 years. + +[Segment](https://segment.com) collects analytics data and allows you to send it to more than 250 apps (such as Google Analytics, Mixpanel, Optimizely, Facebook Ads, Slack, Sentry) just by flipping a switch. You only need one Segment code snippet, and you can turn integrations on and off at will, with no additional code. [Sign up with Segment today](https://app.segment.com/signup). + +### Why? +1. **Power all your analytics apps with the same data**. Instead of writing code to integrate all of your tools individually, send data to Segment, once. + +2. **Install tracking for the last time**. We're the last integration you'll ever need to write. You only need to instrument Segment once. Reduce all of your tracking code and advertising tags into a single set of API calls. + +3. **Send data from anywhere**. Send Segment data from any device, and we'll transform and send it on to any tool. + +4. **Query your data in SQL**. Slice, dice, and analyze your data in detail with Segment SQL. We'll transform and load your customer behavioral data directly from your apps into Amazon Redshift, Google BigQuery, or Postgres. Save weeks of engineering time by not having to invent your own data warehouse and ETL pipeline. + + For example, you can capture data on any app: + ```js + analytics.track('Order Completed', { price: 99.84 }) + ``` + Then, query the resulting data in SQL: + ```sql + select * from app.order_completed + order by price desc + ``` + +### 🚀 Startup Program +
+ +
+If you are part of a new startup (<$5M raised, <2 years since founding), we just launched a new startup program for you. You can get a Segment Team plan (up to $25,000 value in Segment credits) for free up to 2 years — apply here! + ## Install Into Gemfile from rubygems.org: -``` -gem 'analytics-ruby', :require => "segment" +```ruby +gem 'analytics-ruby' ``` Into environment gems from rubygems.org: @@ -20,47 +63,108 @@ gem install 'analytics-ruby' ## Usage Create an instance of the Analytics object: +```ruby +analytics = Segment::Analytics.new(write_key: 'YOUR_WRITE_KEY') ``` -analytics = Segment::Analytics.new({ - write_key: 'YOUR_WRITE_KEY' -}) + +Identify the user for the people section, see more [here](https://segment.com/docs/libraries/ruby/#identify). +```ruby +analytics.identify(user_id: 42, + traits: { + email: 'name@example.com', + first_name: 'Foo', + last_name: 'Bar' + }) ``` -Sample usage: +Alias an user, see more [here](https://segment.com/docs/libraries/ruby/#alias). +```ruby +analytics.alias(user_id: 41) ``` -user = User.last - -# Identify the user for the people section -analytics.identify( - { - user_id: user.id, - traits: { - email: user.email, - first_name: user.first_name, - last_name: user.last_name - } - } -) - -# Track a user event -analytics.track( - { - user_id: user.id, - event: 'Created Account' - } -) + +Track a user event, see more [here](https://segment.com/docs/libraries/ruby/#track). +```ruby +analytics.track(user_id: 42, event: 'Created Account') ``` -Refer to the section below for documenation on individual available calls. +There are a few calls available, please check the documentation section. ## Documentation Documentation is available at [segment.com/docs/sources/server/ruby](https://segment.com/docs/sources/server/ruby/) -## Testing +### Test Queue + +You can use the `test: true` option to Segment::Analytics.new to cause all requests to be saved to a test queue until manually reset. All events will process as specified by the configuration, and they will also be stored in a separate queue for inspection during testing. + +A test queue can be used as follows: + +```ruby +client = Segment::Analytics.new(write_key: 'YOUR_WRITE_KEY', test: true) + +client.test_queue # => # + +client.track(user_id: 'foo', event: 'bar') + +client.test_queue.all +# [ +# { +# :context => { +# :library => { +# :name => "analytics-ruby", +# :version => "2.2.8.pre" +# } +# }, +# :messageId => "e9754cc0-1c5e-47e4-832a-203589d279e4", +# :timestamp => "2021-02-19T13:32:39.547+01:00", +# :userId => "foo", +# :type => "track", +# :event => "bar", +# :properties => {} +# } +# ] + +client.test_queue.track +# [ +# { +# :context => { +# :library => { +# :name => "analytics-ruby", +# :version => "2.2.8.pre" +# } +# }, +# :messageId => "e9754cc0-1c5e-47e4-832a-203589d279e4", +# :timestamp => "2021-02-19T13:32:39.547+01:00", +# :userId => "foo", +# :type => "track", +# :event => "bar", +# :properties => {} +# } +# ] + +# Other available methods +client.test_queue.alias # => [] +client.test_queue.group # => [] +client.test_queue.identify # => [] +client.test_queue.page # => [] +client.test_queue.screen # => [] + +client.test_queue.reset! + +client.test_queue.all # => [] +``` + +Note: It is recommended to call `reset!` before each test to ensure your test queue is empty. For example, in rspec you may have the following: -You can use the `stub` option to Segment::Analytics.new to cause all requests to be stubbed, making it easier to test with this library. +```ruby +RSpec.configure do |config| + config.before do + Analytics.test_queue.reset! + end +end +``` +And also to stub actions use `stub: true` along with `test: true` so that it doesn't send any real calls during specs. ## License ``` @@ -85,7 +189,3 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/segmentio/analytics-ruby/trend.png)](https://bitdeli.com/free "Bitdeli Badge") - diff --git a/RELEASING.md b/RELEASING.md index 0edd12b0..108b55a2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,10 +1,13 @@ -Releasing -========= +We automatically push tags to Rubygems via CI. + +Release +============ + +- Make sure you're on the latest `master` +- Bump the version in [`version.rb`](lib/segment/analytics/version.rb) +- Update [`History.md`](History.md) +- Commit these changes. `git commit -am "Release x.y.z."` +- Tag the release. `git tag -a -m "Version x.y.z" x.y.z` +- `git push -u origin master && git push --tags +- Run the publish action on Github - 1. Verify everything works with `make test build`. - 2. Bump version in [`version.rb`](https://github.com/segmentio/analytics-ruby/blob/master/lib/segment/analytics/version.rb). - 3. Update [`History.md`](https://github.com/segmentio/analytics-ruby/blob/master/History.md). - 4. Commit and tag `git commit -am "Release {version}" && git tag -a {version} -m "Version {version}"`. - 5. Build the gem with the tagged version `make build`. - 6. Upload to RubyGems with `gem push analytics-ruby-{version}.gem`. - 7. Upload to Github with `git push -u origin master && git push --tags`. diff --git a/Rakefile b/Rakefile index 8c4fe809..bb22a9fa 100644 --- a/Rakefile +++ b/Rakefile @@ -1,7 +1,33 @@ require 'rspec/core/rake_task' +default_tasks = [] + RSpec::Core::RakeTask.new(:spec) do |spec| - spec.pattern = 'spec/**/*_spec.rb' + spec.pattern = 'spec/segment/**/*_spec.rb' +end + +default_tasks << :spec + +# Isolated tests are run as separate rake tasks so that gem conflicts can be +# tests in different processes +Dir.glob('spec/isolated/**/*.rb').each do |isolated_test_path| + RSpec::Core::RakeTask.new(isolated_test_path) do |spec| + spec.pattern = isolated_test_path + end + + default_tasks << isolated_test_path +end + +# Older versions of Rubocop don't support a target Ruby version of 2.1 +require 'rubocop/version' +if RuboCop::Version::STRING >= '1.30.0' + require 'rubocop/rake_task' + + RuboCop::RakeTask.new(:rubocop) do |task| + task.patterns = ['lib/**/*.rb','spec/**/*.rb',] + end + + default_tasks << :rubocop end -task :default => :spec +task :default => default_tasks diff --git a/analytics-ruby.gemspec b/analytics-ruby.gemspec index 416bf16c..b45b31d0 100644 --- a/analytics-ruby.gemspec +++ b/analytics-ruby.gemspec @@ -3,7 +3,7 @@ require File.expand_path('../lib/segment/analytics/version', __FILE__) Gem::Specification.new do |spec| spec.name = 'analytics-ruby' spec.version = Segment::Analytics::VERSION - spec.files = Dir.glob('**/*') + spec.files = Dir.glob("{lib,bin}/**/*") spec.require_paths = ['lib'] spec.bindir = 'bin' spec.executables = ['analytics'] @@ -13,14 +13,18 @@ Gem::Specification.new do |spec| spec.email = 'friends@segment.io' spec.homepage = 'https://github.com/segmentio/analytics-ruby' spec.license = 'MIT' + spec.required_ruby_version = '>= 2.0' - # Ruby 1.8 requires json - spec.add_dependency 'json', ['~> 1.7'] if RUBY_VERSION < "1.9" - spec.add_dependency 'commander', '~> 4.4' + # Used in the executable testing script + spec.add_development_dependency 'commander', '~> 4.4' - spec.add_development_dependency 'rake', '~> 10.3' - spec.add_development_dependency 'wrong', '~> 0.0' - spec.add_development_dependency 'rspec', '~> 2.0' - spec.add_development_dependency 'tzinfo', '1.2.1' - spec.add_development_dependency 'activesupport', '>= 3.0.0', '<4.0.0' + # Used in specs + spec.add_development_dependency 'rake', '~> 13.0' + spec.add_development_dependency 'rspec', '~> 3.0' + spec.add_development_dependency 'tzinfo', '~> 1.2' + spec.add_development_dependency 'activesupport', '~> 5.2.0' + if RUBY_PLATFORM != 'java' + spec.add_development_dependency 'oj', '~> 3.6.2' + end + spec.add_development_dependency 'rubocop', '~> 1.0' end diff --git a/bin/analytics b/bin/analytics index e7543cbb..1f4c19e3 100755 --- a/bin/analytics +++ b/bin/analytics @@ -10,143 +10,99 @@ program :name, 'simulator.rb' program :version, '0.0.1' program :description, 'scripting simulator' -# use an env var for write key, instead of a flag -Analytics = Segment::Analytics.new({ - write_key: ENV['SEGMENT_WRITE_KEY'], - on_error: Proc.new { |status, msg| print msg } -}) - -def toObject(str) - return JSON.parse(str) -end - -# high level -# analytics [options] -# SEGMENT_WRITE_KEY= ./analytics.rb [options] -# SEGMENT_WRITE_KEY= ./analytics.rb track --event testing --user 1234 --anonymous 567 --properties '{"hello": "goodbye"}' --context '{"slow":"poke"}' - - - -# track -command :track do |c| - c.description = 'track a user event' - c.option '--user ', String, 'the user id to send the event as' - c.option '--event ', String, 'the event name to send with the event' - c.option '--anonymous ', String, 'the anonymous user id to send the event as' - c.option '--properties ', 'the event properties to send (JSON-encoded)' - c.option '--context ', 'additional context for the event (JSON-encoded)' - - c.action do |args, options| - Analytics.track({ - user_id: options.user, - event: options.event, - anonymous_id: options.anonymous, - properties: toObject(options.properties), - context: toObject(options.context) - }) - Analytics.flush +def json_hash(str) + if str + return JSON.parse(str) end - end -# page -command :page do |c| - c.description = 'track a page view' - c.option '--user ', String, 'the user id to send the event as' - c.option '--anonymous ', String, 'the anonymous user id to send the event as' - c.option '--name ', String, 'the page name' - c.option '--properties ', 'the event properties to send (JSON-encoded)' - c.option '--context ', 'additional context for the event (JSON-encoded)' - c.option '--category ', 'the category of the page' - c.action do |args, options| - Analytics.page({ - user_id: options.user, - anonymous_id: options.anonymous, - name: options.name, - properties: toObject(options.properties), - context: toObject(options.context), - category: options.category - }) - Analytics.flush - end +# analytics -method= -segment-write-key= [options] -end +default_command :send -# identify -command :identify do |c| - c.description = 'identify a user' - c.option '--user ', String, 'the user id to send the event as' - c.option '--anonymous ', String, 'the anonymous user id to send the event as' - c.option '--traits ', String, 'the user traits to send (JSON-encoded)' - c.option '--context ', 'additional context for the event (JSON-encoded)' - c.action do |args, options| - Analytics.identify({ - user_id: options.user, - anonymous_id: options.anonymous, - traits: toObject(options.traits), - context: toObject(options.context) - }) - Analytics.flush - end +command :send do |c| + c.description = 'send a segment message' -end + c.option '--writeKey=', String, 'the Segment writeKey' + c.option '--type=', String, 'The Segment message type' -# screen -command :screen do |c| - c.description = 'track a screen view' - c.option '--user ', String, 'the user id to send the event as' - c.option '--anonymous ', String, 'the anonymous user id to send the event as' - c.option '--name ', String, 'the screen name' - c.option '--properties ', String, 'the event properties to send (JSON-encoded)' - c.option '--context ', 'additional context for the event (JSON-encoded)' - c.action do |args, options| - Analytics.identify({ - user_id: options.user, - anonymous_id: options.anonymous, - name: option.name, - traits: toObject(options.traits), - properties: toObject(option.properties), - context: toObject(options.context) - }) - Analytics.flush - end + c.option '--userId=', String, 'the user id to send the event as' + c.option '--anonymousId=', String, 'the anonymous user id to send the event as' + c.option '--context=', 'additional context for the event (JSON-encoded)' + c.option '--integrations=', 'additional integrations for the event (JSON-encoded)' -end + c.option '--event=', String, 'the event name to send with the event' + c.option '--properties=', 'the event properties to send (JSON-encoded)' + c.option '--name=', 'name of the screen or page to send with the message' -# group -command :group do |c| - c.description = 'identify a group of users' - c.option '--user ', String, 'the user id to send the event as' - c.option '--anonymous ', String, 'the anonymous user id to send the event as' - c.option '--group ', String, 'the group id to associate this user with' - c.option '--traits ', String, 'attributes about the group (JSON-encoded)' - c.option '--context ', 'additional context for the event (JSON-encoded)' - c.action do |args, options| - Analytics.group({ - user_id: options.user, - anonymous_id: options.anonymous, - group_id: options.group, - traits: toObject(options.traits), - context: toObject(options.context) - }) - Analytics.flush - end + c.option '--traits=', 'the identify/group traits to send (JSON-encoded)' -end + c.option '--groupId=', String, 'the group id' + c.option '--previousId=', String, 'the previous id' -# alias -command :alias do |c| - c.description = 'remap a user to a new id' - c.option '--user ', String, 'the user id to send the event as' - c.option '--previous ', String, 'the previous user id (to add the alias for)' c.action do |args, options| - Analytics.alias({ - user_id: options.user, - previous_id: options.previous - }) + Analytics = Segment::Analytics.new({ + :write_key => options.writeKey, + :on_error => Proc.new { |status, msg| print msg } + }) + + case options.type + when "track" + Analytics.track({ + :user_id => options.userId, + :event => options.event, + :anonymous_id => options.anonymousId, + :properties => json_hash(options.properties), + :context => json_hash(options.context), + :integrations => json_hash(options.integrations) + }) + when "page" + Analytics.page({ + :user_id => options.userId, + :anonymous_id => options.anonymousId, + :name => options.name, + :properties => json_hash(options.properties), + :context => json_hash(options.context), + :integrations => json_hash(options.integrations) + }) + when "screen" + Analytics.screen({ + :user_id => options.userId, + :anonymous_id => options.anonymousId, + :name => options.name, + :properties => json_hash(options.properties), + :context => json_hash(options.context), + :integrations => json_hash(options.integrations) + }) + when "identify" + Analytics.identify({ + :user_id => options.userId, + :anonymous_id => options.anonymousId, + :traits => json_hash(options.traits), + :context => json_hash(options.context), + :integrations => json_hash(options.integrations) + }) + when "group" + Analytics.group({ + :user_id => options.userId, + :anonymous_id => options.anonymousId, + :group_id => options.groupId, + :traits => json_hash(options.traits), + :context => json_hash(options.context), + :integrations => json_hash(options.integrations) + }) + when "alias" + Analytics.alias({ + :previous_id => options.previousId, + :user_id => options.userId, + :anonymous_id => options.anonymousId, + :context => json_hash(options.context), + :integrations => json_hash(options.integrations) + }) + else + raise "Invalid Message Type #{options.type}" + end Analytics.flush end - end - diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..c6e5dff8 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,2 @@ +ignore: + - "spec/**/*" diff --git a/lib/analytics-ruby.rb b/lib/analytics-ruby.rb new file mode 100644 index 00000000..1c99ea1c --- /dev/null +++ b/lib/analytics-ruby.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require 'segment' diff --git a/lib/segment.rb b/lib/segment.rb index 1465165d..0efdd704 100644 --- a/lib/segment.rb +++ b/lib/segment.rb @@ -1 +1,3 @@ +# frozen_string_literal: true + require 'segment/analytics' diff --git a/lib/segment/analytics.rb b/lib/segment/analytics.rb index 78d6c28a..707e7c43 100644 --- a/lib/segment/analytics.rb +++ b/lib/segment/analytics.rb @@ -1,20 +1,31 @@ +# frozen_string_literal: true + +require 'segment/analytics/version' require 'segment/analytics/defaults' require 'segment/analytics/utils' -require 'segment/analytics/version' +require 'segment/analytics/field_parser' require 'segment/analytics/client' require 'segment/analytics/worker' -require 'segment/analytics/request' +require 'segment/analytics/transport' require 'segment/analytics/response' require 'segment/analytics/logging' +require 'segment/analytics/test_queue' module Segment class Analytics - def initialize options = {} - Request.stub = options[:stub] if options.has_key?(:stub) + # Initializes a new instance of {Segment::Analytics::Client}, to which all + # method calls are proxied. + # + # @param options includes options that are passed down to + # {Segment::Analytics::Client#initialize} + # @option options [Boolean] :stub (false) If true, requests don't hit the + # server and are stubbed to be successful. + def initialize(options = {}) + Transport.stub = options[:stub] if options.has_key?(:stub) @client = Segment::Analytics::Client.new options end - def method_missing message, *args, &block + def method_missing(message, *args, &block) if @client.respond_to? message @client.send message, *args, &block else @@ -22,7 +33,7 @@ def method_missing message, *args, &block end end - def respond_to? method_name, include_private = false + def respond_to_missing?(method_name, include_private = false) @client.respond_to?(method_name) || super end diff --git a/lib/segment/analytics/backoff_policy.rb b/lib/segment/analytics/backoff_policy.rb new file mode 100644 index 00000000..e6033b1f --- /dev/null +++ b/lib/segment/analytics/backoff_policy.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require 'segment/analytics/defaults' + +module Segment + class Analytics + class BackoffPolicy + include Segment::Analytics::Defaults::BackoffPolicy + + # @param [Hash] opts + # @option opts [Numeric] :min_timeout_ms The minimum backoff timeout + # @option opts [Numeric] :max_timeout_ms The maximum backoff timeout + # @option opts [Numeric] :multiplier The value to multiply the current + # interval with for each retry attempt + # @option opts [Numeric] :randomization_factor The randomization factor + # to use to create a range around the retry interval + def initialize(opts = {}) + @min_timeout_ms = opts[:min_timeout_ms] || MIN_TIMEOUT_MS + @max_timeout_ms = opts[:max_timeout_ms] || MAX_TIMEOUT_MS + @multiplier = opts[:multiplier] || MULTIPLIER + @randomization_factor = opts[:randomization_factor] || RANDOMIZATION_FACTOR + + @attempts = 0 + end + + # @return [Numeric] the next backoff interval, in milliseconds. + def next_interval + interval = @min_timeout_ms * (@multiplier**@attempts) + interval = add_jitter(interval, @randomization_factor) + + @attempts += 1 + + [interval, @max_timeout_ms].min + end + + private + + def add_jitter(base, randomization_factor) + random_number = rand + max_deviation = base * randomization_factor + deviation = random_number * max_deviation + + if random_number < 0.5 + base - deviation + else + base + deviation + end + end + end + end +end diff --git a/lib/segment/analytics/client.rb b/lib/segment/analytics/client.rb index 138f819d..9589f69d 100644 --- a/lib/segment/analytics/client.rb +++ b/lib/segment/analytics/client.rb @@ -1,39 +1,44 @@ +# frozen_string_literal: true + require 'thread' require 'time' + +require 'segment/analytics/defaults' +require 'segment/analytics/logging' require 'segment/analytics/utils' require 'segment/analytics/worker' -require 'segment/analytics/defaults' module Segment class Analytics class Client include Segment::Analytics::Utils + include Segment::Analytics::Logging - # public: Creates a new client - # - # attrs - Hash - # :write_key - String of your project's write_key - # :max_queue_size - Fixnum of the max calls to remain queued (optional) - # :on_error - Proc which handles error calls from the API - def initialize attrs = {} - symbolize_keys! attrs + # @param [Hash] opts + # @option opts [String] :write_key Your project's write_key + # @option opts [FixNum] :max_queue_size Maximum number of calls to be + # remain queued. + # @option opts [Proc] :on_error Handles error calls from the API. + def initialize(opts = {}) + symbolize_keys!(opts) @queue = Queue.new - @write_key = attrs[:write_key] - @max_queue_size = attrs[:max_queue_size] || Defaults::Queue::MAX_SIZE - @options = attrs + @test = opts[:test] + @write_key = opts[:write_key] + @max_queue_size = opts[:max_queue_size] || Defaults::Queue::MAX_SIZE @worker_mutex = Mutex.new - @worker = Worker.new @queue, @write_key, @options + @worker = Worker.new(@queue, @write_key, opts) + @worker_thread = nil check_write_key! at_exit { @worker_thread && @worker_thread[:should_exit] = true } end - # public: Synchronously waits until the worker has flushed the queue. - # Use only for scripts which are not long-running, and will - # specifically exit + # Synchronously waits until the worker has flushed the queue. # + # Use only for scripts which are not long-running, and will specifically + # exit def flush while !@queue.empty? || @worker.is_requesting? ensure_worker_running @@ -41,263 +46,114 @@ def flush end end - # public: Tracks an event + # @!macro common_attrs + # @option attrs [String] :anonymous_id ID for a user when you don't know + # who they are yet. (optional but you must provide either an + # `anonymous_id` or `user_id`) + # @option attrs [Hash] :context ({}) + # @option attrs [Hash] :integrations What integrations this event + # goes to (optional) + # @option attrs [String] :message_id ID that uniquely + # identifies a message across the API. (optional) + # @option attrs [Time] :timestamp When the event occurred (optional) + # @option attrs [String] :user_id The ID for this user in your database + # (optional but you must provide either an `anonymous_id` or `user_id`) + # @option attrs [Hash] :options Options such as user traits (optional) + + # Tracks an event + # + # @see https://segment.com/docs/sources/server/ruby/#track # - # attrs - Hash - # :anonymous_id - String of the user's id when you don't know who they are yet. (optional but you must provide either an anonymous_id or user_id. See: https://segment.io/docs/tracking - api/track/#user - id) - # :context - Hash of context. (optional) - # :event - String of event name. - # :integrations - Hash specifying what integrations this event goes to. (optional) - # :options - Hash specifying options such as user traits. (optional) - # :properties - Hash of event properties. (optional) - # :timestamp - Time of when the event occurred. (optional) - # :user_id - String of the user id. - # :message_id - String of the message id that uniquely identified a message across the API. (optional) - def track attrs + # @param [Hash] attrs + # + # @option attrs [String] :event Event name + # @option attrs [Hash] :properties Event properties (optional) + # @macro common_attrs + def track(attrs) symbolize_keys! attrs - check_user_id! attrs - - event = attrs[:event] - properties = attrs[:properties] || {} - timestamp = attrs[:timestamp] || Time.new - context = attrs[:context] || {} - message_id = attrs[:message_id].to_s if attrs[:message_id] - - check_timestamp! timestamp - - if event.nil? || event.empty? - fail ArgumentError, 'Must supply event as a non-empty string' - end - - fail ArgumentError, 'Properties must be a Hash' unless properties.is_a? Hash - isoify_dates! properties - - add_context context - - enqueue({ - :event => event, - :userId => attrs[:user_id], - :anonymousId => attrs[:anonymous_id], - :context => context, - :options => attrs[:options], - :integrations => attrs[:integrations], - :properties => properties, - :messageId => message_id, - :timestamp => datetime_in_iso8601(timestamp), - :type => 'track' - }) + enqueue(FieldParser.parse_for_track(attrs)) end - # public: Identifies a user + # Identifies a user + # + # @see https://segment.com/docs/sources/server/ruby/#identify # - # attrs - Hash - # :anonymous_id - String of the user's id when you don't know who they are yet. (optional but you must provide either an anonymous_id or user_id. See: https://segment.io/docs/tracking - api/track/#user - id) - # :context - Hash of context. (optional) - # :integrations - Hash specifying what integrations this event goes to. (optional) - # :options - Hash specifying options such as user traits. (optional) - # :timestamp - Time of when the event occurred. (optional) - # :traits - Hash of user traits. (optional) - # :user_id - String of the user id - # :message_id - String of the message id that uniquely identified a message across the API. (optional) - def identify attrs + # @param [Hash] attrs + # + # @option attrs [Hash] :traits User traits (optional) + # @macro common_attrs + def identify(attrs) symbolize_keys! attrs - check_user_id! attrs - - traits = attrs[:traits] || {} - timestamp = attrs[:timestamp] || Time.new - context = attrs[:context] || {} - message_id = attrs[:message_id].to_s if attrs[:message_id] - - check_timestamp! timestamp - - fail ArgumentError, 'Must supply traits as a hash' unless traits.is_a? Hash - isoify_dates! traits - - add_context context - - enqueue({ - :userId => attrs[:user_id], - :anonymousId => attrs[:anonymous_id], - :integrations => attrs[:integrations], - :context => context, - :traits => traits, - :options => attrs[:options], - :messageId => message_id, - :timestamp => datetime_in_iso8601(timestamp), - :type => 'identify' - }) + enqueue(FieldParser.parse_for_identify(attrs)) end - # public: Aliases a user from one id to another + # Aliases a user from one id to another + # + # @see https://segment.com/docs/sources/server/ruby/#alias # - # attrs - Hash - # :context - Hash of context (optional) - # :integrations - Hash specifying what integrations this event goes to. (optional) - # :options - Hash specifying options such as user traits. (optional) - # :previous_id - String of the id to alias from - # :timestamp - Time of when the alias occured (optional) - # :user_id - String of the id to alias to - # :message_id - String of the message id that uniquely identified a message across the API. (optional) + # @param [Hash] attrs + # + # @option attrs [String] :previous_id The ID to alias from + # @macro common_attrs def alias(attrs) symbolize_keys! attrs - - from = attrs[:previous_id] - to = attrs[:user_id] - timestamp = attrs[:timestamp] || Time.new - context = attrs[:context] || {} - message_id = attrs[:message_id].to_s if attrs[:message_id] - - check_presence! from, 'previous_id' - check_presence! to, 'user_id' - check_timestamp! timestamp - add_context context - - enqueue({ - :previousId => from, - :userId => to, - :integrations => attrs[:integrations], - :context => context, - :options => attrs[:options], - :messageId => message_id, - :timestamp => datetime_in_iso8601(timestamp), - :type => 'alias' - }) + enqueue(FieldParser.parse_for_alias(attrs)) end - # public: Associates a user identity with a group. + # Associates a user identity with a group. # - # attrs - Hash - # :context - Hash of context (optional) - # :integrations - Hash specifying what integrations this event goes to. (optional) - # :options - Hash specifying options such as user traits. (optional) - # :previous_id - String of the id to alias from - # :timestamp - Time of when the alias occured (optional) - # :user_id - String of the id to alias to - # :message_id - String of the message id that uniquely identified a message across the API. (optional) + # @see https://segment.com/docs/sources/server/ruby/#group + # + # @param [Hash] attrs + # + # @option attrs [String] :group_id The ID of the group + # @option attrs [Hash] :traits User traits (optional) + # @macro common_attrs def group(attrs) symbolize_keys! attrs - check_user_id! attrs - - group_id = attrs[:group_id] - user_id = attrs[:user_id] - traits = attrs[:traits] || {} - timestamp = attrs[:timestamp] || Time.new - context = attrs[:context] || {} - message_id = attrs[:message_id].to_s if attrs[:message_id] - - fail ArgumentError, '.traits must be a hash' unless traits.is_a? Hash - isoify_dates! traits - - check_presence! group_id, 'group_id' - check_timestamp! timestamp - add_context context - - enqueue({ - :groupId => group_id, - :userId => user_id, - :traits => traits, - :integrations => attrs[:integrations], - :options => attrs[:options], - :context => context, - :messageId => message_id, - :timestamp => datetime_in_iso8601(timestamp), - :type => 'group' - }) + enqueue(FieldParser.parse_for_group(attrs)) end - # public: Records a page view + # Records a page view + # + # @see https://segment.com/docs/sources/server/ruby/#page + # + # @param [Hash] attrs # - # attrs - Hash - # :anonymous_id - String of the user's id when you don't know who they are yet. (optional but you must provide either an anonymous_id or user_id. See: https://segment.io/docs/tracking - api/track/#user - id) - # :category - String of the page category (optional) - # :context - Hash of context (optional) - # :integrations - Hash specifying what integrations this event goes to. (optional) - # :name - String name of the page - # :options - Hash specifying options such as user traits. (optional) - # :properties - Hash of page properties (optional) - # :timestamp - Time of when the pageview occured (optional) - # :user_id - String of the id to alias from - # :message_id - String of the message id that uniquely identified a message across the API. (optional) + # @option attrs [String] :name Name of the page + # @option attrs [Hash] :properties Page properties (optional) + # @macro common_attrs def page(attrs) symbolize_keys! attrs - check_user_id! attrs - - name = attrs[:name].to_s - properties = attrs[:properties] || {} - timestamp = attrs[:timestamp] || Time.new - context = attrs[:context] || {} - message_id = attrs[:message_id].to_s if attrs[:message_id] - - fail ArgumentError, '.properties must be a hash' unless properties.is_a? Hash - isoify_dates! properties - - check_timestamp! timestamp - add_context context - - enqueue({ - :userId => attrs[:user_id], - :anonymousId => attrs[:anonymous_id], - :name => name, - :category => attrs[:category], - :properties => properties, - :integrations => attrs[:integrations], - :options => attrs[:options], - :context => context, - :messageId => message_id, - :timestamp => datetime_in_iso8601(timestamp), - :type => 'page' - }) + enqueue(FieldParser.parse_for_page(attrs)) end - # public: Records a screen view (for a mobile app) + + # Records a screen view (for a mobile app) # - # attrs - Hash - # :anonymous_id - String of the user's id when you don't know who they are yet. (optional but you must provide either an anonymous_id or user_id. See: https://segment.io/docs/tracking - api/track/#user - id) - # :category - String screen category (optional) - # :context - Hash of context (optional) - # :integrations - Hash specifying what integrations this event goes to. (optional) - # :name - String name of the screen - # :options - Hash specifying options such as user traits. (optional) - # :properties - Hash of screen properties (optional) - # :timestamp - Time of when the screen occured (optional) - # :user_id - String of the id to alias from + # @param [Hash] attrs + # + # @option attrs [String] :name Name of the screen + # @option attrs [Hash] :properties Screen properties (optional) + # @option attrs [String] :category The screen category (optional) + # @macro common_attrs def screen(attrs) symbolize_keys! attrs - check_user_id! attrs - - name = attrs[:name].to_s - properties = attrs[:properties] || {} - timestamp = attrs[:timestamp] || Time.new - context = attrs[:context] || {} - message_id = attrs[:message_id].to_s if attrs[:message_id] - - fail ArgumentError, '.properties must be a hash' unless properties.is_a? Hash - isoify_dates! properties - - check_timestamp! timestamp - add_context context - - enqueue({ - :userId => attrs[:user_id], - :anonymousId => attrs[:anonymous_id], - :name => name, - :properties => properties, - :category => attrs[:category], - :options => attrs[:options], - :integrations => attrs[:integrations], - :context => context, - :messageId => message_id, - :timestamp => timestamp.iso8601, - :type => 'screen' - }) + enqueue(FieldParser.parse_for_screen(attrs)) end - # public: Returns the number of queued messages - # - # returns Fixnum of messages in the queue + # @return [Fixnum] number of messages in the queue def queued_messages @queue.length end + def test_queue + unless @test + raise 'Test queue only available when setting :test to true.' + end + + @test_queue ||= TestQueue.new + end + private # private: Enqueues the action. @@ -306,56 +162,28 @@ def queued_messages def enqueue(action) # add our request id for tracing purposes action[:messageId] ||= uid - unless queue_full = @queue.length >= @max_queue_size - ensure_worker_running - @queue << action - end - !queue_full - end - # private: Ensures that a string is non-empty - # - # obj - String|Number that must be non-blank - # name - Name of the validated value - # - def check_presence!(obj, name) - if obj.nil? || (obj.is_a?(String) && obj.empty?) - fail ArgumentError, "#{name} must be given" + if @test + test_queue << action + return true end - end - # private: Adds contextual information to the call - # - # context - Hash of call context - def add_context(context) - context[:library] = { :name => "analytics-ruby", :version => Segment::Analytics::VERSION.to_s } + if @queue.length < @max_queue_size + @queue << action + ensure_worker_running + + true + else + logger.warn( + 'Queue is full, dropping events. The :max_queue_size configuration parameter can be increased to prevent this from happening.' + ) + false + end end # private: Checks that the write_key is properly initialized def check_write_key! - fail ArgumentError, 'Write key must be initialized' if @write_key.nil? - end - - # private: Checks the timstamp option to make sure it is a Time. - def check_timestamp!(timestamp) - fail ArgumentError, 'Timestamp must be a Time' unless timestamp.is_a? Time - end - - def event attrs - symbolize_keys! attrs - - { - :userId => user_id, - :name => name, - :properties => properties, - :context => context, - :timestamp => datetime_in_iso8601(timestamp), - :type => 'screen' - } - end - - def check_user_id! attrs - fail ArgumentError, 'Must supply either user_id or anonymous_id' unless attrs[:user_id] || attrs[:anonymous_id] + raise ArgumentError, 'Write key must be initialized' if @write_key.nil? end def ensure_worker_running diff --git a/lib/segment/analytics/defaults.rb b/lib/segment/analytics/defaults.rb index aeb933e3..aa326974 100644 --- a/lib/segment/analytics/defaults.rb +++ b/lib/segment/analytics/defaults.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Segment class Analytics module Defaults @@ -6,15 +8,31 @@ module Request PORT = 443 PATH = '/v1/import' SSL = true - HEADERS = { :accept => 'application/json' } - RETRIES = 4 - BACKOFF = 30.0 + HEADERS = { 'Accept' => 'application/json', + 'Content-Type' => 'application/json', + 'User-Agent' => "analytics-ruby/#{Analytics::VERSION}" } + RETRIES = 10 end module Queue - BATCH_SIZE = 100 MAX_SIZE = 10000 end + + module Message + MAX_BYTES = 32768 # 32Kb + end + + module MessageBatch + MAX_BYTES = 512_000 # 500Kb + MAX_SIZE = 100 + end + + module BackoffPolicy + MIN_TIMEOUT_MS = 100 + MAX_TIMEOUT_MS = 10000 + MULTIPLIER = 1.5 + RANDOMIZATION_FACTOR = 0.5 + end end end end diff --git a/lib/segment/analytics/field_parser.rb b/lib/segment/analytics/field_parser.rb new file mode 100644 index 00000000..a7364ecd --- /dev/null +++ b/lib/segment/analytics/field_parser.rb @@ -0,0 +1,197 @@ +# frozen_string_literal: true + +module Segment + class Analytics + # Handles parsing fields according to the Segment Spec + # + # @see https://segment.com/docs/spec/ + class FieldParser + class << self + include Segment::Analytics::Utils + + # In addition to the common fields, track accepts: + # + # - "event" + # - "properties" + def parse_for_track(fields) + common = parse_common_fields(fields) + + event = fields[:event] + properties = fields[:properties] || {} + + check_presence!(event, 'event') + check_is_hash!(properties, 'properties') + + isoify_dates! properties + + common.merge({ + :type => 'track', + :event => event.to_s, + :properties => properties + }) + end + + # In addition to the common fields, identify accepts: + # + # - "traits" + def parse_for_identify(fields) + common = parse_common_fields(fields) + + traits = fields[:traits] || {} + check_is_hash!(traits, 'traits') + isoify_dates! traits + + common.merge({ + :type => 'identify', + :traits => traits + }) + end + + # In addition to the common fields, alias accepts: + # + # - "previous_id" + def parse_for_alias(fields) + common = parse_common_fields(fields) + + previous_id = fields[:previous_id] + check_presence!(previous_id, 'previous_id') + + common.merge({ + :type => 'alias', + :previousId => previous_id + }) + end + + # In addition to the common fields, group accepts: + # + # - "group_id" + # - "traits" + def parse_for_group(fields) + common = parse_common_fields(fields) + + group_id = fields[:group_id] + traits = fields[:traits] || {} + + check_presence!(group_id, 'group_id') + check_is_hash!(traits, 'traits') + + isoify_dates! traits + + common.merge({ + :type => 'group', + :groupId => group_id, + :traits => traits + }) + end + + # In addition to the common fields, page accepts: + # + # - "name" + # - "properties" + def parse_for_page(fields) + common = parse_common_fields(fields) + + name = fields[:name] || '' + properties = fields[:properties] || {} + + check_is_hash!(properties, 'properties') + + isoify_dates! properties + + common.merge({ + :type => 'page', + :name => name.to_s, + :properties => properties + }) + end + + # In addition to the common fields, screen accepts: + # + # - "name" + # - "properties" + # - "category" (Not in spec, retained for backward compatibility" + def parse_for_screen(fields) + common = parse_common_fields(fields) + + name = fields[:name] + properties = fields[:properties] || {} + category = fields[:category] + + check_presence!(name, 'name') + check_is_hash!(properties, 'properties') + + isoify_dates! properties + + parsed = common.merge({ + :type => 'screen', + :name => name, + :properties => properties + }) + + parsed[:category] = category if category + + parsed + end + + private + + def parse_common_fields(fields) + timestamp = fields[:timestamp] || Time.new + message_id = fields[:message_id].to_s if fields[:message_id] + context = fields[:context] || {} + + check_user_id! fields + check_timestamp! timestamp + + add_context! context + + parsed = { + :context => context, + :messageId => message_id, + :timestamp => datetime_in_iso8601(timestamp) + } + + parsed[:userId] = fields[:user_id] if fields[:user_id] + parsed[:anonymousId] = fields[:anonymous_id] if fields[:anonymous_id] + parsed[:integrations] = fields[:integrations] if fields[:integrations] + + # Not in spec, retained for backward compatibility + parsed[:options] = fields[:options] if fields[:options] + + parsed + end + + def check_user_id!(fields) + return unless blank?(fields[:user_id]) + return unless blank?(fields[:anonymous_id]) + + raise ArgumentError, 'Must supply either user_id or anonymous_id' + end + + def check_timestamp!(timestamp) + raise ArgumentError, 'Timestamp must be a Time' unless timestamp.is_a? Time + end + + def add_context!(context) + context[:library] = { :name => 'analytics-ruby', :version => Segment::Analytics::VERSION.to_s } + end + + # private: Ensures that a string is non-empty + # + # obj - String|Number that must be non-blank + # name - Name of the validated value + def check_presence!(obj, name) + raise ArgumentError, "#{name} must be given" if blank?(obj) + end + + def blank?(obj) + obj.nil? || (obj.is_a?(String) && obj.empty?) + end + + def check_is_hash!(obj, name) + raise ArgumentError, "#{name} must be a Hash" unless obj.is_a? Hash + end + end + end + end +end diff --git a/lib/segment/analytics/logging.rb b/lib/segment/analytics/logging.rb index e7f5229e..5449878b 100644 --- a/lib/segment/analytics/logging.rb +++ b/lib/segment/analytics/logging.rb @@ -1,25 +1,52 @@ +# frozen_string_literal: true + require 'logger' module Segment class Analytics + # Wraps an existing logger and adds a prefix to all messages + class PrefixedLogger + def initialize(logger, prefix) + @logger = logger + @prefix = prefix + end + + def debug(msg) + @logger.debug("#{@prefix} #{msg}") + end + + def info(msg) + @logger.info("#{@prefix} #{msg}") + end + + def warn(msg) + @logger.warn("#{@prefix} #{msg}") + end + + def error(msg) + @logger.error("#{@prefix} #{msg}") + end + end + module Logging class << self def logger - @logger ||= if defined?(Rails) - Rails.logger - else - logger = Logger.new STDOUT - logger.progname = 'Segment::Analytics' - logger - end - end + return @logger if @logger - def logger= logger - @logger = logger + base_logger = if defined?(Rails) + Rails.logger + else + logger = Logger.new STDOUT + logger.progname = 'Segment::Analytics' + logger + end + @logger = PrefixedLogger.new(base_logger, '[analytics-ruby]') end + + attr_writer :logger end - def self.included base + def self.included(base) class << base def logger Logging.logger diff --git a/lib/segment/analytics/message_batch.rb b/lib/segment/analytics/message_batch.rb new file mode 100644 index 00000000..29e37d20 --- /dev/null +++ b/lib/segment/analytics/message_batch.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require 'forwardable' +require 'segment/analytics/logging' + +module Segment + class Analytics + # A batch of `Message`s to be sent to the API + class MessageBatch + class JSONGenerationError < StandardError; end + + extend Forwardable + include Segment::Analytics::Logging + include Segment::Analytics::Defaults::MessageBatch + + def initialize(max_message_count) + @messages = [] + @max_message_count = max_message_count + @json_size = 0 + end + + def <<(message) + begin + message_json = message.to_json + rescue StandardError => e + raise JSONGenerationError, "Serialization error: #{e}" + end + + message_json_size = message_json.bytesize + if message_too_big?(message_json_size) + logger.error('a message exceeded the maximum allowed size') + raise JSONGenerationError, 'Message Exceeded Maximum Allowed Size' + else + @messages << message + @json_size += message_json_size + 1 # One byte for the comma + end + end + + def full? + item_count_exhausted? || size_exhausted? + end + + def clear + @messages.clear + @json_size = 0 + end + + def_delegators :@messages, :to_json + def_delegators :@messages, :empty? + def_delegators :@messages, :length + + private + + def item_count_exhausted? + @messages.length >= @max_message_count + end + + def message_too_big?(message_json_size) + message_json_size > Defaults::Message::MAX_BYTES + end + + # We consider the max size here as just enough to leave room for one more + # message of the largest size possible. This is a shortcut that allows us + # to use a native Ruby `Queue` that doesn't allow peeking. The tradeoff + # here is that we might fit in less messages than possible into a batch. + # + # The alternative is to use our own `Queue` implementation that allows + # peeking, and to consider the next message size when calculating whether + # the message can be accomodated in this batch. + def size_exhausted? + @json_size >= (MAX_BYTES - Defaults::Message::MAX_BYTES) + end + end + end +end diff --git a/lib/segment/analytics/request.rb b/lib/segment/analytics/request.rb deleted file mode 100644 index 99df8290..00000000 --- a/lib/segment/analytics/request.rb +++ /dev/null @@ -1,82 +0,0 @@ -require 'segment/analytics/defaults' -require 'segment/analytics/utils' -require 'segment/analytics/response' -require 'segment/analytics/logging' -require 'net/http' -require 'net/https' -require 'json' - -module Segment - class Analytics - class Request - include Segment::Analytics::Defaults::Request - include Segment::Analytics::Utils - include Segment::Analytics::Logging - - # public: Creates a new request object to send analytics batch - # - def initialize(options = {}) - options[:host] ||= HOST - options[:port] ||= PORT - options[:ssl] ||= SSL - options[:headers] ||= HEADERS - @path = options[:path] || PATH - @retries = options[:retries] || RETRIES - @backoff = options[:backoff] || BACKOFF - - http = Net::HTTP.new(options[:host], options[:port]) - http.use_ssl = options[:ssl] - http.read_timeout = 8 - http.open_timeout = 4 - - @http = http - end - - # public: Posts the write key and batch of messages to the API. - # - # returns - Response of the status and error if it exists - def post(write_key, batch) - status, error = nil, nil - remaining_retries = @retries - backoff = @backoff - headers = { 'Content-Type' => 'application/json', 'accept' => 'application/json' } - begin - payload = JSON.generate :sentAt => datetime_in_iso8601(Time.new), :batch => batch - request = Net::HTTP::Post.new(@path, headers) - request.basic_auth write_key, nil - - if self.class.stub - status = 200 - error = nil - logger.debug "stubbed request to #{@path}: write key = #{write_key}, payload = #{payload}" - else - res = @http.request(request, payload) - status = res.code.to_i - body = JSON.parse(res.body) - error = body["error"] - end - rescue Exception => e - unless (remaining_retries -=1).zero? - sleep(backoff) - retry - end - - logger.error e.message - e.backtrace.each { |line| logger.error line } - status = -1 - error = "Connection error: #{e}" - end - - Response.new status, error - end - - class << self - attr_accessor :stub - - def stub - @stub || ENV['STUB'] - end - end - end - end -end diff --git a/lib/segment/analytics/response.rb b/lib/segment/analytics/response.rb index ebee2260..c31116a6 100644 --- a/lib/segment/analytics/response.rb +++ b/lib/segment/analytics/response.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Segment class Analytics class Response @@ -13,4 +15,3 @@ def initialize(status = 200, error = nil) end end end - diff --git a/lib/segment/analytics/test_queue.rb b/lib/segment/analytics/test_queue.rb new file mode 100644 index 00000000..c93cad2f --- /dev/null +++ b/lib/segment/analytics/test_queue.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Segment + class Analytics + class TestQueue + attr_reader :messages + + def initialize + reset! + end + + def [](key) + all[key] + end + + def count + all.count + end + + def <<(message) + all << message + send(message[:type]) << message + end + + def alias + messages[:alias] ||= [] + end + + def all + messages[:all] ||= [] + end + + def group + messages[:group] ||= [] + end + + def identify + messages[:identify] ||= [] + end + + def page + messages[:page] ||= [] + end + + def screen + messages[:screen] ||= [] + end + + def track + messages[:track] ||= [] + end + + def reset! + @messages = {} + end + end + end +end diff --git a/lib/segment/analytics/transport.rb b/lib/segment/analytics/transport.rb new file mode 100644 index 00000000..6ee14d88 --- /dev/null +++ b/lib/segment/analytics/transport.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require 'segment/analytics/defaults' +require 'segment/analytics/utils' +require 'segment/analytics/response' +require 'segment/analytics/logging' +require 'segment/analytics/backoff_policy' +require 'net/http' +require 'net/https' +require 'json' + +module Segment + class Analytics + class Transport + include Segment::Analytics::Defaults::Request + include Segment::Analytics::Utils + include Segment::Analytics::Logging + + def initialize(options = {}) + options[:host] ||= HOST + options[:port] ||= PORT + options[:ssl] ||= SSL + @headers = options[:headers] || HEADERS + @path = options[:path] || PATH + @retries = options[:retries] || RETRIES + @backoff_policy = + options[:backoff_policy] || Segment::Analytics::BackoffPolicy.new + + http = Net::HTTP.new(options[:host], options[:port]) + http.use_ssl = options[:ssl] + http.read_timeout = 8 + http.open_timeout = 4 + + @http = http + end + + # Sends a batch of messages to the API + # + # @return [Response] API response + def send(write_key, batch) + logger.debug("Sending request for #{batch.length} items") + + last_response, exception = retry_with_backoff(@retries) do + status_code, body = send_request(write_key, batch) + error = JSON.parse(body)['error'] + should_retry = should_retry_request?(status_code, body) + logger.debug("Response status code: #{status_code}") + logger.debug("Response error: #{error}") if error + + [Response.new(status_code, error), should_retry] + end + + if exception + logger.error(exception.message) + exception.backtrace.each { |line| logger.error(line) } + Response.new(-1, exception.to_s) + else + last_response + end + end + + # Closes a persistent connection if it exists + def shutdown + @http.finish if @http.started? + end + + private + + def should_retry_request?(status_code, body) + if status_code >= 500 + true # Server error + elsif status_code == 429 + true # Rate limited + elsif status_code >= 400 + logger.error(body) + false # Client error. Do not retry, but log + else + false + end + end + + # Takes a block that returns [result, should_retry]. + # + # Retries upto `retries_remaining` times, if `should_retry` is false or + # an exception is raised. `@backoff_policy` is used to determine the + # duration to sleep between attempts + # + # Returns [last_result, raised_exception] + def retry_with_backoff(retries_remaining, &block) + result, caught_exception = nil + should_retry = false + + begin + result, should_retry = yield + return [result, nil] unless should_retry + rescue StandardError => e + should_retry = true + caught_exception = e + end + + if should_retry && (retries_remaining > 1) + logger.debug("Retrying request, #{retries_remaining} retries left") + sleep(@backoff_policy.next_interval.to_f / 1000) + retry_with_backoff(retries_remaining - 1, &block) + else + [result, caught_exception] + end + end + + # Sends a request for the batch, returns [status_code, body] + def send_request(write_key, batch) + payload = JSON.generate( + :sentAt => datetime_in_iso8601(Time.now), + :batch => batch + ) + request = Net::HTTP::Post.new(@path, @headers) + request.basic_auth(write_key, nil) + + if self.class.stub + logger.debug "stubbed request to #{@path}: " \ + "write key = #{write_key}, batch = #{JSON.generate(batch)}" + + [200, '{}'] + else + @http.start unless @http.started? # Maintain a persistent connection + response = @http.request(request, payload) + [response.code.to_i, response.body] + end + end + + class << self + attr_writer :stub + + def stub + @stub || ENV['STUB'] + end + end + end + end +end diff --git a/lib/segment/analytics/utils.rb b/lib/segment/analytics/utils.rb index db964058..62ee69b0 100644 --- a/lib/segment/analytics/utils.rb +++ b/lib/segment/analytics/utils.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'securerandom' module Segment @@ -8,7 +10,9 @@ module Utils # public: Return a new hash with keys converted from strings to symbols # def symbolize_keys(hash) - hash.inject({}) { |memo, (k,v)| memo[k.to_sym] = v; memo } + hash.each_with_object({}) do |(k, v), memo| + memo[k.to_sym] = v + end end # public: Convert hash keys from strings to symbols in place @@ -20,17 +24,18 @@ def symbolize_keys!(hash) # public: Return a new hash with keys as strings # def stringify_keys(hash) - hash.inject({}) { |memo, (k,v)| memo[k.to_s] = v; memo } + hash.each_with_object({}) do |(k, v), memo| + memo[k.to_s] = v + end end # public: Returns a new hash with all the date values in the into iso8601 # strings # def isoify_dates(hash) - hash.inject({}) { |memo, (k, v)| + hash.each_with_object({}) do |(k, v), memo| memo[k] = datetime_in_iso8601(v) - memo - } + end end # public: Converts all the date values in the into iso8601 strings in place @@ -42,18 +47,18 @@ def isoify_dates!(hash) # public: Returns a uid string # def uid - arr = SecureRandom.random_bytes(16).unpack("NnnnnN") + arr = SecureRandom.random_bytes(16).unpack('NnnnnN') arr[2] = (arr[2] & 0x0fff) | 0x4000 arr[3] = (arr[3] & 0x3fff) | 0x8000 - "%08x-%04x-%04x-%04x-%04x%08x" % arr + '%08x-%04x-%04x-%04x-%04x%08x' % arr end - def datetime_in_iso8601 datetime + def datetime_in_iso8601(datetime) case datetime when Time - time_in_iso8601 datetime + time_in_iso8601 datetime when DateTime - time_in_iso8601 datetime.to_time + time_in_iso8601 datetime.to_time when Date date_in_iso8601 datetime else @@ -61,19 +66,15 @@ def datetime_in_iso8601 datetime end end - def time_in_iso8601 time, fraction_digits = 3 - fraction = if fraction_digits > 0 - (".%06i" % time.usec)[0, fraction_digits + 1] - end - - "#{time.strftime("%Y-%m-%dT%H:%M:%S")}#{fraction}#{formatted_offset(time, true, 'Z')}" + def time_in_iso8601(time) + "#{time.strftime('%Y-%m-%dT%H:%M:%S.%3N')}#{formatted_offset(time, true, 'Z')}" end - def date_in_iso8601 date - date.strftime("%F") + def date_in_iso8601(date) + date.strftime('%F') end - def formatted_offset time, colon = true, alternate_utc_string = nil + def formatted_offset(time, colon = true, alternate_utc_string = nil) time.utc? && alternate_utc_string || seconds_to_utc_offset(time.utc_offset, colon) end diff --git a/lib/segment/analytics/version.rb b/lib/segment/analytics/version.rb index 939d3388..84163496 100644 --- a/lib/segment/analytics/version.rb +++ b/lib/segment/analytics/version.rb @@ -1,5 +1,7 @@ +# frozen_string_literal: true + module Segment class Analytics - VERSION = '2.2.2' + VERSION = '2.5.0' end end diff --git a/lib/segment/analytics/worker.rb b/lib/segment/analytics/worker.rb index 9288c85f..6a7d68e7 100644 --- a/lib/segment/analytics/worker.rb +++ b/lib/segment/analytics/worker.rb @@ -1,13 +1,16 @@ +# frozen_string_literal: true + require 'segment/analytics/defaults' +require 'segment/analytics/message_batch' +require 'segment/analytics/transport' require 'segment/analytics/utils' -require 'segment/analytics/defaults' -require 'segment/analytics/request' module Segment class Analytics class Worker include Segment::Analytics::Utils include Segment::Analytics::Defaults + include Segment::Analytics::Logging # public: Creates a new worker # @@ -24,10 +27,11 @@ def initialize(queue, write_key, options = {}) symbolize_keys! options @queue = queue @write_key = write_key - @batch_size = options[:batch_size] || Queue::BATCH_SIZE - @on_error = options[:on_error] || Proc.new { |status, error| } - @batch = [] + @on_error = options[:on_error] || proc { |status, error| } + batch_size = options[:batch_size] || Defaults::MessageBatch::MAX_SIZE + @batch = MessageBatch.new(batch_size) @lock = Mutex.new + @transport = Transport.new(options) end # public: Continuously runs the loop to check for new events @@ -37,17 +41,16 @@ def run return if @queue.empty? @lock.synchronize do - until @batch.length >= @batch_size || @queue.empty? - @batch << @queue.pop - end + consume_message_from_queue! until @batch.full? || @queue.empty? end - res = Request.new.post @write_key, @batch - - @on_error.call res.status, res.error unless res.status == 200 + res = @transport.send @write_key, @batch + @on_error.call(res.status, res.error) unless res.status == 200 @lock.synchronize { @batch.clear } end + ensure + @transport.shutdown end # public: Check whether we have outstanding requests. @@ -55,6 +58,14 @@ def run def is_requesting? @lock.synchronize { !@batch.empty? } end + + private + + def consume_message_from_queue! + @batch << @queue.pop + rescue MessageBatch::JSONGenerationError => e + @on_error.call(-1, e.to_s) + end end end end diff --git a/spec/isolated/json_example.rb b/spec/isolated/json_example.rb new file mode 100644 index 00000000..5f805338 --- /dev/null +++ b/spec/isolated/json_example.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'message_batch_json' do + it 'MessageBatch generates proper JSON' do + batch = Segment::Analytics::MessageBatch.new(100) + batch << { 'a' => 'b' } + batch << { 'c' => 'd' } + + expect(JSON.generate(batch)).to eq('[{"a":"b"},{"c":"d"}]') + end +end diff --git a/spec/isolated/with_active_support.rb b/spec/isolated/with_active_support.rb new file mode 100644 index 00000000..cfd9f2cb --- /dev/null +++ b/spec/isolated/with_active_support.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'isolated/json_example' + +describe 'with active_support' do + before do + require 'active_support' + require 'active_support/json' + end + + include_examples 'message_batch_json' +end diff --git a/spec/isolated/with_active_support_and_oj.rb b/spec/isolated/with_active_support_and_oj.rb new file mode 100644 index 00000000..135be31b --- /dev/null +++ b/spec/isolated/with_active_support_and_oj.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'isolated/json_example' + +if RUBY_VERSION >= '2.0' && RUBY_PLATFORM != 'java' + describe 'with active_support and oj' do + before do + require 'active_support' + require 'active_support/json' + + require 'oj' + Oj.mimic_JSON + end + + include_examples 'message_batch_json' + end +end diff --git a/spec/isolated/with_oj.rb b/spec/isolated/with_oj.rb new file mode 100644 index 00000000..7519238e --- /dev/null +++ b/spec/isolated/with_oj.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'isolated/json_example' + +if RUBY_VERSION >= '2.0' && RUBY_PLATFORM != 'java' + describe 'with oj' do + before do + require 'oj' + Oj.mimic_JSON + end + + include_examples 'message_batch_json' + end +end diff --git a/spec/segment/analytics/backoff_policy_spec.rb b/spec/segment/analytics/backoff_policy_spec.rb new file mode 100644 index 00000000..25ef05ef --- /dev/null +++ b/spec/segment/analytics/backoff_policy_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module Segment + class Analytics + describe BackoffPolicy do + describe '#initialize' do + context 'no options are given' do + it 'sets default min_timeout_ms' do + actual = subject.instance_variable_get(:@min_timeout_ms) + expect(actual).to eq(described_class::MIN_TIMEOUT_MS) + end + + it 'sets default max_timeout_ms' do + actual = subject.instance_variable_get(:@max_timeout_ms) + expect(actual).to eq(described_class::MAX_TIMEOUT_MS) + end + + it 'sets default multiplier' do + actual = subject.instance_variable_get(:@multiplier) + expect(actual).to eq(described_class::MULTIPLIER) + end + + it 'sets default randomization factor' do + actual = subject.instance_variable_get(:@randomization_factor) + expect(actual).to eq(described_class::RANDOMIZATION_FACTOR) + end + end + + context 'options are given' do + let(:min_timeout_ms) { 1234 } + let(:max_timeout_ms) { 5678 } + let(:multiplier) { 24 } + let(:randomization_factor) { 0.4 } + + let(:options) do + { + min_timeout_ms: min_timeout_ms, + max_timeout_ms: max_timeout_ms, + multiplier: multiplier, + randomization_factor: randomization_factor + } + end + + subject { described_class.new(options) } + + it 'sets passed in min_timeout_ms' do + actual = subject.instance_variable_get(:@min_timeout_ms) + expect(actual).to eq(min_timeout_ms) + end + + it 'sets passed in max_timeout_ms' do + actual = subject.instance_variable_get(:@max_timeout_ms) + expect(actual).to eq(max_timeout_ms) + end + + it 'sets passed in multiplier' do + actual = subject.instance_variable_get(:@multiplier) + expect(actual).to eq(multiplier) + end + + it 'sets passed in randomization_factor' do + actual = subject.instance_variable_get(:@randomization_factor) + expect(actual).to eq(randomization_factor) + end + end + end + + describe '#next_interval' do + subject { + described_class.new( + min_timeout_ms: 1000, + max_timeout_ms: 10000, + multiplier: 2, + randomization_factor: 0.5 + ) + } + + it 'returns exponentially increasing durations' do + expect(subject.next_interval).to be_within(500).of(1000) + expect(subject.next_interval).to be_within(1000).of(2000) + expect(subject.next_interval).to be_within(2000).of(4000) + expect(subject.next_interval).to be_within(4000).of(8000) + end + + it 'caps maximum duration at max_timeout_secs' do + 10.times { subject.next_interval } + expect(subject.next_interval).to eq(10000) + end + end + end + end +end diff --git a/spec/segment/analytics/client_spec.rb b/spec/segment/analytics/client_spec.rb index 251962bf..99c973b3 100644 --- a/spec/segment/analytics/client_spec.rb +++ b/spec/segment/analytics/client_spec.rb @@ -1,9 +1,16 @@ +# frozen_string_literal: true + require 'spec_helper' module Segment class Analytics describe Client do - let(:client) { Client.new :write_key => WRITE_KEY } + let(:client) do + Client.new(:write_key => WRITE_KEY).tap { |client| + # Ensure that worker doesn't consume items from the queue + client.instance_variable_set(:@worker, NoopWorker.new) + } + end let(:queue) { client.instance_variable_get :@queue } describe '#initialize' do @@ -38,13 +45,13 @@ class Analytics client.track({ :user_id => 'user', :event => 'Event', - :properties => [1,2,3] + :properties => [1, 2, 3] }) }.to raise_error(ArgumentError) end it 'uses the timestamp given' do - time = Time.parse("1990-07-16 13:30:00.123 UTC") + time = Time.parse('1990-07-16 13:30:00.123 UTC') client.track({ :event => 'testing the timestamp', @@ -71,29 +78,34 @@ class Analytics end.to_not raise_error end - it 'converts time and date traits into iso8601 format' do + it 'converts time and date properties into iso8601 format' do client.track({ :user_id => 'user', :event => 'Event', :properties => { :time => Time.utc(2013), - :time_with_zone => Time.zone.parse('2013-01-01'), - :date_time => DateTime.new(2013,1,1), - :date => Date.new(2013,1,1), + :time_with_zone => Time.zone.parse('2013-01-01'), + :date_time => DateTime.new(2013, 1, 1), + :date => Date.new(2013, 1, 1), :nottime => 'x' } }) + message = queue.pop + properties = message[:properties] - expect(message[:properties][:time]).to eq('2013-01-01T00:00:00.000Z') - expect(message[:properties][:time_with_zone]).to eq('2013-01-01T00:00:00.000Z') - expect(message[:properties][:date_time]).to eq('2013-01-01T00:00:00.000Z') - expect(message[:properties][:date]).to eq('2013-01-01') - expect(message[:properties][:nottime]).to eq('x') + date_time = DateTime.new(2013, 1, 1) + expect(Time.iso8601(properties[:time])).to eq(date_time) + expect(Time.iso8601(properties[:time_with_zone])).to eq(date_time) + expect(Time.iso8601(properties[:date_time])).to eq(date_time) + + date = Date.new(2013, 1, 1) + expect(Date.iso8601(properties[:date])).to eq(date) + + expect(properties[:nottime]).to eq('x') end end - describe '#identify' do it 'errors without any user id' do expect { client.identify({}) }.to raise_error(ArgumentError) @@ -119,19 +131,24 @@ class Analytics :traits => { :time => Time.utc(2013), :time_with_zone => Time.zone.parse('2013-01-01'), - :date_time => DateTime.new(2013,1,1), - :date => Date.new(2013,1,1), + :date_time => DateTime.new(2013, 1, 1), + :date => Date.new(2013, 1, 1), :nottime => 'x' } }) message = queue.pop + traits = message[:traits] + + date_time = DateTime.new(2013, 1, 1) + expect(Time.iso8601(traits[:time])).to eq(date_time) + expect(Time.iso8601(traits[:time_with_zone])).to eq(date_time) + expect(Time.iso8601(traits[:date_time])).to eq(date_time) - expect(message[:traits][:time]).to eq('2013-01-01T00:00:00.000Z') - expect(message[:traits][:time_with_zone]).to eq('2013-01-01T00:00:00.000Z') - expect(message[:traits][:date_time]).to eq('2013-01-01T00:00:00.000Z') - expect(message[:traits][:date]).to eq('2013-01-01') - expect(message[:traits][:nottime]).to eq('x') + date = Date.new(2013, 1, 1) + expect(Date.iso8601(traits[:date])).to eq(date) + + expect(traits[:nottime]).to eq('x') end end @@ -156,10 +173,6 @@ class Analytics end describe '#group' do - after do - client.flush - end - it 'errors without group_id' do expect { client.group :user_id => 'foo' }.to raise_error(ArgumentError) end @@ -183,19 +196,24 @@ class Analytics :traits => { :time => Time.utc(2013), :time_with_zone => Time.zone.parse('2013-01-01'), - :date_time => DateTime.new(2013,1,1), - :date => Date.new(2013,1,1), + :date_time => DateTime.new(2013, 1, 1), + :date => Date.new(2013, 1, 1), :nottime => 'x' } }) message = queue.pop + traits = message[:traits] + + date_time = DateTime.new(2013, 1, 1) + expect(Time.iso8601(traits[:time])).to eq(date_time) + expect(Time.iso8601(traits[:time_with_zone])).to eq(date_time) + expect(Time.iso8601(traits[:date_time])).to eq(date_time) - expect(message[:traits][:time]).to eq('2013-01-01T00:00:00.000Z') - expect(message[:traits][:time_with_zone]).to eq('2013-01-01T00:00:00.000Z') - expect(message[:traits][:date_time]).to eq('2013-01-01T00:00:00.000Z') - expect(message[:traits][:date]).to eq('2013-01-01') - expect(message[:traits][:nottime]).to eq('x') + date = Date.new(2013, 1, 1) + expect(Date.iso8601(traits[:date])).to eq(date) + + expect(traits[:nottime]).to eq('x') end end @@ -208,10 +226,12 @@ class Analytics expect { client.page Queued::PAGE }.to_not raise_error end - it 'does not error with the required options as strings' do - expect do - client.page Utils.stringify_keys(Queued::PAGE) - end.to_not raise_error + it 'accepts name' do + client.page :name => 'foo', :user_id => 1234 + + message = queue.pop + expect(message[:userId]).to eq(1234) + expect(message[:name]).to eq('foo') end end @@ -232,34 +252,43 @@ class Analytics end describe '#flush' do + let(:client_with_worker) { + Client.new(:write_key => WRITE_KEY).tap { |client| + queue = client.instance_variable_get(:@queue) + client.instance_variable_set(:@worker, DummyWorker.new(queue)) + } + } + it 'waits for the queue to finish on a flush' do - client.identify Queued::IDENTIFY - client.track Queued::TRACK - client.flush + client_with_worker.identify Queued::IDENTIFY + client_with_worker.track Queued::TRACK + client_with_worker.flush - expect(client.queued_messages).to eq(0) + expect(client_with_worker.queued_messages).to eq(0) end - it 'completes when the process forks' do - client.identify Queued::IDENTIFY + unless defined? JRUBY_VERSION + it 'completes when the process forks' do + client_with_worker.identify Queued::IDENTIFY - Process.fork do - client.track Queued::TRACK - client.flush - expect(client.queued_messages).to eq(0) - end + Process.fork do + client_with_worker.track Queued::TRACK + client_with_worker.flush + expect(client_with_worker.queued_messages).to eq(0) + end - Process.wait - end unless defined? JRUBY_VERSION + Process.wait + end + end end context 'common' do check_property = proc { |msg, k, v| msg[k] && msg[k] == v } - let(:data) { { :user_id => 1, :group_id => 2, :previous_id => 3, :anonymous_id => 4, :message_id => 5, :event => "coco barked", :name => "coco" } } + let(:data) { { :user_id => 1, :group_id => 2, :previous_id => 3, :anonymous_id => 4, :message_id => 5, :event => 'coco barked', :name => 'coco' } } it 'does not convert ids given as fixnums to strings' do - [:track, :screen, :page, :identify].each do |s| + %i[track screen page identify].each do |s| client.send(s, data) message = queue.pop(true) @@ -268,8 +297,18 @@ class Analytics end end + it 'returns false if queue is full' do + client.instance_variable_set(:@max_queue_size, 1) + + %i[track screen page group identify alias].each do |s| + expect(client.send(s, data)).to eq(true) + expect(client.send(s, data)).to eq(false) # Queue is full + queue.pop(true) + end + end + it 'converts message id to string' do - [:track, :screen, :page, :group, :identify, :alias].each do |s| + %i[track screen page group identify alias].each do |s| client.send(s, data) message = queue.pop(true) @@ -298,13 +337,28 @@ class Analytics end it 'sends integrations' do - [:track, :screen, :page, :group, :identify, :alias].each do |s| - client.send s, :integrations => { :All => true, :Salesforce => false }, :user_id => 1, :group_id => 2, :previous_id => 3, :anonymous_id => 4, :event => "coco barked", :name => "coco" + %i[track screen page group identify alias].each do |s| + client.send s, :integrations => { :All => true, :Salesforce => false }, :user_id => 1, :group_id => 2, :previous_id => 3, :anonymous_id => 4, :event => 'coco barked', :name => 'coco' message = queue.pop(true) expect(message[:integrations][:All]).to eq(true) expect(message[:integrations][:Salesforce]).to eq(false) end end + + it 'does not enqueue the action in test mode' do + client.instance_variable_set(:@test, true) + client.test_queue + test_queue = client.instance_variable_get(:@test_queue) + + %i[track screen page group identify alias].each do |s| + old_test_queue_size = test_queue.count + queue_size = queue.length + client.send(s, data) + + expect(queue.length).to eq(queue_size) # The "real" queue size should not change in test mode + expect(test_queue.count).to_not eq(old_test_queue_size) # The "test" queue size should change in test mode + end + end end end end diff --git a/spec/segment/analytics/message_batch_spec.rb b/spec/segment/analytics/message_batch_spec.rb new file mode 100644 index 00000000..b85930af --- /dev/null +++ b/spec/segment/analytics/message_batch_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module Segment + class Analytics + describe MessageBatch do + subject { described_class.new(100) } + + describe '#<<' do + it 'appends messages' do + expect(subject.length).to eq(0) + + subject << { 'a' => 'b' } + + expect(subject.length).to eq(1) + end + + it 'rejects messages that exceed the maximum allowed size' do + max_bytes = Defaults::Message::MAX_BYTES + message = { 'a' => 'b' * max_bytes } + + expect(subject.length).to eq(0) + + expect { subject << message }.to raise_error(MessageBatch::JSONGenerationError) + .with_message('Message Exceeded Maximum Allowed Size') + end + end + + describe '#full?' do + it 'returns true once item count is exceeded' do + 99.times { subject << { a: 'b' } } + expect(subject.full?).to be(false) + + subject << { a: 'b' } + expect(subject.full?).to be(true) + end + + it 'returns true once max size is almost exceeded' do + message = { a: 'b' * (Defaults::Message::MAX_BYTES - 10) } + + message_size = message.to_json.bytesize + + # Each message is under the individual limit + expect(message_size).to be < Defaults::Message::MAX_BYTES + + # Size of the batch is over the limit + expect(50 * message_size).to be > Defaults::MessageBatch::MAX_BYTES + + expect(subject.full?).to be(false) + 50.times { subject << message } + expect(subject.full?).to be(true) + end + end + end + end +end diff --git a/spec/segment/analytics/request_spec.rb b/spec/segment/analytics/request_spec.rb deleted file mode 100644 index 40bdc3df..00000000 --- a/spec/segment/analytics/request_spec.rb +++ /dev/null @@ -1,191 +0,0 @@ -require 'spec_helper' - -module Segment - class Analytics - describe Request do - before do - # Try and keep debug statements out of tests - allow(subject.logger).to receive(:error) - allow(subject.logger).to receive(:debug) - end - - describe '#initialize' do - let!(:net_http) { Net::HTTP.new(anything, anything) } - - before do - allow(Net::HTTP).to receive(:new) { net_http } - end - - it 'sets an initalized Net::HTTP read_timeout' do - expect(net_http).to receive(:use_ssl=) - described_class.new - end - - it 'sets an initalized Net::HTTP read_timeout' do - expect(net_http).to receive(:read_timeout=) - described_class.new - end - - it 'sets an initalized Net::HTTP open_timeout' do - expect(net_http).to receive(:open_timeout=) - described_class.new - end - - it 'sets the http client' do - expect(subject.instance_variable_get(:@http)).to_not be_nil - end - - context 'no options are set' do - it 'sets a default path' do - expect(subject.instance_variable_get(:@path)).to eq(described_class::PATH) - end - - it 'sets a default retries' do - expect(subject.instance_variable_get(:@retries)).to eq(described_class::RETRIES) - end - - it 'sets a default backoff' do - expect(subject.instance_variable_get(:@backoff)).to eq(described_class::BACKOFF) - end - - it 'initializes a new Net::HTTP with default host and port' do - expect(Net::HTTP).to receive(:new).with(described_class::HOST, described_class::PORT) - described_class.new - end - end - - context 'options are given' do - let(:path) { 'my/cool/path' } - let(:retries) { 1234 } - let(:backoff) { 10 } - let(:host) { 'http://www.example.com' } - let(:port) { 8080 } - let(:options) do - { - path: path, - retries: retries, - backoff: backoff, - host: host, - port: port - } - end - - subject { described_class.new(options) } - - it 'sets passed in path' do - expect(subject.instance_variable_get(:@path)).to eq(path) - end - - it 'sets passed in retries' do - expect(subject.instance_variable_get(:@retries)).to eq(retries) - end - - it 'sets passed in backoff' do - expect(subject.instance_variable_get(:@backoff)).to eq(backoff) - end - - it 'initializes a new Net::HTTP with passed in host and port' do - expect(Net::HTTP).to receive(:new).with(host, port) - described_class.new(options) - end - end - end - - describe '#post' do - let(:response) { Net::HTTPResponse.new(http_version, status_code, response_body) } - let(:http_version) { 1.1 } - let(:status_code) { 200 } - let(:response_body) { {}.to_json } - let(:write_key) { 'abcdefg' } - let(:batch) { [] } - - before do - allow(subject.instance_variable_get(:@http)).to receive(:request) { response } - allow(response).to receive(:body) { response_body } - end - - it 'initalizes a new Net::HTTP::Post with path and default headers' do - path = subject.instance_variable_get(:@path) - default_headers = { 'Content-Type' => 'application/json', 'accept' => 'application/json' } - expect(Net::HTTP::Post).to receive(:new).with(path, default_headers).and_call_original - subject.post(write_key, batch) - end - - it 'adds basic auth to the Net::HTTP::Post' do - expect_any_instance_of(Net::HTTP::Post).to receive(:basic_auth).with(write_key, nil) - subject.post(write_key, batch) - end - - context 'with a stub' do - before do - allow(described_class).to receive(:stub) { true } - end - - it 'returns a 200 response' do - expect(subject.post(write_key, batch).status).to eq(200) - end - - it 'has a nil error' do - expect(subject.post(write_key, batch).error).to be_nil - end - - it 'logs a debug statement' do - expect(subject.logger).to receive(:debug).with(/stubbed request to/) - subject.post(write_key, batch) - end - end - - context 'a real request' do - context 'request is successful' do - let(:status_code) { 201 } - it 'returns a response code' do - expect(subject.post(write_key, batch).status).to eq(status_code) - end - - it 'returns a nil error' do - expect(subject.post(write_key, batch).error).to be_nil - end - end - - context 'request results in errorful response' do - let(:error) { 'this is an error' } - let(:response_body) { { error: error }.to_json } - - it 'returns the parsed error' do - expect(subject.post(write_key, batch).error).to eq(error) - end - end - - context 'request or parsing of response results in an exception' do - let(:response_body) { 'Malformed JSON ---' } - - let(:backoff) { 0 } - - subject { described_class.new(retries: retries, backoff: backoff) } - - context 'remaining retries is > 1' do - let(:retries) { 2 } - - it 'sleeps' do - expect(subject).to receive(:sleep).exactly(retries - 1).times - subject.post(write_key, batch) - end - end - - context 'remaining retries is 1' do - let(:retries) { 1 } - - it 'returns a -1 for status' do - expect(subject.post(write_key, batch).status).to eq(-1) - end - - it 'has a connection error' do - expect(subject.post(write_key, batch).error).to match(/Connection error/) - end - end - end - end - end - end - end -end diff --git a/spec/segment/analytics/response_spec.rb b/spec/segment/analytics/response_spec.rb index 7e099718..bb673dbf 100644 --- a/spec/segment/analytics/response_spec.rb +++ b/spec/segment/analytics/response_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' module Segment diff --git a/spec/segment/analytics/test_queue_spec.rb b/spec/segment/analytics/test_queue_spec.rb new file mode 100644 index 00000000..4aa4f77f --- /dev/null +++ b/spec/segment/analytics/test_queue_spec.rb @@ -0,0 +1,215 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module Segment + class Analytics + describe TestQueue do + let(:test_queue) { described_class.new } + + describe '#initialize' do + it 'starts empty' do + expect(test_queue.messages).to eq({}) + end + end + + describe '#<<' do + let(:message) do + { + type: type, + foo: 'bar' + } + end + + let(:expected_messages) do + { + type.to_sym => [message], + all: [message] + } + end + + context 'when unsupported type' do + let(:type) { :foo } + + it 'raises error' do + expect { test_queue << message }.to raise_error(NoMethodError) + end + end + + context 'when supported type' do + before do + test_queue << message + end + + context 'when type is alias' do + let(:type) { :alias } + + it 'adds messages' do + expect(test_queue.messages).to eq(expected_messages) + end + + it 'adds type to all' do + expect(test_queue.all).to eq([message]) + end + + it 'adds type to alias' do + expect(test_queue.alias).to eq([message]) + end + end + + context 'when type is group' do + let(:type) { :group } + + it 'adds messages' do + expect(test_queue.messages).to eq(expected_messages) + end + + it 'adds type to all' do + expect(test_queue.all).to eq([message]) + end + + it 'adds type to group' do + expect(test_queue.group).to eq([message]) + end + end + + context 'when type is identify' do + let(:type) { :identify } + + it 'adds messages' do + expect(test_queue.messages).to eq(expected_messages) + end + + it 'adds type to all' do + expect(test_queue.all).to eq([message]) + end + + it 'adds type to identify' do + expect(test_queue.identify).to eq([message]) + end + end + + context 'when type is page' do + let(:type) { :page } + + it 'adds messages' do + expect(test_queue.messages).to eq(expected_messages) + end + + it 'adds type to all' do + expect(test_queue.all).to eq([message]) + end + + it 'adds type to page' do + expect(test_queue.page).to eq([message]) + end + end + + context 'when type is screen' do + let(:type) { :screen } + + it 'adds messages' do + expect(test_queue.messages).to eq(expected_messages) + end + + it 'adds type to all' do + expect(test_queue.all).to eq([message]) + end + + it 'adds type to screen' do + expect(test_queue.screen).to eq([message]) + end + end + + context 'when type is track' do + let(:type) { :track } + + it 'adds messages' do + expect(test_queue.messages).to eq(expected_messages) + end + + it 'adds type to all' do + expect(test_queue.all).to eq([message]) + end + + it 'adds type to track' do + expect(test_queue.track).to eq([message]) + end + end + end + end + + describe '#count' do + let(:message) do + { + type: 'alias', + foo: 'bar' + } + end + + it 'returns 0' do + expect(test_queue.count).to eq(0) + end + + it 'returns 1' do + test_queue << message + expect(test_queue.count).to eq(1) + end + + it 'returns 2' do + test_queue << message + test_queue << message + expect(test_queue.count).to eq(2) + end + end + + describe '#[]' do + let(:message1) do + { + type: 'alias', + foo: 'bar' + } + end + + let(:message2) do + { + type: 'identify', + foo: 'baz' + } + end + + it 'returns message1' do + test_queue << message1 + expect(test_queue[0]).to eq(message1) + end + + it 'returns message2' do + test_queue << message2 + expect(test_queue[0]).to eq(message2) + end + + it 'returns message2' do + test_queue << message1 + test_queue << message2 + expect(test_queue[1]).to eq(message2) + end + end + + describe '#reset!' do + let(:message) do + { + type: 'alias', + foo: 'bar' + } + end + + it 'returns message' do + test_queue << message + expect(test_queue.count).to eq(1) + test_queue.reset! + expect(test_queue.messages).to eq({}) + end + end + end + end +end diff --git a/spec/segment/analytics/transport_spec.rb b/spec/segment/analytics/transport_spec.rb new file mode 100644 index 00000000..b73ce9d6 --- /dev/null +++ b/spec/segment/analytics/transport_spec.rb @@ -0,0 +1,247 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module Segment + class Analytics + describe Transport do + before do + # Try and keep debug statements out of tests + allow(subject.logger).to receive(:error) + allow(subject.logger).to receive(:debug) + end + + describe '#initialize' do + let!(:net_http) { Net::HTTP.new(anything, anything) } + + before do + allow(Net::HTTP).to receive(:new) { net_http } + end + + it 'sets an initalized Net::HTTP read_timeout' do + expect(net_http).to receive(:use_ssl=) + described_class.new + end + + it 'sets an initalized Net::HTTP read_timeout' do + expect(net_http).to receive(:read_timeout=) + described_class.new + end + + it 'sets an initalized Net::HTTP open_timeout' do + expect(net_http).to receive(:open_timeout=) + described_class.new + end + + it 'sets the http client' do + expect(subject.instance_variable_get(:@http)).to_not be_nil + end + + context 'no options are set' do + it 'sets a default path' do + path = subject.instance_variable_get(:@path) + expect(path).to eq(described_class::PATH) + end + + it 'sets a default retries' do + retries = subject.instance_variable_get(:@retries) + expect(retries).to eq(described_class::RETRIES) + end + + it 'sets a default backoff policy' do + backoff_policy = subject.instance_variable_get(:@backoff_policy) + expect(backoff_policy).to be_a(Segment::Analytics::BackoffPolicy) + end + + it 'initializes a new Net::HTTP with default host and port' do + expect(Net::HTTP).to receive(:new).with( + described_class::HOST, + described_class::PORT + ) + described_class.new + end + end + + context 'options are given' do + let(:path) { 'my/cool/path' } + let(:retries) { 1234 } + let(:backoff_policy) { FakeBackoffPolicy.new([1, 2, 3]) } + let(:host) { 'http://www.example.com' } + let(:port) { 8080 } + let(:options) do + { + path: path, + retries: retries, + backoff_policy: backoff_policy, + host: host, + port: port + } + end + + subject { described_class.new(options) } + + it 'sets passed in path' do + expect(subject.instance_variable_get(:@path)).to eq(path) + end + + it 'sets passed in retries' do + expect(subject.instance_variable_get(:@retries)).to eq(retries) + end + + it 'sets passed in backoff backoff policy' do + expect(subject.instance_variable_get(:@backoff_policy)) + .to eq(backoff_policy) + end + + it 'initializes a new Net::HTTP with passed in host and port' do + expect(Net::HTTP).to receive(:new).with(host, port) + described_class.new(options) + end + end + end + + describe '#send' do + let(:response) { + Net::HTTPResponse.new(http_version, status_code, response_body) + } + let(:http_version) { 1.1 } + let(:status_code) { 200 } + let(:response_body) { {}.to_json } + let(:write_key) { 'abcdefg' } + let(:batch) { [] } + + before do + http = subject.instance_variable_get(:@http) + allow(http).to receive(:start) + allow(http).to receive(:request) { response } + allow(response).to receive(:body) { response_body } + end + + it 'initalizes a new Net::HTTP::Post with path and default headers' do + path = subject.instance_variable_get(:@path) + default_headers = { + 'Content-Type' => 'application/json', + 'Accept' => 'application/json', + 'User-Agent' => "analytics-ruby/#{Analytics::VERSION}" + } + expect(Net::HTTP::Post).to receive(:new).with( + path, default_headers + ).and_call_original + + subject.send(write_key, batch) + end + + it 'adds basic auth to the Net::HTTP::Post' do + expect_any_instance_of(Net::HTTP::Post).to receive(:basic_auth) + .with(write_key, nil) + + subject.send(write_key, batch) + end + + context 'with a stub' do + before do + allow(described_class).to receive(:stub) { true } + end + + it 'returns a 200 response' do + expect(subject.send(write_key, batch).status).to eq(200) + end + + it 'has a nil error' do + expect(subject.send(write_key, batch).error).to be_nil + end + + it 'logs a debug statement' do + expect(subject.logger).to receive(:debug).with(/stubbed request to/) + subject.send(write_key, batch) + end + end + + context 'a real request' do + RSpec.shared_examples('retried request') do |status_code, body| + let(:status_code) { status_code } + let(:body) { body } + let(:retries) { 4 } + let(:backoff_policy) { FakeBackoffPolicy.new([1000, 1000, 1000]) } + subject { + described_class.new(retries: retries, + backoff_policy: backoff_policy) + } + + it 'retries the request' do + expect(subject) + .to receive(:sleep) + .exactly(retries - 1).times + .with(1) + .and_return(nil) + subject.send(write_key, batch) + end + end + + RSpec.shared_examples('non-retried request') do |status_code, body| + let(:status_code) { status_code } + let(:body) { body } + let(:retries) { 4 } + let(:backoff) { 1 } + subject { described_class.new(retries: retries, backoff: backoff) } + + it 'does not retry the request' do + expect(subject) + .to receive(:sleep) + .never + subject.send(write_key, batch) + end + end + + context 'request is successful' do + let(:status_code) { 201 } + it 'returns a response code' do + expect(subject.send(write_key, batch).status).to eq(status_code) + end + + it 'returns a nil error' do + expect(subject.send(write_key, batch).error).to be_nil + end + end + + context 'request results in errorful response' do + let(:error) { 'this is an error' } + let(:response_body) { { error: error }.to_json } + + it 'returns the parsed error' do + expect(subject.send(write_key, batch).error).to eq(error) + end + end + + context 'a request returns a failure status code' do + # Server errors must be retried + it_behaves_like('retried request', 500, '{}') + it_behaves_like('retried request', 503, '{}') + + # All 4xx errors other than 429 (rate limited) must be retried + it_behaves_like('retried request', 429, '{}') + it_behaves_like('non-retried request', 404, '{}') + it_behaves_like('non-retried request', 400, '{}') + end + + context 'request or parsing of response results in an exception' do + let(:response_body) { 'Malformed JSON ---' } + + subject { described_class.new(retries: 0) } + + it 'returns a -1 for status' do + expect(subject.send(write_key, batch).status).to eq(-1) + end + + it 'has a connection error' do + error = subject.send(write_key, batch).error + expect(error).to match(/Malformed JSON/) + end + + it_behaves_like('retried request', 200, 'Malformed JSON ---') + end + end + end + end + end +end diff --git a/spec/segment/analytics/worker_spec.rb b/spec/segment/analytics/worker_spec.rb index 1accd223..5111943a 100644 --- a/spec/segment/analytics/worker_spec.rb +++ b/spec/segment/analytics/worker_spec.rb @@ -1,13 +1,22 @@ +# frozen_string_literal: true + require 'spec_helper' module Segment class Analytics describe Worker do - describe "#init" do + before do + Segment::Analytics::Transport.stub = true + end + + describe '#init' do it 'accepts string keys' do queue = Queue.new - worker = Segment::Analytics::Worker.new(queue, 'secret', 'batch_size' => 100) - expect(worker.instance_variable_get(:@batch_size)).to eq(100) + worker = Segment::Analytics::Worker.new(queue, + 'secret', + 'batch_size' => 100) + batch = worker.instance_variable_get(:@batch) + expect(batch.instance_variable_get(:@max_message_count)).to eq(100) end end @@ -20,9 +29,12 @@ class Analytics Segment::Analytics::Defaults::Request::BACKOFF = 30.0 end - it 'does not error if the endpoint is unreachable' do + it 'does not error if the request fails' do expect do - Net::HTTP.any_instance.stub(:post).and_raise(Exception) + Segment::Analytics::Transport + .any_instance + .stub(:send) + .and_return(Segment::Analytics::Response.new(-1, 'Unknown error')) queue = Queue.new queue << {} @@ -31,29 +43,33 @@ class Analytics expect(queue).to be_empty - Net::HTTP.any_instance.unstub(:post) + Segment::Analytics::Transport.any_instance.unstub(:send) end.to_not raise_error end - it 'executes the error handler, before the request phase ends, if the request is invalid' do - Segment::Analytics::Request.any_instance.stub(:post).and_return(Segment::Analytics::Response.new(400, "Some error")) + it 'executes the error handler if the request is invalid' do + Segment::Analytics::Transport + .any_instance + .stub(:send) + .and_return(Segment::Analytics::Response.new(400, 'Some error')) status = error = nil - on_error = Proc.new do |yielded_status, yielded_error| + on_error = proc do |yielded_status, yielded_error| sleep 0.2 # Make this take longer than thread spin-up (below) status, error = yielded_status, yielded_error end queue = Queue.new queue << {} - worker = Segment::Analytics::Worker.new queue, 'secret', :on_error => on_error + worker = described_class.new(queue, 'secret', :on_error => on_error) - # This is to ensure that Client#flush doesn’t finish before calling the error handler. + # This is to ensure that Client#flush doesn't finish before calling + # the error handler. Thread.new { worker.run } sleep 0.1 # First give thread time to spin-up. sleep 0.01 while worker.is_requesting? - Segment::Analytics::Request::any_instance.unstub(:post) + Segment::Analytics::Transport.any_instance.unstub(:send) expect(queue).to be_empty expect(status).to eq(400) @@ -61,7 +77,7 @@ class Analytics end it 'does not call on_error if the request is good' do - on_error = Proc.new do |status, error| + on_error = proc do |status, error| puts "#{status}, #{error}" end @@ -69,11 +85,36 @@ class Analytics queue = Queue.new queue << Requested::TRACK - worker = Segment::Analytics::Worker.new queue, 'testsecret', :on_error => on_error + worker = described_class.new(queue, + 'testsecret', + :on_error => on_error) worker.run expect(queue).to be_empty end + + it 'calls on_error for bad json' do + bad_obj = Object.new + def bad_obj.to_json(*_args) + raise "can't serialize to json" + end + + on_error = proc {} + expect(on_error).to receive(:call).once.with(-1, /serialize to json/) + + good_message = Requested::TRACK + bad_message = Requested::TRACK.merge({ 'bad_obj' => bad_obj }) + + queue = Queue.new + queue << good_message + queue << bad_message + + worker = described_class.new(queue, + 'testsecret', + :on_error => on_error) + worker.run + expect(queue).to be_empty + end end describe '#is_requesting?' do @@ -85,16 +126,24 @@ class Analytics end it 'returns true if there is a current batch' do + Segment::Analytics::Transport + .any_instance + .stub(:send) { + sleep(0.2) + Segment::Analytics::Response.new(200, 'Success') + } + queue = Queue.new queue << Requested::TRACK worker = Segment::Analytics::Worker.new(queue, 'testsecret') - Thread.new do - worker.run - expect(worker.is_requesting?).to eq(false) - end - + worker_thread = Thread.new { worker.run } eventually { expect(worker.is_requesting?).to eq(true) } + + worker_thread.join + expect(worker.is_requesting?).to eq(false) + + Segment::Analytics::Transport.any_instance.unstub(:send) end end end diff --git a/spec/segment/analytics_spec.rb b/spec/segment/analytics_spec.rb index a2aa885d..04431de6 100644 --- a/spec/segment/analytics_spec.rb +++ b/spec/segment/analytics_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' module Segment @@ -10,42 +12,50 @@ class Analytics expect { analytics.track(:user_id => 'user') }.to raise_error(ArgumentError) end - it 'errors without a user_id' do - expect { analytics.track(:event => 'Event') }.to raise_error(ArgumentError) + it 'errors without user_id or anonymous_id' do + expect { analytics.track :event => 'event' }.to raise_error(ArgumentError) + expect { analytics.track :event => 'event', user_id: '' }.to raise_error(ArgumentError) + expect { analytics.track :event => 'event', anonymous_id: '' }.to raise_error(ArgumentError) + expect { analytics.track :event => 'event', user_id: '1234' }.to_not raise_error(ArgumentError) + expect { analytics.track :event => 'event', anonymous_id: '2345' }.to_not raise_error(ArgumentError) end it 'does not error with the required options' do expect do analytics.track Queued::TRACK - sleep(1) + analytics.flush end.to_not raise_error end end describe '#identify' do - it 'errors without a user_id' do + it 'errors without user_id or anonymous_id' do expect { analytics.identify :traits => {} }.to raise_error(ArgumentError) + expect { analytics.identify :traits => {}, user_id: '1234' }.to_not raise_error(ArgumentError) + expect { analytics.identify :traits => {}, anonymous_id: '2345' }.to_not raise_error(ArgumentError) end it 'does not error with the required options' do analytics.identify Queued::IDENTIFY - sleep(1) + analytics.flush end end describe '#alias' do - it 'errors without from' do + it 'errors without previous_id' do expect { analytics.alias :user_id => 1234 }.to raise_error(ArgumentError) end - it 'errors without to' do - expect { analytics.alias :previous_id => 1234 }.to raise_error(ArgumentError) + it 'errors without user_id or anonymous_id' do + expect { analytics.alias :previous_id => 'foo' }.to raise_error(ArgumentError) + expect { analytics.alias :previous_id => 'foo', user_id: '1234' }.to_not raise_error(ArgumentError) + expect { analytics.alias :previous_id => 'foo', anonymous_id: '2345' }.to_not raise_error(ArgumentError) end it 'does not error with the required options' do expect do analytics.alias ALIAS - sleep(1) + analytics.flush end.to_not raise_error end end @@ -57,12 +67,14 @@ class Analytics it 'errors without user_id or anonymous_id' do expect { analytics.group :group_id => 'foo' }.to raise_error(ArgumentError) + expect { analytics.group :group_id => 'foo', user_id: '1234' }.to_not raise_error(ArgumentError) + expect { analytics.group :group_id => 'foo', anonymous_id: '2345' }.to_not raise_error(ArgumentError) end it 'does not error with the required options' do expect do analytics.group Queued::GROUP - sleep(1) + analytics.flush end.to_not raise_error end end @@ -70,12 +82,14 @@ class Analytics describe '#page' do it 'errors without user_id or anonymous_id' do expect { analytics.page :name => 'foo' }.to raise_error(ArgumentError) + expect { analytics.page :name => 'foo', user_id: '1234' }.to_not raise_error(ArgumentError) + expect { analytics.page :name => 'foo', anonymous_id: '2345' }.to_not raise_error(ArgumentError) end it 'does not error with the required options' do expect do analytics.page Queued::PAGE - sleep(1) + analytics.flush end.to_not raise_error end end @@ -83,12 +97,14 @@ class Analytics describe '#screen' do it 'errors without user_id or anonymous_id' do expect { analytics.screen :name => 'foo' }.to raise_error(ArgumentError) + expect { analytics.screen :name => 'foo', user_id: '1234' }.to_not raise_error(ArgumentError) + expect { analytics.screen :name => 'foo', anonymous_id: '2345' }.to_not raise_error(ArgumentError) end it 'does not error with the required options' do expect do analytics.screen Queued::SCREEN - sleep(1) + analytics.flush end.to_not raise_error end end @@ -101,6 +117,47 @@ class Analytics end.to_not raise_error end end + + describe '#respond_to?' do + it 'responds to all public instance methods of Segment::Analytics::Client' do + expect(analytics).to respond_to(*Segment::Analytics::Client.public_instance_methods(false)) + end + end + + describe '#method' do + Segment::Analytics::Client.public_instance_methods(false).each do |public_method| + it "returns a Method object with '#{public_method}' as argument" do + expect(analytics.method(public_method).class).to eq(Method) + end + end + end + + describe '#test_queue' do + context 'when not in mode' do + let(:analytics) { Segment::Analytics.new :write_key => WRITE_KEY, :stub => true, :test => true } + + it 'returns TestQueue' do + expect(analytics.test_queue).to be_a(TestQueue) + end + + it 'returns event' do + analytics.track Queued::TRACK + expect(analytics.test_queue[0]).to include(Requested::TRACK) + expect(analytics.test_queue.track[0]).to include(Requested::TRACK) + end + end + + context 'when not in test mode' do + let(:analytics) { Segment::Analytics.new :write_key => WRITE_KEY, :stub => true, :test => false } + + it 'errors when not in test mode' do + expect(analytics.instance_variable_get(:@test)).to be_falsey + expect { analytics.test_queue }.to raise_error( + RuntimeError, 'Test queue only available when setting :test to true.' + ) + end + end + end end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 5fc2ff73..8dc8634c 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,9 +1,13 @@ +# frozen_string_literal: true + +require 'simplecov' +SimpleCov.start +require 'simplecov-cobertura' +SimpleCov.formatter = SimpleCov::Formatter::CoberturaFormatter + require 'segment/analytics' -require 'wrong' require 'active_support/time' -include Wrong - # Setting timezone for ActiveSupport::TimeWithZone to UTC Time.zone = 'UTC' @@ -21,7 +25,7 @@ class Analytics } } - IDENTIFY = { + IDENTIFY = { :traits => { :likes_animals => true, :instrument => 'Guitar', @@ -36,9 +40,7 @@ class Analytics GROUP = {} - PAGE = { - :name => 'home' - } + PAGE = {} SCREEN = { :name => 'main' @@ -79,3 +81,61 @@ module Requested end end end + +# A worker that doesn't consume jobs +class NoopWorker + def run + # Does nothing + end +end + +# A worker that consumes all jobs +class DummyWorker + def initialize(queue) + @queue = queue + end + + def run + @queue.pop until @queue.empty? + end + + def is_requesting? + false + end +end + +# A backoff policy that returns a fixed list of values +class FakeBackoffPolicy + def initialize(interval_values) + @interval_values = interval_values + end + + def next_interval + raise 'FakeBackoffPolicy has no values left' if @interval_values.empty? + @interval_values.shift + end +end + +# usage: +# it "should return a result of 5" do +# eventually(options: {timeout: 1}) { long_running_thing.result.should eq(5) } +# end + +module AsyncHelper + def eventually(options = {}) + timeout = options[:timeout] || 2 + interval = options[:interval] || 0.1 + time_limit = Time.now + timeout + loop do + begin + yield + return + rescue RSpec::Expectations::ExpectationNotMetError => error + raise error if Time.now >= time_limit + sleep interval + end + end + end +end + +include AsyncHelper