From fa25dd0a8f4d0d8c10b784e5c769d3850c28f8c3 Mon Sep 17 00:00:00 2001 From: Aqeel Nazeer Date: Fri, 24 Jan 2020 10:58:59 +0000 Subject: [PATCH] Option to add ignored columns on update --- .ruby-version | 2 +- .travis.yml | 7 +++++-- Gemfile.lock | 2 +- README.md | 22 +++++++++++++++++++++ lib/bulk_insert.rb | 4 ++-- lib/bulk_insert/worker.rb | 8 +++++--- test/bulk_insert/worker_test.rb | 34 +++++++++++++++++++++++++++++++++ 7 files changed, 70 insertions(+), 9 deletions(-) diff --git a/.ruby-version b/.ruby-version index c063f5a..c9b8707 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -ruby-2.2.4 +ruby-2.4.4 diff --git a/.travis.yml b/.travis.yml index 4164274..e56de25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,11 @@ language: ruby rvm: - - 2.2 - - 2.3 - 2.4 - 2.5 + - 2.6 + - 2.7 - ruby-head script: bundle exec rake +jobs: + allow_failures: + - rvm: ruby-head diff --git a/Gemfile.lock b/Gemfile.lock index 6d3034b..760d5bf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -125,4 +125,4 @@ DEPENDENCIES sqlite3 BUNDLED WITH - 1.14.6 + 2.1.4 diff --git a/README.md b/README.md index 969ec43..79fd3ef 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,28 @@ Book.bulk_insert(*destination_columns, update_duplicates: %w[title]) do |worker| end ``` +### Ignored Columns On Update (MySQL, PostgreSQL) + +When there is a conflict in _insert_ and you want to instead update those records you would be using the _update_duplicates_ option. Along with that you can also ignore certain columns which you don't want to be updated. For example you may want to ignore `created_at` field during the update. + +```ruby +destination_columns = [:title, :created_at, :updated_at] + +# Ignored columns on update (MySQL) +Book.bulk_insert(*destination_columns, update_duplicates: true, ignored_columns_on_update: %w(created_at)) do |worker| + worker.add(...) + # ... +end + +# Ignored columns on update (PostgreSQL) +Book.bulk_insert(*destination_columns, update_duplicates: %w[title], ignored_columns_on_update: %w(created_at)) do |worker| + worker.add(...) + # ... +end +``` + +You can pass a single column name, or a list of column names to `ignored_columns_on_update`. + ### Return Primary Keys (PostgreSQL, PostGIS) If you want the worker to store primary keys of inserted records, then you can diff --git a/lib/bulk_insert.rb b/lib/bulk_insert.rb index 7907aff..7ed5e67 100644 --- a/lib/bulk_insert.rb +++ b/lib/bulk_insert.rb @@ -4,9 +4,9 @@ module BulkInsert extend ActiveSupport::Concern module ClassMethods - def bulk_insert(*columns, values: nil, set_size:500, ignore: false, update_duplicates: false, return_primary_keys: false) + def bulk_insert(*columns, values: nil, set_size:500, ignore: false, update_duplicates: false, return_primary_keys: false, ignored_columns_on_update: nil) columns = default_bulk_columns if columns.empty? - worker = BulkInsert::Worker.new(connection, table_name, primary_key, columns, set_size, ignore, update_duplicates, return_primary_keys) + worker = BulkInsert::Worker.new(connection, table_name, primary_key, columns, set_size, ignore, update_duplicates, return_primary_keys, ignored_columns_on_update) if values.present? transaction do diff --git a/lib/bulk_insert/worker.rb b/lib/bulk_insert/worker.rb index a16a938..f42153e 100644 --- a/lib/bulk_insert/worker.rb +++ b/lib/bulk_insert/worker.rb @@ -7,7 +7,7 @@ class Worker attr_accessor :adapter_name attr_reader :ignore, :update_duplicates, :result_sets - def initialize(connection, table_name, primary_key, column_names, set_size=500, ignore=false, update_duplicates=false, return_primary_keys=false) + def initialize(connection, table_name, primary_key, column_names, set_size=500, ignore=false, update_duplicates=false, return_primary_keys=false, ignored_columns_on_update=nil) @connection = connection @set_size = set_size @@ -16,12 +16,14 @@ def initialize(connection, table_name, primary_key, column_names, set_size=500, @ignore = ignore @update_duplicates = update_duplicates @return_primary_keys = return_primary_keys + @ignored_columns_on_update = Array(ignored_columns_on_update) columns = connection.columns(table_name) column_map = columns.inject({}) { |h, c| h.update(c.name => c) } @primary_key = primary_key @columns = column_names.map { |name| column_map[name.to_s] } + @columns_for_update = @columns.reject { |column| @ignored_columns_on_update.include?(column.name) } @table_name = connection.quote_table_name(table_name) @column_names = column_names.map { |name| connection.quote_column_name(name) }.join(",") @@ -154,12 +156,12 @@ def on_conflict_statement if is_postgres && ignore ' ON CONFLICT DO NOTHING' elsif is_postgres && update_duplicates - update_values = @columns.map do |column| + update_values = @columns_for_update.map do |column| "#{column.name}=EXCLUDED.#{column.name}" end.join(', ') ' ON CONFLICT(' + update_duplicates.join(', ') + ') DO UPDATE SET ' + update_values elsif adapter_name =~ /^mysql/i && update_duplicates - update_values = @columns.map do |column| + update_values = @columns_for_update.map do |column| "`#{column.name}`=VALUES(`#{column.name}`)" end.join(', ') ' ON DUPLICATE KEY UPDATE ' + update_values diff --git a/test/bulk_insert/worker_test.rb b/test/bulk_insert/worker_test.rb index 735972b..253e562 100644 --- a/test/bulk_insert/worker_test.rb +++ b/test/bulk_insert/worker_test.rb @@ -418,4 +418,38 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase assert_equal mysql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)" end + + test "mysql adapter can ignore specific columns when updating duplicates" do + mysql_worker = BulkInsert::Worker.new( + Testing.connection, + Testing.table_name, + 'id', + %w(greeting age happy created_at updated_at color), + 500, # batch size + false, # ignore + true, # update_duplicates + false, # return_primary_keys + %w(created_at)) # ignored_columns_on_update + mysql_worker.adapter_name = 'MySQL' + mysql_worker.add ["Yo", 15, false, nil, nil] + + assert_equal mysql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)" + end + + test "pgsql_worker adapter can ignore specific columns when updating duplicates" do + pgsql_worker = BulkInsert::Worker.new( + Testing.connection, + Testing.table_name, + 'id', + %w(greeting age happy created_at updated_at color), + 500, # batch size + false, # ignore + %w(greeting), # update_duplicates + false, # return_primary_keys + %w(created_at)) # ignored_columns_on_update + pgsql_worker.adapter_name = 'PostgreSQL' + pgsql_worker.add ["Yo", 15, false, nil, nil] + + assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT(greeting) DO UPDATE SET greeting=EXCLUDED.greeting, age=EXCLUDED.age, happy=EXCLUDED.happy, updated_at=EXCLUDED.updated_at, color=EXCLUDED.color" + end end