Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ruby-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
ruby-2.2.4
ruby-2.4.4
7 changes: 5 additions & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
language: ruby
rvm:
- 2.2
- 2.3
- 2.4
- 2.5
- 2.6
- 2.7
- ruby-head
script: bundle exec rake
jobs:
allow_failures:
- rvm: ruby-head
2 changes: 1 addition & 1 deletion Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -125,4 +125,4 @@ DEPENDENCIES
sqlite3

BUNDLED WITH
1.14.6
2.1.4
22 changes: 22 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,28 @@ Book.bulk_insert(*destination_columns, update_duplicates: %w[title]) do |worker|
end
```

### Ignored Columns On Update (MySQL, PostgreSQL)

When there is a conflict in _insert_ and you want to instead update those records you would be using the _update_duplicates_ option. Along with that you can also ignore certain columns which you don't want to be updated. For example you may want to ignore `created_at` field during the update.

```ruby
destination_columns = [:title, :created_at, :updated_at]

# Ignored columns on update (MySQL)
Book.bulk_insert(*destination_columns, update_duplicates: true, ignored_columns_on_update: %w(created_at)) do |worker|
worker.add(...)
# ...
end

# Ignored columns on update (PostgreSQL)
Book.bulk_insert(*destination_columns, update_duplicates: %w[title], ignored_columns_on_update: %w(created_at)) do |worker|
worker.add(...)
# ...
end
```

You can pass a single column name, or a list of column names to `ignored_columns_on_update`.

### Return Primary Keys (PostgreSQL, PostGIS)

If you want the worker to store primary keys of inserted records, then you can
Expand Down
4 changes: 2 additions & 2 deletions lib/bulk_insert.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ module BulkInsert
extend ActiveSupport::Concern

module ClassMethods
def bulk_insert(*columns, values: nil, set_size:500, ignore: false, update_duplicates: false, return_primary_keys: false)
def bulk_insert(*columns, values: nil, set_size:500, ignore: false, update_duplicates: false, return_primary_keys: false, ignored_columns_on_update: nil)
columns = default_bulk_columns if columns.empty?
worker = BulkInsert::Worker.new(connection, table_name, primary_key, columns, set_size, ignore, update_duplicates, return_primary_keys)
worker = BulkInsert::Worker.new(connection, table_name, primary_key, columns, set_size, ignore, update_duplicates, return_primary_keys, ignored_columns_on_update)

if values.present?
transaction do
Expand Down
8 changes: 5 additions & 3 deletions lib/bulk_insert/worker.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ class Worker
attr_accessor :adapter_name
attr_reader :ignore, :update_duplicates, :result_sets

def initialize(connection, table_name, primary_key, column_names, set_size=500, ignore=false, update_duplicates=false, return_primary_keys=false)
def initialize(connection, table_name, primary_key, column_names, set_size=500, ignore=false, update_duplicates=false, return_primary_keys=false, ignored_columns_on_update=nil)
@connection = connection
@set_size = set_size

Expand All @@ -16,12 +16,14 @@ def initialize(connection, table_name, primary_key, column_names, set_size=500,
@ignore = ignore
@update_duplicates = update_duplicates
@return_primary_keys = return_primary_keys
@ignored_columns_on_update = Array(ignored_columns_on_update)

columns = connection.columns(table_name)
column_map = columns.inject({}) { |h, c| h.update(c.name => c) }

@primary_key = primary_key
@columns = column_names.map { |name| column_map[name.to_s] }
@columns_for_update = @columns.reject { |column| @ignored_columns_on_update.include?(column.name) }
@table_name = connection.quote_table_name(table_name)
@column_names = column_names.map { |name| connection.quote_column_name(name) }.join(",")

Expand Down Expand Up @@ -154,12 +156,12 @@ def on_conflict_statement
if is_postgres && ignore
' ON CONFLICT DO NOTHING'
elsif is_postgres && update_duplicates
update_values = @columns.map do |column|
update_values = @columns_for_update.map do |column|
"#{column.name}=EXCLUDED.#{column.name}"
end.join(', ')
' ON CONFLICT(' + update_duplicates.join(', ') + ') DO UPDATE SET ' + update_values
elsif adapter_name =~ /^mysql/i && update_duplicates
update_values = @columns.map do |column|
update_values = @columns_for_update.map do |column|
"`#{column.name}`=VALUES(`#{column.name}`)"
end.join(', ')
' ON DUPLICATE KEY UPDATE ' + update_values
Expand Down
34 changes: 34 additions & 0 deletions test/bulk_insert/worker_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -418,4 +418,38 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase

assert_equal mysql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
end

test "mysql adapter can ignore specific columns when updating duplicates" do
mysql_worker = BulkInsert::Worker.new(
Testing.connection,
Testing.table_name,
'id',
%w(greeting age happy created_at updated_at color),
500, # batch size
false, # ignore
true, # update_duplicates
false, # return_primary_keys
%w(created_at)) # ignored_columns_on_update
mysql_worker.adapter_name = 'MySQL'
mysql_worker.add ["Yo", 15, false, nil, nil]

assert_equal mysql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
end

test "pgsql_worker adapter can ignore specific columns when updating duplicates" do
pgsql_worker = BulkInsert::Worker.new(
Testing.connection,
Testing.table_name,
'id',
%w(greeting age happy created_at updated_at color),
500, # batch size
false, # ignore
%w(greeting), # update_duplicates
false, # return_primary_keys
%w(created_at)) # ignored_columns_on_update
pgsql_worker.adapter_name = 'PostgreSQL'
pgsql_worker.add ["Yo", 15, false, nil, nil]

assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT(greeting) DO UPDATE SET greeting=EXCLUDED.greeting, age=EXCLUDED.age, happy=EXCLUDED.happy, updated_at=EXCLUDED.updated_at, color=EXCLUDED.color"
end
end