From d36c64ee2973942558b960c706848db06f50ffde Mon Sep 17 00:00:00 2001 From: bwarminski Date: Tue, 15 Jan 2019 16:23:33 -0500 Subject: [PATCH 01/49] Tests are passing in Docker container --- .rubocop.yml | 1 - Dockerfile.dev | 8 ++++++++ README.md | 19 +++++++++++++++++++ Rakefile | 3 ++- docker/docker-entrypoint.sh | 21 +++++++++++++++++++++ docker/test.sh | 4 ++++ odbc_adapter.gemspec | 2 -- 7 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 Dockerfile.dev create mode 100755 docker/docker-entrypoint.sh create mode 100755 docker/test.sh diff --git a/.rubocop.yml b/.rubocop.yml index 9055a997..34c28c16 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,7 +1,6 @@ AllCops: DisplayCopNames: true DisplayStyleGuide: true - TargetRubyVersion: 2.1 Exclude: - 'vendor/**/*' diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 00000000..0a3c557d --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,8 @@ +FROM ruby:2.4.0 +MAINTAINER data@localytics.com + +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get -y install libnss3-tools unixodbc-dev libmyodbc mysql-client odbc-postgresql postgresql + +WORKDIR /workspace +CMD docker/docker-entrypoint.sh \ No newline at end of file diff --git a/README.md b/README.md index 27f46dcc..0f5b5519 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,25 @@ ActiveRecord models that use this connection will now be connecting to the confi To run the tests, you'll need the ODBC driver as well as the connection adapter for each database against which you're trying to test. Then run `DSN=MyDatabaseDSN bundle exec rake test` and the test suite will be run by connecting to your database. +## Testing Using a Docker Container Because ODBC on Mac is Hard + +Tested on Sierra. + + +Run from project root: + +``` +bundle package +docker build -f Dockerfile.dev -t odbc-dev . + +# Local mount mysql directory to avoid some permissions problems +mkdir -p /tmp/mysql +docker run -it --rm -v $(pwd):/workspace -v /tmp/mysql:/var/lib/mysql odbc-dev:latest + +# In container +docker/test.sh +``` + ## Contributing Bug reports and pull requests are welcome on GitHub at https://github.com/localytics/odbc_adapter. diff --git a/Rakefile b/Rakefile index 6af9c2b8..2d7bced8 100644 --- a/Rakefile +++ b/Rakefile @@ -9,6 +9,7 @@ Rake::TestTask.new(:test) do |t| end RuboCop::RakeTask.new(:rubocop) -Rake::Task[:test].prerequisites << :rubocop +# Temporarily removing as a prereq to get tests back up and working +# Rake::Task[:test].prerequisites << :rubocop task default: :test diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100755 index 00000000..be611261 --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e -x + +# Installing mysql at startup due to file permissions: https://github.com/geerlingguy/drupal-vm/issues/1497 +apt-get install -y mysql-server +bundle install --local +service mysql start + +# Allows passwordless auth from command line and odbc +sed -i "s/local all postgres peer/local all postgres trust/" /etc/postgresql/9.4/main/pg_hba.conf +sed -i "s/host all all 127.0.0.1\/32 md5/host all all 127.0.0.1\/32 trust/" /etc/postgresql/9.4/main/pg_hba.conf +service postgresql start + +odbcinst -i -d -f /usr/share/libmyodbc/odbcinst.ini +mysql -e "DROP DATABASE IF EXISTS odbc_test; CREATE DATABASE IF NOT EXISTS odbc_test;" -uroot +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost';" -uroot + +odbcinst -i -d -f /usr/share/psqlodbc/odbcinst.ini.template +psql -c "CREATE DATABASE odbc_test;" -U postgres + +/bin/bash diff --git a/docker/test.sh b/docker/test.sh new file mode 100755 index 00000000..adcd1a62 --- /dev/null +++ b/docker/test.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "Testing mysql" && CONN_STR='DRIVER=MySQL;SERVER=localhost;DATABASE=odbc_test;USER=root;PASSWORD=;' bundle exec rake && \ + echo "Testing postgres" && CONN_STR='DRIVER={PostgreSQL ANSI};SERVER=localhost;PORT=5432;DATABASE=odbc_test;UID=postgres;' bundle exec rake \ No newline at end of file diff --git a/odbc_adapter.gemspec b/odbc_adapter.gemspec index ae02a406..d0bcd61c 100644 --- a/odbc_adapter.gemspec +++ b/odbc_adapter.gemspec @@ -1,5 +1,3 @@ -# coding: utf-8 - lib = File.expand_path('../lib', __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) require 'odbc_adapter/version' From 7e27df4852b66ef1b2a80ff08eab4ee7026b9065 Mon Sep 17 00:00:00 2001 From: Brett Warminski Date: Thu, 17 Jan 2019 16:51:32 -0500 Subject: [PATCH 02/49] Reenable Rubocop, add encoding config parameter (#29) Reenable Rubocop, add utf8 encoding config parameter --- .rubocop.yml | 1 + Rakefile | 3 +-- docker/test.sh | 3 ++- .../connection_adapters/odbc_adapter.rb | 27 ++++++++++++------- lib/odbc_adapter/database_metadata.rb | 11 ++++++-- lib/odbc_adapter/version.rb | 2 +- odbc_adapter.gemspec | 2 +- test/test_helper.rb | 2 +- 8 files changed, 34 insertions(+), 17 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 34c28c16..9055a997 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,6 +1,7 @@ AllCops: DisplayCopNames: true DisplayStyleGuide: true + TargetRubyVersion: 2.1 Exclude: - 'vendor/**/*' diff --git a/Rakefile b/Rakefile index 2d7bced8..6af9c2b8 100644 --- a/Rakefile +++ b/Rakefile @@ -9,7 +9,6 @@ Rake::TestTask.new(:test) do |t| end RuboCop::RakeTask.new(:rubocop) -# Temporarily removing as a prereq to get tests back up and working -# Rake::Task[:test].prerequisites << :rubocop +Rake::Task[:test].prerequisites << :rubocop task default: :test diff --git a/docker/test.sh b/docker/test.sh index adcd1a62..067de0c3 100755 --- a/docker/test.sh +++ b/docker/test.sh @@ -1,4 +1,5 @@ #!/bin/bash echo "Testing mysql" && CONN_STR='DRIVER=MySQL;SERVER=localhost;DATABASE=odbc_test;USER=root;PASSWORD=;' bundle exec rake && \ - echo "Testing postgres" && CONN_STR='DRIVER={PostgreSQL ANSI};SERVER=localhost;PORT=5432;DATABASE=odbc_test;UID=postgres;' bundle exec rake \ No newline at end of file + echo "Testing postgres" && CONN_STR='DRIVER={PostgreSQL ANSI};SERVER=localhost;PORT=5432;DATABASE=odbc_test;UID=postgres;' bundle exec rake && \ + echo "Testing postgres utf8" && CONN_STR='DRIVER={PostgreSQL UNICODE};SERVER=localhost;PORT=5432;DATABASE=odbc_test;UID=postgres;ENCODING=utf8' bundle exec rake \ No newline at end of file diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 672c5db1..33f900a2 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -1,6 +1,7 @@ require 'active_record' require 'arel/visitors/bind_visitor' require 'odbc' +require 'odbc_utf8' require 'odbc_adapter/database_limits' require 'odbc_adapter/database_statements' @@ -30,7 +31,7 @@ def odbc_connection(config) raise ArgumentError, 'No data source name (:dsn) or connection string (:conn_str) specified.' end - database_metadata = ::ODBCAdapter::DatabaseMetadata.new(connection) + database_metadata = ::ODBCAdapter::DatabaseMetadata.new(connection, config[:encoding_bug]) database_metadata.adapter_class.new(connection, logger, config, database_metadata) end @@ -40,8 +41,11 @@ def odbc_connection(config) def odbc_dsn_connection(config) username = config[:username] ? config[:username].to_s : nil password = config[:password] ? config[:password].to_s : nil - connection = ODBC.connect(config[:dsn], username, password) - [connection, config.merge(username: username, password: password)] + odbc_module = config[:encoding] == 'utf8' ? ODBC_UTF8 : ODBC + connection = odbc_module.connect(config[:dsn], username, password) + + # encoding_bug indicates that the driver is using non ASCII and has the issue referenced here https://github.com/larskanis/ruby-odbc/issues/2 + [connection, config.merge(username: username, password: password, encoding_bug: config[:encoding] == 'utf8')] end # Connect using ODBC connection string @@ -49,12 +53,15 @@ def odbc_dsn_connection(config) # e.g. "DSN=virt5;UID=rails;PWD=rails" # "DRIVER={OpenLink Virtuoso};HOST=carlmbp;UID=rails;PWD=rails" def odbc_conn_str_connection(config) - driver = ODBC::Driver.new + attrs = config[:conn_str].split(';').map { |option| option.split('=', 2) }.to_h + odbc_module = attrs['ENCODING'] == 'utf8' ? ODBC_UTF8 : ODBC + driver = odbc_module::Driver.new driver.name = 'odbc' - driver.attrs = config[:conn_str].split(';').map { |option| option.split('=', 2) }.to_h + driver.attrs = attrs - connection = ODBC::Database.new.drvconnect(driver) - [connection, config.merge(driver: driver)] + connection = odbc_module::Database.new.drvconnect(driver) + # encoding_bug indicates that the driver is using non ASCII and has the issue referenced here https://github.com/larskanis/ruby-odbc/issues/2 + [connection, config.merge(driver: driver, encoding: attrs['ENCODING'], encoding_bug: attrs['ENCODING'] == 'utf8')] end end end @@ -107,11 +114,12 @@ def active? # new connection with the database. def reconnect! disconnect! + odbc_module = @config[:encoding] == 'utf8' ? ODBC_UTF8 : ODBC @connection = if @config.key?(:dsn) - ODBC.connect(@config[:dsn], @config[:username], @config[:password]) + odbc_module.connect(@config[:dsn], @config[:username], @config[:password]) else - ODBC::Database.new.drvconnect(@config[:driver]) + odbc_module::Database.new.drvconnect(@config[:driver]) end configure_time_options(@connection) super @@ -134,6 +142,7 @@ def new_column(name, default, sql_type_metadata, null, table_name, default_funct protected # Build the type map for ActiveRecord + # Here, ODBC and ODBC_UTF8 constants are interchangeable def initialize_type_map(map) map.register_type 'boolean', Type::Boolean.new map.register_type ODBC::SQL_CHAR, Type::String.new diff --git a/lib/odbc_adapter/database_metadata.rb b/lib/odbc_adapter/database_metadata.rb index f3572e9c..11fa9255 100644 --- a/lib/odbc_adapter/database_metadata.rb +++ b/lib/odbc_adapter/database_metadata.rb @@ -15,8 +15,15 @@ class DatabaseMetadata attr_reader :values - def initialize(connection) - @values = Hash[FIELDS.map { |field| [field, connection.get_info(ODBC.const_get(field))] }] + # has_encoding_bug refers to https://github.com/larskanis/ruby-odbc/issues/2 where ruby-odbc in UTF8 mode + # returns incorrectly encoded responses to getInfo + def initialize(connection, has_encoding_bug = false) + @values = Hash[FIELDS.map do |field| + info = connection.get_info(ODBC.const_get(field)) + info = info.encode(Encoding.default_external, 'UTF-16LE') if info.is_a?(String) && has_encoding_bug + + [field, info] + end] end def adapter_class diff --git a/lib/odbc_adapter/version.rb b/lib/odbc_adapter/version.rb index 693cb713..fdebf9fb 100644 --- a/lib/odbc_adapter/version.rb +++ b/lib/odbc_adapter/version.rb @@ -1,3 +1,3 @@ module ODBCAdapter - VERSION = '5.0.3'.freeze + VERSION = '5.0.4'.freeze end diff --git a/odbc_adapter.gemspec b/odbc_adapter.gemspec index d0bcd61c..4bf0142d 100644 --- a/odbc_adapter.gemspec +++ b/odbc_adapter.gemspec @@ -24,6 +24,6 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'bundler', '~> 1.14' spec.add_development_dependency 'minitest', '~> 5.10' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 0.48' + spec.add_development_dependency 'rubocop', '0.48.1' spec.add_development_dependency 'simplecov', '~> 0.14' end diff --git a/test/test_helper.rb b/test/test_helper.rb index 65cc6d52..623b1960 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -42,7 +42,7 @@ class User < ActiveRecord::Base { first_name: 'Jason', last_name: 'Dsouza', letters: 11 }, { first_name: 'Ash', last_name: 'Hepburn', letters: 10 }, { first_name: 'Sharif', last_name: 'Younes', letters: 12 }, - { first_name: 'Ryan', last_name: 'Brown', letters: 9 } + { first_name: 'Ryan', last_name: 'Brüwn', letters: 9 } ] ) end From 8e0b7b397658820bcdb9135fbeeacff6dcd45fb3 Mon Sep 17 00:00:00 2001 From: Michael Shearer Date: Fri, 5 Apr 2019 14:52:09 -0400 Subject: [PATCH 03/49] Handle connection failure by trying to reconnect and translating the error to ConnectionFailedError so the caller knows to retry --- Dockerfile.dev | 6 +++++- .../connection_adapters/odbc_adapter.rb | 15 ++++++++++++--- lib/odbc_adapter/error.rb | 2 ++ test/connection_fail_test.rb | 19 +++++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 test/connection_fail_test.rb diff --git a/Dockerfile.dev b/Dockerfile.dev index 0a3c557d..3f602747 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -2,7 +2,11 @@ FROM ruby:2.4.0 MAINTAINER data@localytics.com ENV DEBIAN_FRONTEND noninteractive +RUN echo "deb http://deb.debian.org/debian/ jessie main" > /etc/apt/sources.list +RUN echo "deb-src http://deb.debian.org/debian/ jessie main" >> /etc/apt/sources.list +RUN echo "deb http://security.debian.org/ jessie/updates main" >> /etc/apt/sources.list +RUN echo "deb-src http://security.debian.org/ jessie/updates main" >> /etc/apt/sources.list RUN apt-get update && apt-get -y install libnss3-tools unixodbc-dev libmyodbc mysql-client odbc-postgresql postgresql WORKDIR /workspace -CMD docker/docker-entrypoint.sh \ No newline at end of file +CMD docker/docker-entrypoint.sh diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 33f900a2..06686785 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -76,9 +76,11 @@ class ODBCAdapter < AbstractAdapter ADAPTER_NAME = 'ODBC'.freeze BOOLEAN_TYPE = 'BOOLEAN'.freeze - ERR_DUPLICATE_KEY_VALUE = 23_505 - ERR_QUERY_TIMED_OUT = 57_014 - ERR_QUERY_TIMED_OUT_MESSAGE = /Query has timed out/ + ERR_DUPLICATE_KEY_VALUE = 23_505 + ERR_QUERY_TIMED_OUT = 57_014 + ERR_QUERY_TIMED_OUT_MESSAGE = /Query has timed out/ + ERR_CONNECTION_FAILED_REGEX = '^08[0S]0[12347]'.freeze + ERR_CONNECTION_FAILED_MESSAGE = /Client connection failed/ # The object that stores the information that is fetched from the DBMS # when a connection is first established. @@ -184,6 +186,13 @@ def translate_exception(exception, message) ActiveRecord::RecordNotUnique.new(message, exception) elsif error_number == ERR_QUERY_TIMED_OUT || exception.message =~ ERR_QUERY_TIMED_OUT_MESSAGE ::ODBCAdapter::QueryTimeoutError.new(message, exception) + elsif exception.message.match(ERR_CONNECTION_FAILED_REGEX) || exception.message =~ ERR_CONNECTION_FAILED_MESSAGE + begin + reconnect! + ::ODBCAdapter::ConnectionFailedError.new(message, exception) + rescue => e + puts "unable to reconnect #{e}" + end else super end diff --git a/lib/odbc_adapter/error.rb b/lib/odbc_adapter/error.rb index 1f8acfc9..d0e0172b 100644 --- a/lib/odbc_adapter/error.rb +++ b/lib/odbc_adapter/error.rb @@ -1,4 +1,6 @@ module ODBCAdapter class QueryTimeoutError < ActiveRecord::StatementInvalid end + class ConnectionFailedError < ActiveRecord::StatementInvalid + end end diff --git a/test/connection_fail_test.rb b/test/connection_fail_test.rb new file mode 100644 index 00000000..8061af5e --- /dev/null +++ b/test/connection_fail_test.rb @@ -0,0 +1,19 @@ +require 'test_helper' + +class ConnectionFailTest < Minitest::Test + def test_connection_fail + # We're only interested in testing a MySQL connection failure for now. + # Postgres disconnects generate a different class of errors + skip 'Only executed for MySQL' unless ActiveRecord::Base.connection.instance_values['config'][:conn_str].include? 'MySQL' + begin + conn.execute('KILL CONNECTION_ID();') + rescue => e + puts "caught exception #{e}" + end + assert_raises(ODBCAdapter::ConnectionFailedError) { User.average(:letters).round(2) } + end + + def conn + ActiveRecord::Base.connection + end +end From 96bbdfe4eb5ead8776d42604af057c6361c2d5f0 Mon Sep 17 00:00:00 2001 From: Michael Shearer Date: Fri, 5 Apr 2019 14:54:06 -0400 Subject: [PATCH 04/49] Bump version --- lib/odbc_adapter/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/odbc_adapter/version.rb b/lib/odbc_adapter/version.rb index fdebf9fb..4a65ac67 100644 --- a/lib/odbc_adapter/version.rb +++ b/lib/odbc_adapter/version.rb @@ -1,3 +1,3 @@ module ODBCAdapter - VERSION = '5.0.4'.freeze + VERSION = '5.0.5'.freeze end From 95c3965a4a302571c49727095b5a3c607267cc71 Mon Sep 17 00:00:00 2001 From: Shehbaz Date: Tue, 1 Sep 2020 13:47:22 +0100 Subject: [PATCH 05/49] Change gemspecs and gemfile --- Gemfile | 3 --- odbc_adapter.gemspec | 12 +++++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index c0cf8f42..fa75df15 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,3 @@ source 'https://rubygems.org' gemspec - -gem 'activerecord', '5.0.1' -gem 'pry', '~> 0.11.1' diff --git a/odbc_adapter.gemspec b/odbc_adapter.gemspec index 4bf0142d..7e3fbd48 100644 --- a/odbc_adapter.gemspec +++ b/odbc_adapter.gemspec @@ -19,11 +19,13 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ['lib'] + spec.add_dependency 'activerecord', '>= 5.0' spec.add_dependency 'ruby-odbc', '~> 0.9' - spec.add_development_dependency 'bundler', '~> 1.14' - spec.add_development_dependency 'minitest', '~> 5.10' - spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '0.48.1' - spec.add_development_dependency 'simplecov', '~> 0.14' + spec.add_development_dependency 'bundler', '>= 1.14' + spec.add_development_dependency 'minitest', '>= 5.10' + spec.add_development_dependency 'rake', '>= 12.0' + spec.add_development_dependency 'rubocop', '<= 0.58' + spec.add_development_dependency 'simplecov', '>= 0.14' + spec.add_development_dependency 'pry', '~> 0.13.1' end From af47f3d61e25d7d4733cd5d07bd18b70fa47b4b9 Mon Sep 17 00:00:00 2001 From: Shehbaz Date: Wed, 2 Sep 2020 13:48:53 +0100 Subject: [PATCH 06/49] Upgrade to rails5.2.1 --- .../connection_adapters/odbc_adapter.rb | 1 - lib/odbc_adapter/adapters/mysql_odbc_adapter.rb | 6 +----- lib/odbc_adapter/adapters/null_odbc_adapter.rb | 5 +---- lib/odbc_adapter/database_statements.rb | 11 ++++++++--- lib/odbc_adapter/version.rb | 2 +- odbc_adapter.gemspec | 14 +++++++------- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 06686785..12ef819e 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -1,5 +1,4 @@ require 'active_record' -require 'arel/visitors/bind_visitor' require 'odbc' require 'odbc_utf8' diff --git a/lib/odbc_adapter/adapters/mysql_odbc_adapter.rb b/lib/odbc_adapter/adapters/mysql_odbc_adapter.rb index eaa690ef..0d439462 100644 --- a/lib/odbc_adapter/adapters/mysql_odbc_adapter.rb +++ b/lib/odbc_adapter/adapters/mysql_odbc_adapter.rb @@ -5,12 +5,8 @@ module Adapters class MySQLODBCAdapter < ActiveRecord::ConnectionAdapters::ODBCAdapter PRIMARY_KEY = 'INT(11) NOT NULL AUTO_INCREMENT PRIMARY KEY'.freeze - class BindSubstitution < Arel::Visitors::MySQL - include Arel::Visitors::BindVisitor - end - def arel_visitor - BindSubstitution.new(self) + Arel::Visitors::MySQL.new(self) end # Explicitly turning off prepared statements in the MySQL adapter because diff --git a/lib/odbc_adapter/adapters/null_odbc_adapter.rb b/lib/odbc_adapter/adapters/null_odbc_adapter.rb index 1a179905..98cb4149 100644 --- a/lib/odbc_adapter/adapters/null_odbc_adapter.rb +++ b/lib/odbc_adapter/adapters/null_odbc_adapter.rb @@ -4,15 +4,12 @@ module Adapters # registry. This allows for minimal support for DBMSs for which we don't # have an explicit adapter. class NullODBCAdapter < ActiveRecord::ConnectionAdapters::ODBCAdapter - class BindSubstitution < Arel::Visitors::ToSql - include Arel::Visitors::BindVisitor - end # Using a BindVisitor so that the SQL string gets substituted before it is # sent to the DBMS (to attempt to get as much coverage as possible for # DBMSs we don't support). def arel_visitor - BindSubstitution.new(self) + Arel::Visitors::PostgreSQL.new(self) end # Explicitly turning off prepared_statements in the null adapter because diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index cac31682..1922dd1c 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -10,7 +10,7 @@ module DatabaseStatements def execute(sql, name = nil, binds = []) log(sql, name) do if prepared_statements - @connection.do(sql, *prepared_binds(binds)) + @connection.do(prepare_statement_sub(sql), *prepared_binds(binds)) else @connection.do(sql) end @@ -24,7 +24,7 @@ def exec_query(sql, name = 'SQL', binds = [], prepare: false) # rubocop:disable log(sql, name) do stmt = if prepared_statements - @connection.run(sql, *prepared_binds(binds)) + @connection.do(prepare_statement_sub(sql), *prepared_binds(binds)) else @connection.run(sql) end @@ -127,8 +127,13 @@ def nullability(col_name, is_nullable, nullable) col_name == 'id' ? false : result end + # Adapt to Rails 5.2 + def prepare_statement_sub(sql) + sql.gsub(/\$\d+/, '?') + end + def prepared_binds(binds) - prepare_binds_for_database(binds).map { |bind| _type_cast(bind) } + binds.map(&:value_for_database).map { |bind| _type_cast(bind) } end end end diff --git a/lib/odbc_adapter/version.rb b/lib/odbc_adapter/version.rb index 4a65ac67..598c96ec 100644 --- a/lib/odbc_adapter/version.rb +++ b/lib/odbc_adapter/version.rb @@ -1,3 +1,3 @@ module ODBCAdapter - VERSION = '5.0.5'.freeze + VERSION = '5.0.6'.freeze end diff --git a/odbc_adapter.gemspec b/odbc_adapter.gemspec index 7e3fbd48..d108cb70 100644 --- a/odbc_adapter.gemspec +++ b/odbc_adapter.gemspec @@ -19,13 +19,13 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ['lib'] - spec.add_dependency 'activerecord', '>= 5.0' + spec.add_dependency 'activerecord', '>= 5.2.1' spec.add_dependency 'ruby-odbc', '~> 0.9' - spec.add_development_dependency 'bundler', '>= 1.14' - spec.add_development_dependency 'minitest', '>= 5.10' - spec.add_development_dependency 'rake', '>= 12.0' - spec.add_development_dependency 'rubocop', '<= 0.58' - spec.add_development_dependency 'simplecov', '>= 0.14' - spec.add_development_dependency 'pry', '~> 0.13.1' + spec.add_development_dependency 'bundler', '~> 1.14' + spec.add_development_dependency 'minitest', '~> 5.10' + spec.add_development_dependency 'rake', '~> 12.0' + spec.add_development_dependency 'rubocop', '0.48.1' + spec.add_development_dependency 'simplecov', '~> 0.14' + spec.add_development_dependency 'pry', '~> 0.11.1' end From abdfd4593514d395c35b98c01bd5330e008f82d9 Mon Sep 17 00:00:00 2001 From: Shehbaz Date: Tue, 29 Sep 2020 08:04:33 +0100 Subject: [PATCH 07/49] Upgrade to rails 6 --- lib/active_record/connection_adapters/odbc_adapter.rb | 2 +- lib/odbc_adapter/column.rb | 4 ++-- lib/odbc_adapter/schema_statements.rb | 2 +- odbc_adapter.gemspec | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 12ef819e..17a0ef47 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -136,7 +136,7 @@ def disconnect! # Build a new column object from the given options. Effectively the same # as super except that it also passes in the native type. # rubocop:disable Metrics/ParameterLists - def new_column(name, default, sql_type_metadata, null, table_name, default_function = nil, collation = nil, native_type = nil) + def new_column(name, default, sql_type_metadata, null, default_function = nil) ::ODBCAdapter::Column.new(name, default, sql_type_metadata, null, table_name, default_function, collation, native_type) end diff --git a/lib/odbc_adapter/column.rb b/lib/odbc_adapter/column.rb index 36492a82..12f3f565 100644 --- a/lib/odbc_adapter/column.rb +++ b/lib/odbc_adapter/column.rb @@ -5,8 +5,8 @@ class Column < ActiveRecord::ConnectionAdapters::Column # Add the native_type accessor to allow the native DBMS to report back what # it uses to represent the column internally. # rubocop:disable Metrics/ParameterLists - def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, native_type = nil, default_function = nil, collation = nil) - super(name, default, sql_type_metadata, null, table_name, default_function, collation) + def initialize(name, default, sql_type_metadata = nil, null = true, native_type = nil, default_function = nil) + super(name, default, sql_type_metadata, null, default_function) @native_type = native_type end end diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index df149765..88fd2ad7 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -83,7 +83,7 @@ def columns(table_name, _name = nil) end sql_type_metadata = ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new(**args) - cols << new_column(format_case(col_name), col_default, sql_type_metadata, col_nullable, table_name, col_native_type) + cols << new_column(format_case(col_name), col_default, sql_type_metadata, col_nullable, col_native_type) end end diff --git a/odbc_adapter.gemspec b/odbc_adapter.gemspec index d108cb70..dd63ea94 100644 --- a/odbc_adapter.gemspec +++ b/odbc_adapter.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ['lib'] - spec.add_dependency 'activerecord', '>= 5.2.1' + spec.add_dependency 'activerecord', '>= 6.0.1' spec.add_dependency 'ruby-odbc', '~> 0.9' spec.add_development_dependency 'bundler', '~> 1.14' From b51a5939595ceb08267d48049cf18d263366b7d2 Mon Sep 17 00:00:00 2001 From: Shehbaz Date: Tue, 29 Sep 2020 08:27:00 +0100 Subject: [PATCH 08/49] Backward compatibility --- .rubocop.yml | 2 +- odbc_adapter.gemspec | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 9055a997..28d2dcd7 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,7 +1,7 @@ AllCops: DisplayCopNames: true DisplayStyleGuide: true - TargetRubyVersion: 2.1 + TargetRubyVersion: 2.2 Exclude: - 'vendor/**/*' diff --git a/odbc_adapter.gemspec b/odbc_adapter.gemspec index dd63ea94..0eafa155 100644 --- a/odbc_adapter.gemspec +++ b/odbc_adapter.gemspec @@ -19,10 +19,10 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ['lib'] - spec.add_dependency 'activerecord', '>= 6.0.1' + spec.add_dependency 'activerecord', '>= 5.0.1' spec.add_dependency 'ruby-odbc', '~> 0.9' - spec.add_development_dependency 'bundler', '~> 1.14' + spec.add_development_dependency 'bundler', '>= 1.14' spec.add_development_dependency 'minitest', '~> 5.10' spec.add_development_dependency 'rake', '~> 12.0' spec.add_development_dependency 'rubocop', '0.48.1' From 7a43f84c15a0f714c691768df902ccbabfe94a11 Mon Sep 17 00:00:00 2001 From: Shehbaz Date: Tue, 29 Sep 2020 12:57:18 +0100 Subject: [PATCH 09/49] Update bind params --- .../adapters/postgresql_odbc_adapter.rb | 2 +- lib/odbc_adapter/database_statements.rb | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb b/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb index 28a28f7c..79529b4d 100644 --- a/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb +++ b/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb @@ -35,7 +35,7 @@ def default_sequence_name(table_name, pk = nil) "#{table_name}_#{pk || 'id'}_seq" end - def sql_for_insert(sql, pk, _id_value, _sequence_name, binds) + def sql_for_insert(sql, pk, binds) unless pk table_ref = extract_table_ref_from_insert_sql(sql) pk = primary_key(table_ref) if table_ref diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index 1922dd1c..859fc43a 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -9,11 +9,8 @@ module DatabaseStatements # Returns the number of rows affected. def execute(sql, name = nil, binds = []) log(sql, name) do - if prepared_statements - @connection.do(prepare_statement_sub(sql), *prepared_binds(binds)) - else - @connection.do(sql) - end + sql = bind_params(binds, sql) if prepared_statements + @connection.do(sql) end end @@ -22,12 +19,8 @@ def execute(sql, name = nil, binds = []) # the executed +sql+ statement. def exec_query(sql, name = 'SQL', binds = [], prepare: false) # rubocop:disable Lint/UnusedMethodArgument log(sql, name) do - stmt = - if prepared_statements - @connection.do(prepare_statement_sub(sql), *prepared_binds(binds)) - else - @connection.run(sql) - end + sql = bind_params(binds, sql) if prepared_statements + stmt = @connection.run(sql) columns = stmt.columns values = stmt.to_a @@ -81,6 +74,14 @@ def dbms_type_cast(_columns, values) values end + def bind_params(binds, sql) + prepared_binds = *prepared_binds(binds) + prepared_binds.each.with_index(1) do |val, ind| + sql = sql.gsub("$#{ind}", "'#{val}'") + end + sql + end + # Assume received identifier is in DBMS's data dictionary case. def format_case(identifier) if database_metadata.upcase_identifiers? From 7c56bcc61bd6c75d61c23be45cbcb3c9448483eb Mon Sep 17 00:00:00 2001 From: Shehbaz Date: Thu, 8 Oct 2020 14:24:32 +0100 Subject: [PATCH 10/49] Change params for Column class initializer --- lib/active_record/connection_adapters/odbc_adapter.rb | 4 ++-- lib/odbc_adapter/column.rb | 4 ++-- lib/odbc_adapter/schema_statements.rb | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 17a0ef47..3422cd61 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -136,8 +136,8 @@ def disconnect! # Build a new column object from the given options. Effectively the same # as super except that it also passes in the native type. # rubocop:disable Metrics/ParameterLists - def new_column(name, default, sql_type_metadata, null, default_function = nil) - ::ODBCAdapter::Column.new(name, default, sql_type_metadata, null, table_name, default_function, collation, native_type) + def new_column(name, default, sql_type_metadata, null, table_name, native_type = nil) + ::ODBCAdapter::Column.new(name, default, sql_type_metadata, null, table_name, native_type) end protected diff --git a/lib/odbc_adapter/column.rb b/lib/odbc_adapter/column.rb index 12f3f565..93044fea 100644 --- a/lib/odbc_adapter/column.rb +++ b/lib/odbc_adapter/column.rb @@ -5,8 +5,8 @@ class Column < ActiveRecord::ConnectionAdapters::Column # Add the native_type accessor to allow the native DBMS to report back what # it uses to represent the column internally. # rubocop:disable Metrics/ParameterLists - def initialize(name, default, sql_type_metadata = nil, null = true, native_type = nil, default_function = nil) - super(name, default, sql_type_metadata, null, default_function) + def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, native_type = nil) + super(name, default, sql_type_metadata, null, table_name) @native_type = native_type end end diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index 88fd2ad7..df149765 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -83,7 +83,7 @@ def columns(table_name, _name = nil) end sql_type_metadata = ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new(**args) - cols << new_column(format_case(col_name), col_default, sql_type_metadata, col_nullable, col_native_type) + cols << new_column(format_case(col_name), col_default, sql_type_metadata, col_nullable, table_name, col_native_type) end end From bf7a016c37be8ce05861952e7698dee7ae5b65c6 Mon Sep 17 00:00:00 2001 From: Shehbaz Date: Fri, 16 Oct 2020 13:40:38 +0100 Subject: [PATCH 11/49] Support json type field --- lib/odbc_adapter/database_statements.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index 859fc43a..d4eed5bf 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -28,6 +28,12 @@ def exec_query(sql, name = 'SQL', binds = [], prepare: false) # rubocop:disable values = dbms_type_cast(columns.values, values) column_names = columns.keys.map { |key| format_case(key) } + upcase_coulmn_names = columns.keys + values.each.with_index(0) do |_row_value, row_index| + columns.each.with_index(0) do |_col_value, col_index| + values[row_index][col_index] = json_parsing(values[row_index][col_index], columns[upcase_coulmn_names[col_index]]) + end + end ActiveRecord::Result.new(column_names, values) end end @@ -74,6 +80,17 @@ def dbms_type_cast(_columns, values) values end + # A custom fuction to check the string and json column. + # If the column value is string, JSON.parse will raise expection, which will just + # return the original value, Otherwise this will return parsed value. + def json_parsing(values, column) + return values unless column.type == SQL_CHARACTER_VARYING_DATATYPE && values.include?('{') && values.include?('}') + + JSON.parse values + rescue + values + end + def bind_params(binds, sql) prepared_binds = *prepared_binds(binds) prepared_binds.each.with_index(1) do |val, ind| From 2696c7a7ae0a50002d086edc1d70051846ff9fc4 Mon Sep 17 00:00:00 2001 From: Shehbaz Date: Thu, 29 Oct 2020 06:30:36 +0000 Subject: [PATCH 12/49] Support json and date datatype --- .../connection_adapters/odbc_adapter.rb | 4 ++++ lib/odbc_adapter/adapters/null_odbc_adapter.rb | 4 +++- .../adapters/postgresql_odbc_adapter.rb | 4 ++++ lib/odbc_adapter/database_statements.rb | 17 ----------------- lib/odbc_adapter/schema_statements.rb | 2 ++ 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 3422cd61..b7242022 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -74,6 +74,9 @@ class ODBCAdapter < AbstractAdapter ADAPTER_NAME = 'ODBC'.freeze BOOLEAN_TYPE = 'BOOLEAN'.freeze + VARIANT_TYPE = 'VARIANT'.freeze + DATE_TYPE = 'DATE'.freeze + JSON_TYPE = 'json'.freeze ERR_DUPLICATE_KEY_VALUE = 23_505 ERR_QUERY_TIMED_OUT = 57_014 @@ -146,6 +149,7 @@ def new_column(name, default, sql_type_metadata, null, table_name, native_type = # Here, ODBC and ODBC_UTF8 constants are interchangeable def initialize_type_map(map) map.register_type 'boolean', Type::Boolean.new + map.register_type 'json', Type::Json.new map.register_type ODBC::SQL_CHAR, Type::String.new map.register_type ODBC::SQL_LONGVARCHAR, Type::Text.new map.register_type ODBC::SQL_TINYINT, Type::Integer.new(limit: 4) diff --git a/lib/odbc_adapter/adapters/null_odbc_adapter.rb b/lib/odbc_adapter/adapters/null_odbc_adapter.rb index 98cb4149..70a51e01 100644 --- a/lib/odbc_adapter/adapters/null_odbc_adapter.rb +++ b/lib/odbc_adapter/adapters/null_odbc_adapter.rb @@ -4,7 +4,9 @@ module Adapters # registry. This allows for minimal support for DBMSs for which we don't # have an explicit adapter. class NullODBCAdapter < ActiveRecord::ConnectionAdapters::ODBCAdapter - + VARIANT_TYPE = 'VARIANT'.freeze + DATE_TYPE = 'DATE'.freeze + JSON_TYPE = 'json'.freeze # Using a BindVisitor so that the SQL string gets substituted before it is # sent to the DBMS (to attempt to get as much coverage as possible for # DBMSs we don't support). diff --git a/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb b/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb index 79529b4d..81cd4f27 100644 --- a/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb +++ b/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb @@ -5,12 +5,16 @@ module Adapters class PostgreSQLODBCAdapter < ActiveRecord::ConnectionAdapters::ODBCAdapter BOOLEAN_TYPE = 'bool'.freeze PRIMARY_KEY = 'SERIAL PRIMARY KEY'.freeze + VARIANT_TYPE = 'VARIANT'.freeze + DATE_TYPE = 'DATE'.freeze + JSON_TYPE = 'json'.freeze alias create insert # Override to handle booleans appropriately def native_database_types @native_database_types ||= super.merge(boolean: { name: 'bool' }) + @native_database_types ||= super.merge(json: { name: 'json' }) end def arel_visitor diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index d4eed5bf..859fc43a 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -28,12 +28,6 @@ def exec_query(sql, name = 'SQL', binds = [], prepare: false) # rubocop:disable values = dbms_type_cast(columns.values, values) column_names = columns.keys.map { |key| format_case(key) } - upcase_coulmn_names = columns.keys - values.each.with_index(0) do |_row_value, row_index| - columns.each.with_index(0) do |_col_value, col_index| - values[row_index][col_index] = json_parsing(values[row_index][col_index], columns[upcase_coulmn_names[col_index]]) - end - end ActiveRecord::Result.new(column_names, values) end end @@ -80,17 +74,6 @@ def dbms_type_cast(_columns, values) values end - # A custom fuction to check the string and json column. - # If the column value is string, JSON.parse will raise expection, which will just - # return the original value, Otherwise this will return parsed value. - def json_parsing(values, column) - return values unless column.type == SQL_CHARACTER_VARYING_DATATYPE && values.include?('{') && values.include?('}') - - JSON.parse values - rescue - values - end - def bind_params(binds, sql) prepared_binds = *prepared_binds(binds) prepared_binds.each.with_index(1) do |val, ind| diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index df149765..3df30c9c 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -76,6 +76,8 @@ def columns(table_name, _name = nil) args = { sql_type: col_sql_type, type: col_sql_type, limit: col_limit } args[:sql_type] = 'boolean' if col_native_type == self.class::BOOLEAN_TYPE + args[:sql_type] = 'json' if col_native_type == self.class::VARIANT_TYPE || col_native_type == self.class::JSON_TYPE + args[:sql_type] = 'date' if col_native_type == self.class::DATE_TYPE if [ODBC::SQL_DECIMAL, ODBC::SQL_NUMERIC].include?(col_sql_type) args[:scale] = col_scale || 0 From fb7aa4a781a34df3122c9d7af939a7728157f699 Mon Sep 17 00:00:00 2001 From: Shehbaz Date: Thu, 29 Oct 2020 08:25:43 +0000 Subject: [PATCH 13/49] Typo fix --- lib/active_record/connection_adapters/odbc_adapter.rb | 2 +- lib/odbc_adapter/adapters/null_odbc_adapter.rb | 2 +- lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index b7242022..e4887723 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -76,7 +76,7 @@ class ODBCAdapter < AbstractAdapter BOOLEAN_TYPE = 'BOOLEAN'.freeze VARIANT_TYPE = 'VARIANT'.freeze DATE_TYPE = 'DATE'.freeze - JSON_TYPE = 'json'.freeze + JSON_TYPE = 'JSON'.freeze ERR_DUPLICATE_KEY_VALUE = 23_505 ERR_QUERY_TIMED_OUT = 57_014 diff --git a/lib/odbc_adapter/adapters/null_odbc_adapter.rb b/lib/odbc_adapter/adapters/null_odbc_adapter.rb index 70a51e01..c78e991f 100644 --- a/lib/odbc_adapter/adapters/null_odbc_adapter.rb +++ b/lib/odbc_adapter/adapters/null_odbc_adapter.rb @@ -6,7 +6,7 @@ module Adapters class NullODBCAdapter < ActiveRecord::ConnectionAdapters::ODBCAdapter VARIANT_TYPE = 'VARIANT'.freeze DATE_TYPE = 'DATE'.freeze - JSON_TYPE = 'json'.freeze + JSON_TYPE = 'JSON'.freeze # Using a BindVisitor so that the SQL string gets substituted before it is # sent to the DBMS (to attempt to get as much coverage as possible for # DBMSs we don't support). diff --git a/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb b/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb index 81cd4f27..05fa17e0 100644 --- a/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb +++ b/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb @@ -7,14 +7,13 @@ class PostgreSQLODBCAdapter < ActiveRecord::ConnectionAdapters::ODBCAdapter PRIMARY_KEY = 'SERIAL PRIMARY KEY'.freeze VARIANT_TYPE = 'VARIANT'.freeze DATE_TYPE = 'DATE'.freeze - JSON_TYPE = 'json'.freeze + JSON_TYPE = 'JSON'.freeze alias create insert # Override to handle booleans appropriately def native_database_types @native_database_types ||= super.merge(boolean: { name: 'bool' }) - @native_database_types ||= super.merge(json: { name: 'json' }) end def arel_visitor From 6722dc70241eaca857524d2231e3e632acdb1d77 Mon Sep 17 00:00:00 2001 From: BFredenburg Date: Wed, 25 Aug 2021 11:20:32 -0400 Subject: [PATCH 14/49] Multi-database support Changes to support multiple databases in the Snowflake warehouse. The INFORMATION_SCHEMA tables contain information for all databases in the warehouse. --- lib/odbc_adapter/schema_statements.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index 3df30c9c..4d17369f 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -14,7 +14,9 @@ def tables(_name = nil) result = stmt.fetch_all || [] stmt.drop + db_regex = /^#{current_database}$/i result.each_with_object([]) do |row, table_names| + next unless row[0] =~ db_regex schema_name, table_name, table_type = row[1..3] next if respond_to?(:table_filtered?) && table_filtered?(schema_name, table_type) table_names << format_case(table_name) @@ -36,7 +38,9 @@ def indexes(table_name, _name = nil) index_name = nil unique = nil + db_regex = /^#{current_database}$/i result.each_with_object([]).with_index do |(row, indices), row_idx| + next unless row[0] =~ db_regex # Skip table statistics next if row[6].zero? # SQLStatistics: TYPE @@ -63,7 +67,9 @@ def columns(table_name, _name = nil) result = stmt.fetch_all || [] stmt.drop + db_regex = /^#{current_database}$/i result.each_with_object([]) do |col, cols| + next unless col[0] =~ db_regex col_name = col[3] # SQLColumns: COLUMN_NAME col_default = col[12] # SQLColumns: COLUMN_DEF col_sql_type = col[4] # SQLColumns: DATA_TYPE @@ -94,7 +100,9 @@ def primary_key(table_name) stmt = @connection.primary_keys(native_case(table_name.to_s)) result = stmt.fetch_all || [] stmt.drop unless stmt.nil? - result[0] && result[0][3] + + db_regex = /^#{current_database}$/i + result.reduce(nil) { |pkey, key| key[0] =~ db_regex ? key[3] : pkey } end def foreign_keys(table_name) @@ -102,7 +110,9 @@ def foreign_keys(table_name) result = stmt.fetch_all || [] stmt.drop unless stmt.nil? + db_regex = /^#{current_database}$/i result.map do |key| + next unless key[0] =~ db_regex fk_from_table = key[2] # PKTABLE_NAME fk_to_table = key[6] # FKTABLE_NAME From bf065a804c0cc9f5f50a0bd5889764b92d80ac86 Mon Sep 17 00:00:00 2001 From: Chris Flack Date: Thu, 2 Dec 2021 11:58:45 -0500 Subject: [PATCH 15/49] [APP-2051] Update Column#initialize to pass on **kwargs (rails 6.1 compatibility) Source ====== https://springbuk-glass.atlassian.net/browse/APP-2051 Problem ======= The linked story is the first implementation of snowflake in the main Springbuk app. Springbuk is on 6.1, and ActiveRecord 6.1 is passing ** to allow kwargs through. In our use case, there's nothing actually being passed here, but we should just keep passing them along in the call chain. Solution ======== Adds `**kwargs` to initialize signature and pass along to super. Testing/QA Notes ================ Tested locally in springbuk (6.1.4) and edison (6.0.3). Post Merge Notes ================ n/a --- lib/odbc_adapter/column.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/odbc_adapter/column.rb b/lib/odbc_adapter/column.rb index 93044fea..ee5bc0c3 100644 --- a/lib/odbc_adapter/column.rb +++ b/lib/odbc_adapter/column.rb @@ -5,8 +5,8 @@ class Column < ActiveRecord::ConnectionAdapters::Column # Add the native_type accessor to allow the native DBMS to report back what # it uses to represent the column internally. # rubocop:disable Metrics/ParameterLists - def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, native_type = nil) - super(name, default, sql_type_metadata, null, table_name) + def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, native_type = nil, **kwargs) + super(name, default, sql_type_metadata, null, table_name, kwargs) @native_type = native_type end end From 7df4b1dac79f7702abe4801ea41e863c33ea6a6e Mon Sep 17 00:00:00 2001 From: BFredenburg Date: Tue, 1 Feb 2022 09:38:33 -0500 Subject: [PATCH 16/49] Update schema_statements.rb --- lib/odbc_adapter/schema_statements.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index 4d17369f..817f0ddf 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -15,8 +15,9 @@ def tables(_name = nil) stmt.drop db_regex = /^#{current_database}$/i + schema_regex = /^#{current_schema}$/i result.each_with_object([]) do |row, table_names| - next unless row[0] =~ db_regex + next unless row[0] =~ db_regex && row[1] =~ schema_regex schema_name, table_name, table_type = row[1..3] next if respond_to?(:table_filtered?) && table_filtered?(schema_name, table_type) table_names << format_case(table_name) @@ -39,8 +40,9 @@ def indexes(table_name, _name = nil) unique = nil db_regex = /^#{current_database}$/i + schema_regex = /^#{current_schema}$/i result.each_with_object([]).with_index do |(row, indices), row_idx| - next unless row[0] =~ db_regex + next unless row[0] =~ db_regex && row[1] =~ schema_regex # Skip table statistics next if row[6].zero? # SQLStatistics: TYPE @@ -68,8 +70,9 @@ def columns(table_name, _name = nil) stmt.drop db_regex = /^#{current_database}$/i + schema_regex = /^#{current_schema}$/i result.each_with_object([]) do |col, cols| - next unless col[0] =~ db_regex + next unless col[0] =~ db_regex && row[1] =~ schema_regex col_name = col[3] # SQLColumns: COLUMN_NAME col_default = col[12] # SQLColumns: COLUMN_DEF col_sql_type = col[4] # SQLColumns: DATA_TYPE @@ -102,7 +105,8 @@ def primary_key(table_name) stmt.drop unless stmt.nil? db_regex = /^#{current_database}$/i - result.reduce(nil) { |pkey, key| key[0] =~ db_regex ? key[3] : pkey } + schema_regex = /^#{current_schema}$/i + result.reduce(nil) { |pkey, key| (key[0] =~ db_regex && key[1] =~ schema_regex) ? key[3] : pkey } end def foreign_keys(table_name) @@ -111,8 +115,9 @@ def foreign_keys(table_name) stmt.drop unless stmt.nil? db_regex = /^#{current_database}$/i + schema_regex = /^#{current_schema}$/i result.map do |key| - next unless key[0] =~ db_regex + next unless key[0] =~ db_regex && key[1] =~ schema_regex fk_from_table = key[2] # PKTABLE_NAME fk_to_table = key[6] # FKTABLE_NAME @@ -138,5 +143,9 @@ def index_name(table_name, options) def current_database database_metadata.database_name.strip end + + def current_schema + @config[:driver].attrs['schema'] + end end end From 86c96b72bba65793b31f35f5d8d47999faa0e9b2 Mon Sep 17 00:00:00 2001 From: BFredenburg Date: Thu, 10 Feb 2022 16:45:58 -0500 Subject: [PATCH 17/49] Corrections for handling names --- lib/odbc_adapter/schema_statements.rb | 28 +++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index 817f0ddf..a3742266 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -14,8 +14,8 @@ def tables(_name = nil) result = stmt.fetch_all || [] stmt.drop - db_regex = /^#{current_database}$/i - schema_regex = /^#{current_schema}$/i + db_regex = name_regex(current_database) + schema_regex = name_regex(current_schema) result.each_with_object([]) do |row, table_names| next unless row[0] =~ db_regex && row[1] =~ schema_regex schema_name, table_name, table_type = row[1..3] @@ -39,8 +39,8 @@ def indexes(table_name, _name = nil) index_name = nil unique = nil - db_regex = /^#{current_database}$/i - schema_regex = /^#{current_schema}$/i + db_regex = name_regex(current_database) + schema_regex = name_regex(current_schema) result.each_with_object([]).with_index do |(row, indices), row_idx| next unless row[0] =~ db_regex && row[1] =~ schema_regex # Skip table statistics @@ -69,8 +69,8 @@ def columns(table_name, _name = nil) result = stmt.fetch_all || [] stmt.drop - db_regex = /^#{current_database}$/i - schema_regex = /^#{current_schema}$/i + db_regex = name_regex(current_database) + schema_regex = name_regex(current_schema) result.each_with_object([]) do |col, cols| next unless col[0] =~ db_regex && row[1] =~ schema_regex col_name = col[3] # SQLColumns: COLUMN_NAME @@ -104,8 +104,8 @@ def primary_key(table_name) result = stmt.fetch_all || [] stmt.drop unless stmt.nil? - db_regex = /^#{current_database}$/i - schema_regex = /^#{current_schema}$/i + db_regex = name_regex(current_database) + schema_regex = name_regex(current_schema) result.reduce(nil) { |pkey, key| (key[0] =~ db_regex && key[1] =~ schema_regex) ? key[3] : pkey } end @@ -114,8 +114,8 @@ def foreign_keys(table_name) result = stmt.fetch_all || [] stmt.drop unless stmt.nil? - db_regex = /^#{current_database}$/i - schema_regex = /^#{current_schema}$/i + db_regex = name_regex(current_database) + schema_regex = name_regex(current_schema) result.map do |key| next unless key[0] =~ db_regex && key[1] =~ schema_regex fk_from_table = key[2] # PKTABLE_NAME @@ -147,5 +147,13 @@ def current_database def current_schema @config[:driver].attrs['schema'] end + + def name_regex(name) + if name =~ /^".*"$/ + /^#{name.delete_prefix('"').delete_suffix('"')}$/ + else + /^#{name}$/i + end + end end end From 9b8b8de60b74c610f7732f38666ba592bb8de45a Mon Sep 17 00:00:00 2001 From: BFredenburg Date: Fri, 11 Feb 2022 14:40:26 -0500 Subject: [PATCH 18/49] Oops --- lib/odbc_adapter/schema_statements.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index a3742266..24c2107a 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -72,7 +72,7 @@ def columns(table_name, _name = nil) db_regex = name_regex(current_database) schema_regex = name_regex(current_schema) result.each_with_object([]) do |col, cols| - next unless col[0] =~ db_regex && row[1] =~ schema_regex + next unless col[0] =~ db_regex && col[1] =~ schema_regex col_name = col[3] # SQLColumns: COLUMN_NAME col_default = col[12] # SQLColumns: COLUMN_DEF col_sql_type = col[4] # SQLColumns: DATA_TYPE From ef14e72069478c9e0bb354bdbf5b58ce8bcb3140 Mon Sep 17 00:00:00 2001 From: BFredenburg Date: Wed, 16 Feb 2022 09:36:48 -0500 Subject: [PATCH 19/49] Changes because arg list to ActiveRecord::ConnectionAdapters::Column.new changed --- lib/active_record/connection_adapters/odbc_adapter.rb | 4 ++-- lib/odbc_adapter/column.rb | 4 ++-- lib/odbc_adapter/schema_statements.rb | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index e4887723..7564806c 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -139,8 +139,8 @@ def disconnect! # Build a new column object from the given options. Effectively the same # as super except that it also passes in the native type. # rubocop:disable Metrics/ParameterLists - def new_column(name, default, sql_type_metadata, null, table_name, native_type = nil) - ::ODBCAdapter::Column.new(name, default, sql_type_metadata, null, table_name, native_type) + def new_column(name, default, sql_type_metadata, null, native_type = nil) + ::ODBCAdapter::Column.new(name, default, sql_type_metadata, null, native_type) end protected diff --git a/lib/odbc_adapter/column.rb b/lib/odbc_adapter/column.rb index ee5bc0c3..4f2901ac 100644 --- a/lib/odbc_adapter/column.rb +++ b/lib/odbc_adapter/column.rb @@ -5,8 +5,8 @@ class Column < ActiveRecord::ConnectionAdapters::Column # Add the native_type accessor to allow the native DBMS to report back what # it uses to represent the column internally. # rubocop:disable Metrics/ParameterLists - def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, native_type = nil, **kwargs) - super(name, default, sql_type_metadata, null, table_name, kwargs) + def initialize(name, default, sql_type_metadata = nil, null = true, native_type = nil, **kwargs) + super(name, default, sql_type_metadata, null, **kwargs) @native_type = native_type end end diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index 24c2107a..741b36bd 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -94,7 +94,7 @@ def columns(table_name, _name = nil) end sql_type_metadata = ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new(**args) - cols << new_column(format_case(col_name), col_default, sql_type_metadata, col_nullable, table_name, col_native_type) + cols << new_column(format_case(col_name), col_default, sql_type_metadata, col_nullable, col_native_type) end end From 3d5481c4a5e3cb810cdee614918798b6639f02b6 Mon Sep 17 00:00:00 2001 From: BFredenburg Date: Wed, 16 Feb 2022 09:37:36 -0500 Subject: [PATCH 20/49] Change to return primary key name as lowercase --- lib/odbc_adapter/schema_statements.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index 741b36bd..a0599d86 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -106,7 +106,7 @@ def primary_key(table_name) db_regex = name_regex(current_database) schema_regex = name_regex(current_schema) - result.reduce(nil) { |pkey, key| (key[0] =~ db_regex && key[1] =~ schema_regex) ? key[3] : pkey } + result.reduce(nil) { |pkey, key| (key[0] =~ db_regex && key[1] =~ schema_regex) ? format_case(key[3]) : pkey } end def foreign_keys(table_name) From 715f9e7223500046e68f5a1154264bf6afdc1ae2 Mon Sep 17 00:00:00 2001 From: BFredenburg Date: Wed, 16 Feb 2022 09:38:22 -0500 Subject: [PATCH 21/49] Changes for columns function to return better types --- lib/odbc_adapter/schema_statements.rb | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index a0599d86..6f28a928 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -83,12 +83,19 @@ def columns(table_name, _name = nil) # SQLColumns: IS_NULLABLE, SQLColumns: NULLABLE col_nullable = nullability(col_name, col[17], col[10]) - args = { sql_type: col_sql_type, type: col_sql_type, limit: col_limit } - args[:sql_type] = 'boolean' if col_native_type == self.class::BOOLEAN_TYPE - args[:sql_type] = 'json' if col_native_type == self.class::VARIANT_TYPE || col_native_type == self.class::JSON_TYPE - args[:sql_type] = 'date' if col_native_type == self.class::DATE_TYPE + # This section has been customized for Snowflake and will not work in general. + args = { sql_type: col_native_type, type: col_native_type, limit: col_limit } + args[:type] = 'boolean' if col_native_type == "BOOLEAN" # self.class::BOOLEAN_TYPE + args[:type] = 'json' if col_native_type == "VARIANT" || col_native_type == "JSON" + args[:type] = 'date' if col_native_type == "DATE" + args[:type] = 'string' if col_native_type == "VARCHAR" + args[:type] = 'datetime' if col_native_type == "TIMESTAMP" + args[:type] = 'time' if col_native_type == "TIME" + args[:type] = 'binary' if col_native_type == "BINARY" + args[:type] = 'float' if col_native_type == "DOUBLE" if [ODBC::SQL_DECIMAL, ODBC::SQL_NUMERIC].include?(col_sql_type) + args[:type] = col_scale == 0 ? 'integer' : 'decimal' args[:scale] = col_scale || 0 args[:precision] = col_limit end From b47df4336ea1b5ee4062cb1ffb6b76da84fdedb4 Mon Sep 17 00:00:00 2001 From: Ross Crenshaw Date: Thu, 3 Mar 2022 12:32:36 -0500 Subject: [PATCH 22/49] [APP-2295] Update signature of translate_exception for ruby 3 --- lib/active_record/connection_adapters/odbc_adapter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 7564806c..1397550f 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -182,7 +182,7 @@ def initialize_type_map(map) # Translate an exception from the native DBMS to something usable by # ActiveRecord. - def translate_exception(exception, message) + def translate_exception(exception, message:, sql:, binds:) error_number = exception.message[/^\d+/].to_i if error_number == ERR_DUPLICATE_KEY_VALUE From e31722913ed7172ef22fde06b74c34d97ba514b1 Mon Sep 17 00:00:00 2001 From: BFredenburg Date: Thu, 3 Mar 2022 22:01:48 -0500 Subject: [PATCH 23/49] Correct column default values --- lib/odbc_adapter/schema_statements.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index 6f28a928..3f1f926a 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -101,6 +101,13 @@ def columns(table_name, _name = nil) end sql_type_metadata = ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new(**args) + # The @connection.columns function returns empty strings for column defaults. + # Even when the column has a default value. This is a call to the ODBC layer + # with only enough Ruby to make the call happen. Replacing the empty string + # with nil permits Rails to set the current datetime for created_at and + # updated_at on model creates and updates. + col_default = nil if col_default == "" + cols << new_column(format_case(col_name), col_default, sql_type_metadata, col_nullable, col_native_type) end end From 87f077c0421049aa648532b2d5005d9ee2ae0531 Mon Sep 17 00:00:00 2001 From: dracco1993 Date: Fri, 11 Mar 2022 13:32:50 -0500 Subject: [PATCH 24/49] Add better return types --- lib/odbc_adapter/database_statements.rb | 67 ++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index 859fc43a..8a2e3b9b 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -1,3 +1,5 @@ +require "json" + module ODBCAdapter module DatabaseStatements # ODBC constants missing from Christian Werner's Ruby ODBC driver @@ -24,6 +26,7 @@ def exec_query(sql, name = 'SQL', binds = [], prepare: false) # rubocop:disable columns = stmt.columns values = stmt.to_a + # binding.pry stmt.drop values = dbms_type_cast(columns.values, values) @@ -70,8 +73,68 @@ def default_sequence_name(table, _column) # A custom hook to allow end users to overwrite the type casting before it # is returned to ActiveRecord. Useful before a full adapter has made its way # back into this repository. - def dbms_type_cast(_columns, values) - values + def dbms_type_cast(columns, rows) + # Cast the values to the correct type + columns.map.with_index do |column, col_index| + column_type = @connection.types(column.type).first[0] + + # binding.pry + # @connection.types(3).columns.keys + + # pp column + # pp "Column type: #{column.type} | #{@connection.types(column.type).first}" + + rows.each_with_index do |row, row_index| + value = row[col_index] + + binding.pry + + new_value = case + when ["CHAR", "VARCHAR", "LONGVARCHAR"].include?(column_type) + # Do nothing, because the data defaults to strings + # This also covers null values, as they are VARCHARs of length 0 + value = value.force_encoding("UTF-8") if value.is_a?(String) + value + when ["NUMERIC", "DECIMAL", "FLOAT", "REAL", "DOUBLE"].include?(column_type) + value.to_f + when ["INTEGER"].include?(column_type) + value.to_i + when ["BOOLEAN"].include?(column_type) + value == 1 + when ["DATE"].include?(column_type) + value.to_date + when ["TIME"].include?(column_type) + value.to_time + when ["TIMESTAMP", "TIMESTAMP_LTZ", "TIMESTAMP_NTZ", "TIMESTAMP_TZ"].include?(column_type) + value.to_datetime + when ["ARRAY", "OBJECT", "VARIANT"].include?(column_type) + # TODO: "ARRAY", "OBJECT", "VARIANT" all return as VARCHAR + # so we'd need to parse them to make them the correct type + + # As of now, we are just going to return the value as a string + # and let the consumer handle it. In the future, we could handle + # if here, but there's not a good way to tell what the type is + # without trying to parse the value as JSON as see if it works + # JSON.parse(value) + when ["BINARY", "VARBINARY"].include?(column_type) + # These don't actually ever seem to return, even though they are + # defined in the ODBC driver, but I left them in here just in case + # so that future us can see what they should be + else + raise "Unknown column type: #{column_type}" + end + + pp "Column type: #{column_type}" + pp "Value: #{value}(#{value.class})" + pp "New Value: #{new_value}(#{new_value.class})" + # binding.pry + + rows[row_index][col_index] = new_value + # .cast(row[cidx]) + end + + # @connection.types.to_a[3] + end end def bind_params(binds, sql) From 041ae8231384a77e727ba714bafbc7daf4cdd76d Mon Sep 17 00:00:00 2001 From: dracco1993 Date: Fri, 11 Mar 2022 13:33:53 -0500 Subject: [PATCH 25/49] Clean up dev code --- lib/odbc_adapter/database_statements.rb | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index 8a2e3b9b..bbff48f7 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -1,5 +1,3 @@ -require "json" - module ODBCAdapter module DatabaseStatements # ODBC constants missing from Christian Werner's Ruby ODBC driver @@ -26,7 +24,6 @@ def exec_query(sql, name = 'SQL', binds = [], prepare: false) # rubocop:disable columns = stmt.columns values = stmt.to_a - # binding.pry stmt.drop values = dbms_type_cast(columns.values, values) @@ -78,17 +75,9 @@ def dbms_type_cast(columns, rows) columns.map.with_index do |column, col_index| column_type = @connection.types(column.type).first[0] - # binding.pry - # @connection.types(3).columns.keys - - # pp column - # pp "Column type: #{column.type} | #{@connection.types(column.type).first}" - rows.each_with_index do |row, row_index| value = row[col_index] - binding.pry - new_value = case when ["CHAR", "VARCHAR", "LONGVARCHAR"].include?(column_type) # Do nothing, because the data defaults to strings @@ -124,16 +113,8 @@ def dbms_type_cast(columns, rows) raise "Unknown column type: #{column_type}" end - pp "Column type: #{column_type}" - pp "Value: #{value}(#{value.class})" - pp "New Value: #{new_value}(#{new_value.class})" - # binding.pry - rows[row_index][col_index] = new_value - # .cast(row[cidx]) end - - # @connection.types.to_a[3] end end From 9b6fdb2b054ac3e4c5ab2fc0a7085cf80e147a08 Mon Sep 17 00:00:00 2001 From: dracco1993 Date: Mon, 14 Mar 2022 19:03:20 -0400 Subject: [PATCH 26/49] Add better handling of other types of null --- lib/odbc_adapter/database_statements.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index bbff48f7..641f6dc1 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -79,6 +79,8 @@ def dbms_type_cast(columns, rows) value = row[col_index] new_value = case + when value.nil? + nil when ["CHAR", "VARCHAR", "LONGVARCHAR"].include?(column_type) # Do nothing, because the data defaults to strings # This also covers null values, as they are VARCHARs of length 0 From dfeefcd14517e0af74e0818017a56ad67b4967a7 Mon Sep 17 00:00:00 2001 From: Asher Cerka Date: Thu, 17 Mar 2022 15:18:41 -0400 Subject: [PATCH 27/49] Claims page was broken, discovered the bug is in the ODBC Adapter. dbms_type_cast needs to return rows at the end. Also added code to raise exceptions, since currently these circumstances shouldn't be encountered and if that changes spontaneously (such as through an snowflake odbc driver update) we want to know rather than silently have nil values. --- lib/odbc_adapter/database_statements.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index 641f6dc1..08cafeb9 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -107,10 +107,12 @@ def dbms_type_cast(columns, rows) # if here, but there's not a good way to tell what the type is # without trying to parse the value as JSON as see if it works # JSON.parse(value) + raise "Unhandled column type: #{column_type}" when ["BINARY", "VARBINARY"].include?(column_type) # These don't actually ever seem to return, even though they are # defined in the ODBC driver, but I left them in here just in case # so that future us can see what they should be + raise "Unhandled column type: #{column_type}" else raise "Unknown column type: #{column_type}" end @@ -118,6 +120,7 @@ def dbms_type_cast(columns, rows) rows[row_index][col_index] = new_value end end + rows end def bind_params(binds, sql) From 7713fcb1e3c5699456a23b8fd78778e4f44f118d Mon Sep 17 00:00:00 2001 From: BFredenburg Date: Thu, 17 Mar 2022 23:15:16 -0400 Subject: [PATCH 28/49] Changes to eliminate garbage collection warning Plus some code simplification. --- lib/odbc_adapter/database_statements.rb | 38 ++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index 08cafeb9..9705b962 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -72,33 +72,33 @@ def default_sequence_name(table, _column) # back into this repository. def dbms_type_cast(columns, rows) # Cast the values to the correct type - columns.map.with_index do |column, col_index| - column_type = @connection.types(column.type).first[0] - - rows.each_with_index do |row, row_index| + columns.each_with_index do |column, col_index| + #puts " #{column.name} type #{column.type} length #{column.length} nullable #{column.nullable} scale #{column.scale} precision #{column.precision} searchable #{column.searchable} unsigned #{column.unsigned}" + rows.each do |row| value = row[col_index] new_value = case when value.nil? nil - when ["CHAR", "VARCHAR", "LONGVARCHAR"].include?(column_type) + when [ODBC::SQL_CHAR, ODBC::SQL_VARCHAR, ODBC::SQL_LONGVARCHAR].include?(column.type) # Do nothing, because the data defaults to strings # This also covers null values, as they are VARCHARs of length 0 - value = value.force_encoding("UTF-8") if value.is_a?(String) - value - when ["NUMERIC", "DECIMAL", "FLOAT", "REAL", "DOUBLE"].include?(column_type) + value.is_a?(String) ? value.force_encoding("UTF-8") : value + when [ODBC::SQL_DECIMAL, ODBC::SQL_NUMERIC].include?(column.type) + column.scale == 0 ? value.to_i : value.to_f + when [ODBC::SQL_REAL, ODBC::SQL_FLOAT, ODBC::SQL_DOUBLE].include?(column.type) value.to_f - when ["INTEGER"].include?(column_type) + when [ODBC::SQL_INTEGER, ODBC::SQL_SMALLINT, ODBC::SQL_TINYINT, ODBC::SQL_BIGINT].include?(column.type) value.to_i - when ["BOOLEAN"].include?(column_type) + when [ODBC::SQL_BIT].include?(column.type) value == 1 - when ["DATE"].include?(column_type) + when [ODBC::SQL_DATE].include?(column.type) value.to_date - when ["TIME"].include?(column_type) + when [ODBC::SQL_TIME].include?(column.type) value.to_time - when ["TIMESTAMP", "TIMESTAMP_LTZ", "TIMESTAMP_NTZ", "TIMESTAMP_TZ"].include?(column_type) + when [ODBC::SQL_DATETIME, ODBC::SQL_TIMESTAMP].include?(column.type) value.to_datetime - when ["ARRAY", "OBJECT", "VARIANT"].include?(column_type) + # when ["ARRAY"?, "OBJECT"?, "VARIANT"?].include?(column.type) # TODO: "ARRAY", "OBJECT", "VARIANT" all return as VARCHAR # so we'd need to parse them to make them the correct type @@ -107,17 +107,17 @@ def dbms_type_cast(columns, rows) # if here, but there's not a good way to tell what the type is # without trying to parse the value as JSON as see if it works # JSON.parse(value) - raise "Unhandled column type: #{column_type}" - when ["BINARY", "VARBINARY"].include?(column_type) + when [ODBC::SQL_BINARY].include?(column.type) # These don't actually ever seem to return, even though they are # defined in the ODBC driver, but I left them in here just in case # so that future us can see what they should be - raise "Unhandled column type: #{column_type}" + value else - raise "Unknown column type: #{column_type}" + # the use of @connection.types() results in a "was not dropped before garbage collection" warning. + raise "Unknown column type: #{column.type} #{@connection.types(column.type).first[0]}" end - rows[row_index][col_index] = new_value + row[col_index] = new_value end end rows From a008f83e4e15595ea84e9909efb6c15e85c804a3 Mon Sep 17 00:00:00 2001 From: Asher Cerka Date: Mon, 11 Apr 2022 16:38:21 -0400 Subject: [PATCH 29/49] MOJ-137 Can't quote error is caused by active_record not knowing what types columns are Updated infrastructure needed for models to know what types their attributes are so it will properly handle type casting automatically. This also removes the necessity of specifying json attributes on models. The only reason that dbms_type_cast is still useful is for when we're executing raw queries instead of using the object model. --- .../connection_adapters/odbc_adapter.rb | 43 +++++-------------- lib/odbc_adapter/quoting.rb | 4 ++ lib/odbc_adapter/schema_statements.rb | 18 ++++---- 3 files changed, 24 insertions(+), 41 deletions(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 1397550f..4db1f816 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -145,39 +145,18 @@ def new_column(name, default, sql_type_metadata, null, native_type = nil) protected - # Build the type map for ActiveRecord - # Here, ODBC and ODBC_UTF8 constants are interchangeable + #Snowflake ODBC Adapter specific def initialize_type_map(map) - map.register_type 'boolean', Type::Boolean.new - map.register_type 'json', Type::Json.new - map.register_type ODBC::SQL_CHAR, Type::String.new - map.register_type ODBC::SQL_LONGVARCHAR, Type::Text.new - map.register_type ODBC::SQL_TINYINT, Type::Integer.new(limit: 4) - map.register_type ODBC::SQL_SMALLINT, Type::Integer.new(limit: 8) - map.register_type ODBC::SQL_INTEGER, Type::Integer.new(limit: 16) - map.register_type ODBC::SQL_BIGINT, Type::BigInteger.new(limit: 32) - map.register_type ODBC::SQL_REAL, Type::Float.new(limit: 24) - map.register_type ODBC::SQL_FLOAT, Type::Float.new - map.register_type ODBC::SQL_DOUBLE, Type::Float.new(limit: 53) - map.register_type ODBC::SQL_DECIMAL, Type::Float.new - map.register_type ODBC::SQL_NUMERIC, Type::Integer.new - map.register_type ODBC::SQL_BINARY, Type::Binary.new - map.register_type ODBC::SQL_DATE, Type::Date.new - map.register_type ODBC::SQL_DATETIME, Type::DateTime.new - map.register_type ODBC::SQL_TIME, Type::Time.new - map.register_type ODBC::SQL_TIMESTAMP, Type::DateTime.new - map.register_type ODBC::SQL_GUID, Type::String.new - - alias_type map, ODBC::SQL_BIT, 'boolean' - alias_type map, ODBC::SQL_VARCHAR, ODBC::SQL_CHAR - alias_type map, ODBC::SQL_WCHAR, ODBC::SQL_CHAR - alias_type map, ODBC::SQL_WVARCHAR, ODBC::SQL_CHAR - alias_type map, ODBC::SQL_WLONGVARCHAR, ODBC::SQL_LONGVARCHAR - alias_type map, ODBC::SQL_VARBINARY, ODBC::SQL_BINARY - alias_type map, ODBC::SQL_LONGVARBINARY, ODBC::SQL_BINARY - alias_type map, ODBC::SQL_TYPE_DATE, ODBC::SQL_DATE - alias_type map, ODBC::SQL_TYPE_TIME, ODBC::SQL_TIME - alias_type map, ODBC::SQL_TYPE_TIMESTAMP, ODBC::SQL_TIMESTAMP + map.register_type :boolean, Type::Boolean.new + map.register_type :json, Type::Json.new + map.register_type :date, Type::Date.new + map.register_type :string, Type::String.new + map.register_type :datetime, Type::DateTime.new + map.register_type :time, Type::Time.new + map.register_type :binary, Type::Binary.new + map.register_type :float, Type::Float.new + map.register_type :integer, Type::Integer.new + map.register_type :decimal, Type::Decimal.new end # Translate an exception from the native DBMS to something usable by diff --git a/lib/odbc_adapter/quoting.rb b/lib/odbc_adapter/quoting.rb index a499612e..b41fa878 100644 --- a/lib/odbc_adapter/quoting.rb +++ b/lib/odbc_adapter/quoting.rb @@ -38,5 +38,9 @@ def quoted_date(value) value.strftime('%Y-%m-%d') # Date end end + + def lookup_cast_type_from_column(column) # :nodoc: + type_map.lookup(column.type) + end end end diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index 3f1f926a..ae331ba1 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -85,17 +85,17 @@ def columns(table_name, _name = nil) # This section has been customized for Snowflake and will not work in general. args = { sql_type: col_native_type, type: col_native_type, limit: col_limit } - args[:type] = 'boolean' if col_native_type == "BOOLEAN" # self.class::BOOLEAN_TYPE - args[:type] = 'json' if col_native_type == "VARIANT" || col_native_type == "JSON" - args[:type] = 'date' if col_native_type == "DATE" - args[:type] = 'string' if col_native_type == "VARCHAR" - args[:type] = 'datetime' if col_native_type == "TIMESTAMP" - args[:type] = 'time' if col_native_type == "TIME" - args[:type] = 'binary' if col_native_type == "BINARY" - args[:type] = 'float' if col_native_type == "DOUBLE" + args[:type] = :boolean if col_native_type == "BOOLEAN" # self.class::BOOLEAN_TYPE + args[:type] = :json if col_native_type == "VARIANT" || col_native_type == "JSON" + args[:type] = :date if col_native_type == "DATE" + args[:type] = :string if col_native_type == "VARCHAR" + args[:type] = :datetime if col_native_type == "TIMESTAMP" + args[:type] = :time if col_native_type == "TIME" + args[:type] = :binary if col_native_type == "BINARY" + args[:type] = :float if col_native_type == "DOUBLE" if [ODBC::SQL_DECIMAL, ODBC::SQL_NUMERIC].include?(col_sql_type) - args[:type] = col_scale == 0 ? 'integer' : 'decimal' + args[:type] = col_scale == 0 ? :integer : :decimal args[:scale] = col_scale || 0 args[:precision] = col_limit end From 9beade6b93364105ed3f11859a3d407c5682a065 Mon Sep 17 00:00:00 2001 From: Asher Cerka Date: Tue, 19 Apr 2022 14:11:05 -0400 Subject: [PATCH 30/49] MOJ-150 MOJ-151 Snowflake type mapping & array_of... types for Adds mapping for the remaining snowflake types, removes mapping for JSON type (as this isn't actually a type returned by snowflake ever) Add array_of_intgers, array_of_binaries, etc... types that can be used with attribute tagging to quickly perform useful type coercion. --- .../connection_adapters/odbc_adapter.rb | 6 ++- lib/odbc_adapter/database_statements.rb | 4 ++ lib/odbc_adapter/quoting.rb | 2 +- lib/odbc_adapter/schema_statements.rb | 36 ++++++++++------- lib/odbc_adapter/type/array.rb | 39 +++++++++++++++++++ 5 files changed, 70 insertions(+), 17 deletions(-) create mode 100644 lib/odbc_adapter/type/array.rb diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 4db1f816..750a885a 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -14,6 +14,8 @@ require 'odbc_adapter/registry' require 'odbc_adapter/version' +require 'odbc_adapter/type/array' + module ActiveRecord class Base class << self @@ -156,7 +158,9 @@ def initialize_type_map(map) map.register_type :binary, Type::Binary.new map.register_type :float, Type::Float.new map.register_type :integer, Type::Integer.new - map.register_type :decimal, Type::Decimal.new + map.register_type(:decimal) do |_sql_type, column_data| + Type::Decimal.new(precision: column_data.precision, scale: column_data.scale) + end end # Translate an exception from the native DBMS to something usable by diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index 9705b962..61076aa6 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -65,6 +65,10 @@ def default_sequence_name(table, _column) "#{table}_seq" end + def empty_insert_statement_value(primary_key = nil) + "(#{primary_key}) VALUES (DEFAULT)" + end + private # A custom hook to allow end users to overwrite the type casting before it diff --git a/lib/odbc_adapter/quoting.rb b/lib/odbc_adapter/quoting.rb index b41fa878..677a91c9 100644 --- a/lib/odbc_adapter/quoting.rb +++ b/lib/odbc_adapter/quoting.rb @@ -40,7 +40,7 @@ def quoted_date(value) end def lookup_cast_type_from_column(column) # :nodoc: - type_map.lookup(column.type) + type_map.lookup(column.type, column) end end end diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index ae331ba1..e168260a 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -75,7 +75,6 @@ def columns(table_name, _name = nil) next unless col[0] =~ db_regex && col[1] =~ schema_regex col_name = col[3] # SQLColumns: COLUMN_NAME col_default = col[12] # SQLColumns: COLUMN_DEF - col_sql_type = col[4] # SQLColumns: DATA_TYPE col_native_type = col[5] # SQLColumns: TYPE_NAME col_limit = col[6] # SQLColumns: COLUMN_SIZE col_scale = col[8] # SQLColumns: DECIMAL_DIGITS @@ -85,20 +84,27 @@ def columns(table_name, _name = nil) # This section has been customized for Snowflake and will not work in general. args = { sql_type: col_native_type, type: col_native_type, limit: col_limit } - args[:type] = :boolean if col_native_type == "BOOLEAN" # self.class::BOOLEAN_TYPE - args[:type] = :json if col_native_type == "VARIANT" || col_native_type == "JSON" - args[:type] = :date if col_native_type == "DATE" - args[:type] = :string if col_native_type == "VARCHAR" - args[:type] = :datetime if col_native_type == "TIMESTAMP" - args[:type] = :time if col_native_type == "TIME" - args[:type] = :binary if col_native_type == "BINARY" - args[:type] = :float if col_native_type == "DOUBLE" - - if [ODBC::SQL_DECIMAL, ODBC::SQL_NUMERIC].include?(col_sql_type) - args[:type] = col_scale == 0 ? :integer : :decimal - args[:scale] = col_scale || 0 - args[:precision] = col_limit - end + args[:type] = case col_native_type + when "BOOLEAN" then :boolean + when "VARIANT", "ARRAY", "STRUCT" then :json + when "DATE" then :date + when "VARCHAR" then :string + when "TIMESTAMP" then :datetime + when "TIME" then :time + when "BINARY" then :binary + when "DOUBLE" then :float + when "DECIMAL" + if col_scale == 0 + :integer + else + args[:scale] = col_scale + args[:precision] = col_limit + :decimal + end + else + nil + end + sql_type_metadata = ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new(**args) # The @connection.columns function returns empty strings for column defaults. diff --git a/lib/odbc_adapter/type/array.rb b/lib/odbc_adapter/type/array.rb new file mode 100644 index 00000000..3a2b567c --- /dev/null +++ b/lib/odbc_adapter/type/array.rb @@ -0,0 +1,39 @@ +module ODBCAdapter + module Type + def Type.array(type) + newArrayClass = Class.new(ActiveRecord::Type::Value) + newArrayClass.define_method :cast_value do |value| + base_array = ActiveSupport::JSON.decode(value) rescue nil + base_array.map do |element| + ret = type.cast(element) + ret + end + end + + newArrayClass.define_method :serialize do |value| + ActiveSupport::JSON.encode(value.map { |element| type.serialize(element)}) unless value.nil? + end + + newArrayClass.define_method :changed_in_place? do |raw_old_value, new_value| + deserialize(raw_old_value) != new_value + end + + newArrayClass + end + end + + +end + +ActiveRecord::Type.register(:array_of_big_integers, ODBCAdapter::Type.array(ActiveRecord::Type::BigInteger.new)) +ActiveRecord::Type.register(:array_of_binaries, ODBCAdapter::Type.array(ActiveRecord::Type::Binary.new)) +ActiveRecord::Type.register(:array_of_booleans, ODBCAdapter::Type.array(ActiveRecord::Type::Boolean.new)) +ActiveRecord::Type.register(:array_of_dates, ODBCAdapter::Type.array(ActiveRecord::Type::Date.new)) +ActiveRecord::Type.register(:array_of_date_times, ODBCAdapter::Type.array(ActiveRecord::Type::DateTime.new)) +ActiveRecord::Type.register(:array_of_decimals, ODBCAdapter::Type.array(ActiveRecord::Type::Decimal.new)) +ActiveRecord::Type.register(:array_of_floats, ODBCAdapter::Type.array(ActiveRecord::Type::Float.new)) +ActiveRecord::Type.register(:array_of_immutable_strings, ODBCAdapter::Type.array(ActiveRecord::Type::ImmutableString.new)) +ActiveRecord::Type.register(:array_of_integers, ODBCAdapter::Type.array(ActiveRecord::Type::Integer.new)) +ActiveRecord::Type.register(:array_of_strings, ODBCAdapter::Type.array(ActiveRecord::Type::String.new)) +ActiveRecord::Type.register(:array_of_times, ODBCAdapter::Type.array(ActiveRecord::Type::Time.new)) +ActiveRecord::Type.register(:array_of_values, ODBCAdapter::Type.array(ActiveRecord::Type::Value.new)) From f906f7a608d358c99e2b1dc233fb646a47005f04 Mon Sep 17 00:00:00 2001 From: Asher Cerka Date: Fri, 22 Apr 2022 17:26:39 -0400 Subject: [PATCH 31/49] MOJ-154 This is all of the work to get objects and arrays quoted properly for the format we'll need them in for updates. Next task is to control saves so that they omit objects/arrays on insert and then go back to update them after the fact --- .../connection_adapters/odbc_adapter.rb | 6 ++- lib/odbc_adapter/quoting.rb | 15 +++++++ lib/odbc_adapter/schema_statements.rb | 4 +- lib/odbc_adapter/type/array.rb | 39 ------------------- lib/odbc_adapter/type/array_of.rb | 23 +++++++++++ .../type/internal/snowflake_variant.rb | 18 +++++++++ lib/odbc_adapter/type/object.rb | 20 ++++++++++ lib/odbc_adapter/type/type.rb | 39 +++++++++++++++++++ lib/odbc_adapter/type/variant.rb | 24 ++++++++++++ 9 files changed, 146 insertions(+), 42 deletions(-) delete mode 100644 lib/odbc_adapter/type/array.rb create mode 100644 lib/odbc_adapter/type/array_of.rb create mode 100644 lib/odbc_adapter/type/internal/snowflake_variant.rb create mode 100644 lib/odbc_adapter/type/object.rb create mode 100644 lib/odbc_adapter/type/type.rb create mode 100644 lib/odbc_adapter/type/variant.rb diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 750a885a..fa016ff5 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -14,7 +14,7 @@ require 'odbc_adapter/registry' require 'odbc_adapter/version' -require 'odbc_adapter/type/array' +require 'odbc_adapter/type/type' module ActiveRecord class Base @@ -150,7 +150,6 @@ def new_column(name, default, sql_type_metadata, null, native_type = nil) #Snowflake ODBC Adapter specific def initialize_type_map(map) map.register_type :boolean, Type::Boolean.new - map.register_type :json, Type::Json.new map.register_type :date, Type::Date.new map.register_type :string, Type::String.new map.register_type :datetime, Type::DateTime.new @@ -161,6 +160,9 @@ def initialize_type_map(map) map.register_type(:decimal) do |_sql_type, column_data| Type::Decimal.new(precision: column_data.precision, scale: column_data.scale) end + map.register_type :object, ::ODBCAdapter::Type::SnowflakeObject.new + map.register_type :array, ::ODBCAdapter::Type::ArrayOfValues.new + map.register_type :variant, ::ODBCAdapter::Type::Variant.new end # Translate an exception from the native DBMS to something usable by diff --git a/lib/odbc_adapter/quoting.rb b/lib/odbc_adapter/quoting.rb index 677a91c9..fe74a247 100644 --- a/lib/odbc_adapter/quoting.rb +++ b/lib/odbc_adapter/quoting.rb @@ -42,5 +42,20 @@ def quoted_date(value) def lookup_cast_type_from_column(column) # :nodoc: type_map.lookup(column.type, column) end + + def quote_hash(hash:) + "OBJECT_CONSTRUCT(" + hash.map {|key, value| quote(key) + "," + quote(value)}.join(",") + ")" + end + + def quote_array(array:) + "ARRAY_CONSTRUCT(" + array.map { |element| quote(element) }.join(",") + ")" + end + + def quote(value) + if value.is_a? Hash then return quote_hash hash: value end + if value.is_a? Array then return quote_array array: value end + if value.is_a? Type::SnowflakeVariant then return value.quote self end + super + end end end diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index e168260a..90e8c755 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -86,7 +86,9 @@ def columns(table_name, _name = nil) args = { sql_type: col_native_type, type: col_native_type, limit: col_limit } args[:type] = case col_native_type when "BOOLEAN" then :boolean - when "VARIANT", "ARRAY", "STRUCT" then :json + when "VARIANT" then :variant + when "ARRAY" then :array + when "STRUCT" then :object when "DATE" then :date when "VARCHAR" then :string when "TIMESTAMP" then :datetime diff --git a/lib/odbc_adapter/type/array.rb b/lib/odbc_adapter/type/array.rb deleted file mode 100644 index 3a2b567c..00000000 --- a/lib/odbc_adapter/type/array.rb +++ /dev/null @@ -1,39 +0,0 @@ -module ODBCAdapter - module Type - def Type.array(type) - newArrayClass = Class.new(ActiveRecord::Type::Value) - newArrayClass.define_method :cast_value do |value| - base_array = ActiveSupport::JSON.decode(value) rescue nil - base_array.map do |element| - ret = type.cast(element) - ret - end - end - - newArrayClass.define_method :serialize do |value| - ActiveSupport::JSON.encode(value.map { |element| type.serialize(element)}) unless value.nil? - end - - newArrayClass.define_method :changed_in_place? do |raw_old_value, new_value| - deserialize(raw_old_value) != new_value - end - - newArrayClass - end - end - - -end - -ActiveRecord::Type.register(:array_of_big_integers, ODBCAdapter::Type.array(ActiveRecord::Type::BigInteger.new)) -ActiveRecord::Type.register(:array_of_binaries, ODBCAdapter::Type.array(ActiveRecord::Type::Binary.new)) -ActiveRecord::Type.register(:array_of_booleans, ODBCAdapter::Type.array(ActiveRecord::Type::Boolean.new)) -ActiveRecord::Type.register(:array_of_dates, ODBCAdapter::Type.array(ActiveRecord::Type::Date.new)) -ActiveRecord::Type.register(:array_of_date_times, ODBCAdapter::Type.array(ActiveRecord::Type::DateTime.new)) -ActiveRecord::Type.register(:array_of_decimals, ODBCAdapter::Type.array(ActiveRecord::Type::Decimal.new)) -ActiveRecord::Type.register(:array_of_floats, ODBCAdapter::Type.array(ActiveRecord::Type::Float.new)) -ActiveRecord::Type.register(:array_of_immutable_strings, ODBCAdapter::Type.array(ActiveRecord::Type::ImmutableString.new)) -ActiveRecord::Type.register(:array_of_integers, ODBCAdapter::Type.array(ActiveRecord::Type::Integer.new)) -ActiveRecord::Type.register(:array_of_strings, ODBCAdapter::Type.array(ActiveRecord::Type::String.new)) -ActiveRecord::Type.register(:array_of_times, ODBCAdapter::Type.array(ActiveRecord::Type::Time.new)) -ActiveRecord::Type.register(:array_of_values, ODBCAdapter::Type.array(ActiveRecord::Type::Value.new)) diff --git a/lib/odbc_adapter/type/array_of.rb b/lib/odbc_adapter/type/array_of.rb new file mode 100644 index 00000000..01b3ac46 --- /dev/null +++ b/lib/odbc_adapter/type/array_of.rb @@ -0,0 +1,23 @@ +module ODBCAdapter + module Type + def Type.array_of(type) + newArrayClass = Class.new(ActiveRecord::Type::Value) + + newArrayClass.define_method :cast_value do |value| + return value unless value.is_a? String + base_array = ActiveSupport::JSON.decode(value) rescue nil + base_array.map { |element| type.cast(element) } + end + + newArrayClass.define_method :serialize do |value| + value.to_a.map { |element| type.serialize(element)} unless value.nil? + end + + newArrayClass.define_method :changed_in_place? do |raw_old_value, new_value| + deserialize(raw_old_value) != new_value + end + + newArrayClass + end + end +end diff --git a/lib/odbc_adapter/type/internal/snowflake_variant.rb b/lib/odbc_adapter/type/internal/snowflake_variant.rb new file mode 100644 index 00000000..7875f332 --- /dev/null +++ b/lib/odbc_adapter/type/internal/snowflake_variant.rb @@ -0,0 +1,18 @@ +module ODBCAdapter + module Type + class SnowflakeVariant + # Acts as a wrapper around other data types to make sure that they get typecasted into variants during quoting + def initialize(internal_data) + @internal_data = internal_data + end + + def quote(adapter) + adapter.quote(@internal_data) + "::VARIANT" + end + + def internal_data + @internal_data + end + end + end +end diff --git a/lib/odbc_adapter/type/object.rb b/lib/odbc_adapter/type/object.rb new file mode 100644 index 00000000..d655260d --- /dev/null +++ b/lib/odbc_adapter/type/object.rb @@ -0,0 +1,20 @@ +module ODBCAdapter + module Type + class SnowflakeObject < ActiveRecord::Type::Value + + def cast_value(value) + # deserialize can contain the results of the previous serialize, rather than the database returned value + if value.is_a? Hash then return value end + ActiveSupport::JSON.decode(value) rescue nil + end + + def serialize(value) + value.to_h unless value.nil? + end + + def changed_in_place?(raw_old_value, new_value) + deserialize(raw_old_value) != new_value + end + end + end +end diff --git a/lib/odbc_adapter/type/type.rb b/lib/odbc_adapter/type/type.rb new file mode 100644 index 00000000..d7e757e8 --- /dev/null +++ b/lib/odbc_adapter/type/type.rb @@ -0,0 +1,39 @@ +require 'odbc_adapter/type/array_of' +require 'odbc_adapter/type/object' +require 'odbc_adapter/type/variant' + +require 'odbc_adapter/type/internal/snowflake_variant' + +module ODBCAdapter + module Type + ArrayOfBigIntegers = array_of(ActiveRecord::Type::BigInteger.new) + ArrayOfBinaries = array_of(ActiveRecord::Type::Binary.new) + ArrayOfBooleans = array_of(ActiveRecord::Type::Boolean.new) + ArrayOfDates = array_of(ActiveRecord::Type::Date.new) + ArrayOfDateTimes = array_of(ActiveRecord::Type::DateTime.new) + ArrayOfDecimals = array_of(ActiveRecord::Type::Decimal.new) + ArrayOfFloats = array_of(ActiveRecord::Type::Float.new) + ArrayOfImmutableStrings = array_of(ActiveRecord::Type::ImmutableString.new) + ArrayOfIntegers = array_of(ActiveRecord::Type::Integer.new) + ArrayOfStrings = array_of(ActiveRecord::Type::String.new) + ArrayOfTimes = array_of(ActiveRecord::Type::Time.new) + ArrayOfValues = array_of(ActiveRecord::Type::Value.new) + + ActiveRecord::Type.register(:array_of_big_integers, ArrayOfBigIntegers) + ActiveRecord::Type.register(:array_of_binaries, ArrayOfBinaries) + ActiveRecord::Type.register(:array_of_booleans, ArrayOfBooleans) + ActiveRecord::Type.register(:array_of_dates, ArrayOfDates) + ActiveRecord::Type.register(:array_of_date_times, ArrayOfDateTimes) + ActiveRecord::Type.register(:array_of_decimals, ArrayOfDecimals) + ActiveRecord::Type.register(:array_of_floats, ArrayOfFloats) + ActiveRecord::Type.register(:array_of_immutable_strings, ArrayOfImmutableStrings) + ActiveRecord::Type.register(:array_of_integers, ArrayOfIntegers) + ActiveRecord::Type.register(:array_of_strings, ArrayOfStrings) + ActiveRecord::Type.register(:array_of_times, ArrayOfTimes) + ActiveRecord::Type.register(:array_of_values, ArrayOfValues) + + ActiveRecord::Type.register(:object, Object) + + ActiveRecord::Type.register(:variant, Variant) + end +end diff --git a/lib/odbc_adapter/type/variant.rb b/lib/odbc_adapter/type/variant.rb new file mode 100644 index 00000000..82b3e903 --- /dev/null +++ b/lib/odbc_adapter/type/variant.rb @@ -0,0 +1,24 @@ +module ODBCAdapter + module Type + class Variant < ActiveRecord::Type::Value + + def deserialize(value) + # deserialize can contain the results of the previous serialize, rather than the database returned value + if value.is_a? SnowflakeVariant then return value.internal_data end + ActiveSupport::JSON.decode(value) rescue nil + end + + def cast(value) + value + end + + def serialize(value) + SnowflakeVariant.new(value) unless value.nil? + end + + def changed_in_place?(raw_old_value, new_value) + deserialize(raw_old_value) != new_value + end + end + end +end From 13d49893c36b6300c531bb029c71752c05c53af4 Mon Sep 17 00:00:00 2001 From: Asher Cerka Date: Thu, 28 Apr 2022 13:27:55 -0400 Subject: [PATCH 32/49] MOJ-156 This is a working methodology for automatically generating IDs. EasyIdentified can be safely included in the ApplicationRecord as it will do nothing under most circumstances. Same for InsertAttributeStripper Still need to do a little bit of cleanup, but wanted to get the code committed and pushed in it's current state. --- .../connection_adapters/odbc_adapter.rb | 3 +- lib/odbc_adapter/concerns/auto_identified.rb | 21 +++++++ lib/odbc_adapter/concerns/concern.rb | 4 ++ lib/odbc_adapter/concerns/easy_identified.rb | 32 ++++++++++ .../concerns/insert_attribute_stripper.rb | 60 +++++++++++++++++++ lib/odbc_adapter/type/snowflake_integer.rb | 15 +++++ lib/odbc_adapter/type/type.rb | 1 + 7 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 lib/odbc_adapter/concerns/auto_identified.rb create mode 100644 lib/odbc_adapter/concerns/concern.rb create mode 100644 lib/odbc_adapter/concerns/easy_identified.rb create mode 100644 lib/odbc_adapter/concerns/insert_attribute_stripper.rb create mode 100644 lib/odbc_adapter/type/snowflake_integer.rb diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index fa016ff5..a859d1c7 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -15,6 +15,7 @@ require 'odbc_adapter/version' require 'odbc_adapter/type/type' +require 'odbc_adapter/concerns/concern' module ActiveRecord class Base @@ -156,7 +157,7 @@ def initialize_type_map(map) map.register_type :time, Type::Time.new map.register_type :binary, Type::Binary.new map.register_type :float, Type::Float.new - map.register_type :integer, Type::Integer.new + map.register_type :integer, ::ODBCAdapter::Type::SnowflakeInteger.new map.register_type(:decimal) do |_sql_type, column_data| Type::Decimal.new(precision: column_data.precision, scale: column_data.scale) end diff --git a/lib/odbc_adapter/concerns/auto_identified.rb b/lib/odbc_adapter/concerns/auto_identified.rb new file mode 100644 index 00000000..8fcef4b0 --- /dev/null +++ b/lib/odbc_adapter/concerns/auto_identified.rb @@ -0,0 +1,21 @@ +module ODBCAdapter + module AutoIdentified + extend ActiveSupport::Concern + include EasyIdentified + + included do + alias_method :pre_auto_identified_save, :save + alias_method :pre_auto_identified_save!, :save! + + def save(**options, &block) + generate_id + pre_auto_identified_save(**options, &block) + end + + def save!(**options, &block) + generate_id + pre_auto_identified_save!(**options, &block) + end + end + end +end diff --git a/lib/odbc_adapter/concerns/concern.rb b/lib/odbc_adapter/concerns/concern.rb new file mode 100644 index 00000000..40ac354e --- /dev/null +++ b/lib/odbc_adapter/concerns/concern.rb @@ -0,0 +1,4 @@ + +require 'odbc_adapter/concerns/easy_identified' +require 'odbc_adapter/concerns/auto_identified' +require 'odbc_adapter/concerns/insert_attribute_stripper' diff --git a/lib/odbc_adapter/concerns/easy_identified.rb b/lib/odbc_adapter/concerns/easy_identified.rb new file mode 100644 index 00000000..c391cf1a --- /dev/null +++ b/lib/odbc_adapter/concerns/easy_identified.rb @@ -0,0 +1,32 @@ + +module ODBCAdapter + module EasyIdentified + extend ActiveSupport::Concern + + included do + alias_method :pre_easy_identified_save, :save + alias_method :pre_easy_identified_save!, :save! + + def save(**options, &block) + if self[:id] == :auto_generate then generate_id(true) end + pre_easy_identified_save(**options, &block) + end + + def save!(**options, &block) + if self[:id] == :auto_generate then generate_id(true) end + pre_easy_identified_save!(**options, &block) + end + + def generate_id(force_new = false) + if self[:id] == nil || force_new then self[:id] = retrieve_id end + end + + private + + def retrieve_id + sequence_name = self.class.table_name + "_ID_SEQ" + ApplicationRecord.connection.exec_query("Select #{sequence_name}.nextval as new_id")[0]["new_id"] + end + end + end +end diff --git a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb new file mode 100644 index 00000000..990488ef --- /dev/null +++ b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb @@ -0,0 +1,60 @@ +module ODBCAdapter + module InsertAttributeStripper + extend ActiveSupport::Concern + include EasyIdentified + + included do + alias_method :pre_insert_attribute_stripper_save, :save + alias_method :pre_insert_attribute_stripper_save!, :save! + + def save(**options, &block) + ActiveRecord::Base.transaction do + if new_record? + striped_attributes = strip_unsafe_to_insert + if striped_attributes.any? then generate_id end + end + pre_insert_attribute_stripper_save(**options, &block) + if striped_attributes.any? + restore_stripped_attributes(striped_attributes) + pre_insert_attribute_stripper_save(**options, &block) + end + end + end + + def save!(**options, &block) + ActiveRecord::Base.transaction do + if new_record? + striped_attributes = strip_unsafe_to_insert + if striped_attributes.any? then generate_id end + end + pre_insert_attribute_stripper_save!(**options, &block) + if striped_attributes.any? + restore_stripped_attributes(striped_attributes) + pre_insert_attribute_stripper_save!(**options, &block) + end + end + end + + private + + UNSAFE_INSERT_TYPES ||= %i(variant object array) + + def strip_unsafe_to_insert + striped_attributes = {} + self.class.columns.each do |column| + if UNSAFE_INSERT_TYPES.include?(column.type) && attributes[column.name] != nil + striped_attributes[column.name] = attributes[column.name] + self[column.name] = nil + end + end + striped_attributes + end + + def restore_stripped_attributes(stripped_attributes) + stripped_attributes.each do |key, value| + self[key] = value + end + end + end + end +end diff --git a/lib/odbc_adapter/type/snowflake_integer.rb b/lib/odbc_adapter/type/snowflake_integer.rb new file mode 100644 index 00000000..cbedcc20 --- /dev/null +++ b/lib/odbc_adapter/type/snowflake_integer.rb @@ -0,0 +1,15 @@ + +module ODBCAdapter + module Type + class SnowflakeInteger < ActiveRecord::Type::Integer + # In order to allow for querying of IDs, + def cast(value) + if value == :auto_generate + return value + else + super + end + end + end + end +end diff --git a/lib/odbc_adapter/type/type.rb b/lib/odbc_adapter/type/type.rb index d7e757e8..862b0eec 100644 --- a/lib/odbc_adapter/type/type.rb +++ b/lib/odbc_adapter/type/type.rb @@ -1,6 +1,7 @@ require 'odbc_adapter/type/array_of' require 'odbc_adapter/type/object' require 'odbc_adapter/type/variant' +require 'odbc_adapter/type/snowflake_integer' require 'odbc_adapter/type/internal/snowflake_variant' From 6ddc2514e0304c81f7766316cdadf34c4ec53ff2 Mon Sep 17 00:00:00 2001 From: Asher Cerka Date: Fri, 29 Apr 2022 11:06:22 -0400 Subject: [PATCH 33/49] MOJ-156 Removing the AutoIdentified concern. No longer necessary or useful after reworking the EasyIdentified concern. --- lib/odbc_adapter/concerns/auto_identified.rb | 21 -------------------- lib/odbc_adapter/concerns/concern.rb | 1 - 2 files changed, 22 deletions(-) delete mode 100644 lib/odbc_adapter/concerns/auto_identified.rb diff --git a/lib/odbc_adapter/concerns/auto_identified.rb b/lib/odbc_adapter/concerns/auto_identified.rb deleted file mode 100644 index 8fcef4b0..00000000 --- a/lib/odbc_adapter/concerns/auto_identified.rb +++ /dev/null @@ -1,21 +0,0 @@ -module ODBCAdapter - module AutoIdentified - extend ActiveSupport::Concern - include EasyIdentified - - included do - alias_method :pre_auto_identified_save, :save - alias_method :pre_auto_identified_save!, :save! - - def save(**options, &block) - generate_id - pre_auto_identified_save(**options, &block) - end - - def save!(**options, &block) - generate_id - pre_auto_identified_save!(**options, &block) - end - end - end -end diff --git a/lib/odbc_adapter/concerns/concern.rb b/lib/odbc_adapter/concerns/concern.rb index 40ac354e..54a24986 100644 --- a/lib/odbc_adapter/concerns/concern.rb +++ b/lib/odbc_adapter/concerns/concern.rb @@ -1,4 +1,3 @@ require 'odbc_adapter/concerns/easy_identified' -require 'odbc_adapter/concerns/auto_identified' require 'odbc_adapter/concerns/insert_attribute_stripper' From acd567fa47c6a9f0db27e97e7dce32ca066625f3 Mon Sep 17 00:00:00 2001 From: Asher Cerka Date: Fri, 29 Apr 2022 18:09:50 -0400 Subject: [PATCH 34/49] MOJ-156 updates from comments Updated striped to stripped Added snowflake integer to the typemap and went ahead and updated all of the new types that were added to be specific to the odbc adapter now that I've discovered that's a thing. --- .../concerns/insert_attribute_stripper.rb | 22 +++++++------- lib/odbc_adapter/type/type.rb | 30 ++++++++++--------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb index 990488ef..f0214d5a 100644 --- a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb +++ b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb @@ -10,12 +10,12 @@ module InsertAttributeStripper def save(**options, &block) ActiveRecord::Base.transaction do if new_record? - striped_attributes = strip_unsafe_to_insert - if striped_attributes.any? then generate_id end + stripped_attributes = strip_unsafe_to_insert + if stripped_attributes.any? then generate_id end end pre_insert_attribute_stripper_save(**options, &block) - if striped_attributes.any? - restore_stripped_attributes(striped_attributes) + if stripped_attributes.any? + restore_stripped_attributes(stripped_attributes) pre_insert_attribute_stripper_save(**options, &block) end end @@ -24,12 +24,12 @@ def save(**options, &block) def save!(**options, &block) ActiveRecord::Base.transaction do if new_record? - striped_attributes = strip_unsafe_to_insert - if striped_attributes.any? then generate_id end + stripped_attributes = strip_unsafe_to_insert + if stripped_attributes.any? then generate_id end end pre_insert_attribute_stripper_save!(**options, &block) - if striped_attributes.any? - restore_stripped_attributes(striped_attributes) + if stripped_attributes.any? + restore_stripped_attributes(stripped_attributes) pre_insert_attribute_stripper_save!(**options, &block) end end @@ -40,14 +40,14 @@ def save!(**options, &block) UNSAFE_INSERT_TYPES ||= %i(variant object array) def strip_unsafe_to_insert - striped_attributes = {} + stripped_attributes = {} self.class.columns.each do |column| if UNSAFE_INSERT_TYPES.include?(column.type) && attributes[column.name] != nil - striped_attributes[column.name] = attributes[column.name] + stripped_attributes[column.name] = attributes[column.name] self[column.name] = nil end end - striped_attributes + stripped_attributes end def restore_stripped_attributes(stripped_attributes) diff --git a/lib/odbc_adapter/type/type.rb b/lib/odbc_adapter/type/type.rb index 862b0eec..c199e716 100644 --- a/lib/odbc_adapter/type/type.rb +++ b/lib/odbc_adapter/type/type.rb @@ -20,21 +20,23 @@ module Type ArrayOfTimes = array_of(ActiveRecord::Type::Time.new) ArrayOfValues = array_of(ActiveRecord::Type::Value.new) - ActiveRecord::Type.register(:array_of_big_integers, ArrayOfBigIntegers) - ActiveRecord::Type.register(:array_of_binaries, ArrayOfBinaries) - ActiveRecord::Type.register(:array_of_booleans, ArrayOfBooleans) - ActiveRecord::Type.register(:array_of_dates, ArrayOfDates) - ActiveRecord::Type.register(:array_of_date_times, ArrayOfDateTimes) - ActiveRecord::Type.register(:array_of_decimals, ArrayOfDecimals) - ActiveRecord::Type.register(:array_of_floats, ArrayOfFloats) - ActiveRecord::Type.register(:array_of_immutable_strings, ArrayOfImmutableStrings) - ActiveRecord::Type.register(:array_of_integers, ArrayOfIntegers) - ActiveRecord::Type.register(:array_of_strings, ArrayOfStrings) - ActiveRecord::Type.register(:array_of_times, ArrayOfTimes) - ActiveRecord::Type.register(:array_of_values, ArrayOfValues) + ActiveRecord::Type.register(:array_of_big_integers, ArrayOfBigIntegers, adapter: :odbc) + ActiveRecord::Type.register(:array_of_binaries, ArrayOfBinaries, adapter: :odbc) + ActiveRecord::Type.register(:array_of_booleans, ArrayOfBooleans, adapter: :odbc) + ActiveRecord::Type.register(:array_of_dates, ArrayOfDates, adapter: :odbc) + ActiveRecord::Type.register(:array_of_date_times, ArrayOfDateTimes, adapter: :odbc) + ActiveRecord::Type.register(:array_of_decimals, ArrayOfDecimals, adapter: :odbc) + ActiveRecord::Type.register(:array_of_floats, ArrayOfFloats, adapter: :odbc) + ActiveRecord::Type.register(:array_of_immutable_strings, ArrayOfImmutableStrings, adapter: :odbc) + ActiveRecord::Type.register(:array_of_integers, ArrayOfIntegers, adapter: :odbc) + ActiveRecord::Type.register(:array_of_strings, ArrayOfStrings, adapter: :odbc) + ActiveRecord::Type.register(:array_of_times, ArrayOfTimes, adapter: :odbc) + ActiveRecord::Type.register(:array_of_values, ArrayOfValues, adapter: :odbc) - ActiveRecord::Type.register(:object, Object) + ActiveRecord::Type.register(:object, Object, adapter: :odbc) - ActiveRecord::Type.register(:variant, Variant) + ActiveRecord::Type.register(:variant, Variant, adapter: :odbc) + + ActiveRecord::Type.register(:integer, SnowflakeInteger, adapter: :odbc) end end From 31aae0b7e2486daa01a88ef84144d65d4057013f Mon Sep 17 00:00:00 2001 From: Asher Cerka Date: Thu, 12 May 2022 14:16:28 -0400 Subject: [PATCH 35/49] MOJ-178 Updates to issues identified when working on MOJ-178 EasyIdentified now uses the connection of it's own class instead of using the connection of ApplicationRecord. - The connection of ApplicationRecord in springbuk is the postgres connection InsertAttributeStripped was bugged. Inserts worked fine, updates did not. Added necessary bits to fix that and realized that save and save! had grown to a considerable size with the only difference between them being the method that they called. Refactored them to call a common function and pass a method into that function so it calls the correct parent method. --- lib/odbc_adapter/concerns/easy_identified.rb | 2 +- .../concerns/insert_attribute_stripper.rb | 49 ++++++++----------- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/lib/odbc_adapter/concerns/easy_identified.rb b/lib/odbc_adapter/concerns/easy_identified.rb index c391cf1a..0a214e1f 100644 --- a/lib/odbc_adapter/concerns/easy_identified.rb +++ b/lib/odbc_adapter/concerns/easy_identified.rb @@ -25,7 +25,7 @@ def generate_id(force_new = false) def retrieve_id sequence_name = self.class.table_name + "_ID_SEQ" - ApplicationRecord.connection.exec_query("Select #{sequence_name}.nextval as new_id")[0]["new_id"] + self.class.connection.exec_query("Select #{sequence_name}.nextval as new_id")[0]["new_id"] end end end diff --git a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb index f0214d5a..76077ea1 100644 --- a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb +++ b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb @@ -8,46 +8,37 @@ module InsertAttributeStripper alias_method :pre_insert_attribute_stripper_save!, :save! def save(**options, &block) - ActiveRecord::Base.transaction do - if new_record? - stripped_attributes = strip_unsafe_to_insert - if stripped_attributes.any? then generate_id end - end - pre_insert_attribute_stripper_save(**options, &block) - if stripped_attributes.any? - restore_stripped_attributes(stripped_attributes) - pre_insert_attribute_stripper_save(**options, &block) - end - end + save_internal(method(:pre_insert_attribute_stripper_save), **options, &block) end def save!(**options, &block) - ActiveRecord::Base.transaction do - if new_record? - stripped_attributes = strip_unsafe_to_insert - if stripped_attributes.any? then generate_id end - end - pre_insert_attribute_stripper_save!(**options, &block) - if stripped_attributes.any? - restore_stripped_attributes(stripped_attributes) - pre_insert_attribute_stripper_save!(**options, &block) - end - end + save_internal(method(:pre_insert_attribute_stripper_save!), **options, &block) end private UNSAFE_INSERT_TYPES ||= %i(variant object array) - def strip_unsafe_to_insert - stripped_attributes = {} - self.class.columns.each do |column| - if UNSAFE_INSERT_TYPES.include?(column.type) && attributes[column.name] != nil - stripped_attributes[column.name] = attributes[column.name] - self[column.name] = nil + def save_internal(base_function, **options, &block) + ActiveRecord::Base.transaction do + if new_record? + stripped_attributes = {} + self.class.columns.each do |column| + if UNSAFE_INSERT_TYPES.include?(column.type) && attributes[column.name] != nil + stripped_attributes[column.name] = attributes[column.name] + self[column.name] = nil + end + end + if stripped_attributes.any? then generate_id end + else + stripped_attributes = {} + end + base_function.call(**options, &block) + if stripped_attributes.any? + restore_stripped_attributes(stripped_attributes) + base_function.call(**options, &block) end end - stripped_attributes end def restore_stripped_attributes(stripped_attributes) From 0104f254d486bf72aabe0007a4b20751643098b3 Mon Sep 17 00:00:00 2001 From: Asher Cerka Date: Fri, 13 May 2022 16:46:25 -0400 Subject: [PATCH 36/49] MOJ-178 More fixes discovered while working on MOJ-178 RecordNotUnique, QueryTimeoutError, and ConnectionFailedError all inherit from StatementInvalid which no longer accepts an exception as it's second argument, but does take the SQL and binds. Use self.class for the InsertAttributeStripper transaction. ActiveRecord::Base may reference a different database - Currently in springbuk, ActiveRecord::Base goes to the postgres database. self.class will go to the database of whatever parent class (correctly going to snowflake where it's used) --- lib/active_record/connection_adapters/odbc_adapter.rb | 6 +++--- lib/odbc_adapter/concerns/insert_attribute_stripper.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index a859d1c7..d6436181 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -172,13 +172,13 @@ def translate_exception(exception, message:, sql:, binds:) error_number = exception.message[/^\d+/].to_i if error_number == ERR_DUPLICATE_KEY_VALUE - ActiveRecord::RecordNotUnique.new(message, exception) + ActiveRecord::RecordNotUnique.new(message, sql: sql, binds: binds) elsif error_number == ERR_QUERY_TIMED_OUT || exception.message =~ ERR_QUERY_TIMED_OUT_MESSAGE - ::ODBCAdapter::QueryTimeoutError.new(message, exception) + ::ODBCAdapter::QueryTimeoutError.new(message, sql: sql, binds: binds) elsif exception.message.match(ERR_CONNECTION_FAILED_REGEX) || exception.message =~ ERR_CONNECTION_FAILED_MESSAGE begin reconnect! - ::ODBCAdapter::ConnectionFailedError.new(message, exception) + ::ODBCAdapter::ConnectionFailedError.new(message, sql: sql, binds: binds) rescue => e puts "unable to reconnect #{e}" end diff --git a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb index 76077ea1..0ace5dbb 100644 --- a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb +++ b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb @@ -20,7 +20,7 @@ def save!(**options, &block) UNSAFE_INSERT_TYPES ||= %i(variant object array) def save_internal(base_function, **options, &block) - ActiveRecord::Base.transaction do + self.class.transaction do if new_record? stripped_attributes = {} self.class.columns.each do |column| From 7f938a273b891b42e656fcdabcf79d0474a9873c Mon Sep 17 00:00:00 2001 From: Asher Cerka Date: Fri, 22 Jul 2022 15:24:42 -0400 Subject: [PATCH 37/49] MOJ-203 Discovered that snowflake returns slightly different types enumerations when you enable utf8. --- lib/odbc_adapter/database_statements.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/odbc_adapter/database_statements.rb b/lib/odbc_adapter/database_statements.rb index 61076aa6..cb7ad683 100644 --- a/lib/odbc_adapter/database_statements.rb +++ b/lib/odbc_adapter/database_statements.rb @@ -96,11 +96,11 @@ def dbms_type_cast(columns, rows) value.to_i when [ODBC::SQL_BIT].include?(column.type) value == 1 - when [ODBC::SQL_DATE].include?(column.type) + when [ODBC::SQL_DATE, ODBC::SQL_TYPE_DATE].include?(column.type) value.to_date - when [ODBC::SQL_TIME].include?(column.type) + when [ODBC::SQL_TIME, ODBC::SQL_TYPE_TIME].include?(column.type) value.to_time - when [ODBC::SQL_DATETIME, ODBC::SQL_TIMESTAMP].include?(column.type) + when [ODBC::SQL_DATETIME, ODBC::SQL_TIMESTAMP, ODBC::SQL_TYPE_TIMESTAMP].include?(column.type) value.to_datetime # when ["ARRAY"?, "OBJECT"?, "VARIANT"?].include?(column.type) # TODO: "ARRAY", "OBJECT", "VARIANT" all return as VARCHAR From 3f94eabb3509925299a5c4f65664eb6691e9d4aa Mon Sep 17 00:00:00 2001 From: Mark Goudie Date: Mon, 1 Aug 2022 21:49:57 -0500 Subject: [PATCH 38/49] changed Integer to bigInteger --- lib/odbc_adapter/type/snowflake_integer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/odbc_adapter/type/snowflake_integer.rb b/lib/odbc_adapter/type/snowflake_integer.rb index cbedcc20..516c5c73 100644 --- a/lib/odbc_adapter/type/snowflake_integer.rb +++ b/lib/odbc_adapter/type/snowflake_integer.rb @@ -1,7 +1,7 @@ module ODBCAdapter module Type - class SnowflakeInteger < ActiveRecord::Type::Integer + class SnowflakeInteger < ActiveRecord::Type::BigInteger # In order to allow for querying of IDs, def cast(value) if value == :auto_generate From b047f334b2ce4281490b76aad706f5de644e8126 Mon Sep 17 00:00:00 2001 From: Ross Crenshaw Date: Tue, 2 Aug 2022 12:04:11 -0400 Subject: [PATCH 39/49] Fix associated record saving --- lib/odbc_adapter/concerns/insert_attribute_stripper.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb index 0ace5dbb..35dce2a9 100644 --- a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb +++ b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb @@ -33,10 +33,12 @@ def save_internal(base_function, **options, &block) else stripped_attributes = {} end - base_function.call(**options, &block) + first_call_result = base_function.call(**options, &block) if stripped_attributes.any? restore_stripped_attributes(stripped_attributes) - base_function.call(**options, &block) + return base_function.call(**options, &block) + else + return first_call_result end end end From 1c59af45a6e75fe6184af3d1f2daecd3fc78609d Mon Sep 17 00:00:00 2001 From: Asher Cerka <83976948+ACerka-Springbuk@users.noreply.github.com> Date: Mon, 19 Dec 2022 10:59:54 -0500 Subject: [PATCH 40/49] Moj 450 update with prefetch primary key (#19) * MOJ-343 Active Record does support, even if it's not well documented, a method of automatically getting IDs from the database. Implemented this native method to deal with an issue with ActiveStorage. * MOJ-450 Removed the EasyIdentified concern & corrected a minor bug --- lib/active_record/connection_adapters/odbc_adapter.rb | 9 +++++++++ lib/odbc_adapter/concerns/insert_attribute_stripper.rb | 3 +-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index d6436181..26dcc3b1 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -146,6 +146,15 @@ def new_column(name, default, sql_type_metadata, null, native_type = nil) ::ODBCAdapter::Column.new(name, default, sql_type_metadata, null, native_type) end + #Snowflake doesn't have a mechanism to return the primary key on inserts, it needs prefetched + def prefetch_primary_key?(table_name = nil) + true + end + + def next_sequence_value(table_name = nil) + exec_query("SELECT #{table_name}.NEXTVAL as new_id").first["new_id"] + end + protected #Snowflake ODBC Adapter specific diff --git a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb index 35dce2a9..556526df 100644 --- a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb +++ b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb @@ -1,7 +1,6 @@ module ODBCAdapter module InsertAttributeStripper extend ActiveSupport::Concern - include EasyIdentified included do alias_method :pre_insert_attribute_stripper_save, :save @@ -29,11 +28,11 @@ def save_internal(base_function, **options, &block) self[column.name] = nil end end - if stripped_attributes.any? then generate_id end else stripped_attributes = {} end first_call_result = base_function.call(**options, &block) + return false if first_call_result == false if stripped_attributes.any? restore_stripped_attributes(stripped_attributes) return base_function.call(**options, &block) From 83586f46e9b367cd257ae203ad9164aa5088d328 Mon Sep 17 00:00:00 2001 From: Bob Fredenburg <71334116+bfredenburg@users.noreply.github.com> Date: Sun, 8 Jan 2023 22:46:31 -0500 Subject: [PATCH 41/49] Do not use bool alias (#21) --- lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb b/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb index 05fa17e0..2cb76617 100644 --- a/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb +++ b/lib/odbc_adapter/adapters/postgresql_odbc_adapter.rb @@ -3,7 +3,7 @@ module Adapters # Overrides specific to PostgreSQL. Mostly taken from # ActiveRecord::ConnectionAdapters::PostgreSQLAdapter class PostgreSQLODBCAdapter < ActiveRecord::ConnectionAdapters::ODBCAdapter - BOOLEAN_TYPE = 'bool'.freeze + BOOLEAN_TYPE = 'boolean'.freeze PRIMARY_KEY = 'SERIAL PRIMARY KEY'.freeze VARIANT_TYPE = 'VARIANT'.freeze DATE_TYPE = 'DATE'.freeze @@ -13,7 +13,7 @@ class PostgreSQLODBCAdapter < ActiveRecord::ConnectionAdapters::ODBCAdapter # Override to handle booleans appropriately def native_database_types - @native_database_types ||= super.merge(boolean: { name: 'bool' }) + @native_database_types ||= super.merge(boolean: { name: 'boolean' }) end def arel_visitor From e3ec758a37c6989439a7e22749336a794d0c4c79 Mon Sep 17 00:00:00 2001 From: Asher Cerka <83976948+ACerka-Springbuk@users.noreply.github.com> Date: Mon, 9 Jan 2023 12:43:23 -0500 Subject: [PATCH 42/49] MOJ-366 Two updates for added functionality (#20) * MOJ-366 Two updates for added functionality - Varient & object fields now support store_accessor attribute on models - Insert attribute stripper now validates the record before stripping the attributes, but then disables validation when doing the initial save so that it doesn't fail validation when required attributes are stripped off. * MOJ-366 Switched InsertAttributeStripper back to use calling the base function, so that it will always correctly return false or raise the error depending on if save or save! is called. * MOJ-366 Need to use options, not temp_options here. temp_options doesn't even exist yet. --- lib/odbc_adapter/concerns/insert_attribute_stripper.rb | 9 ++++++++- lib/odbc_adapter/type/object.rb | 4 ++++ lib/odbc_adapter/type/variant.rb | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb index 556526df..4af38cb8 100644 --- a/lib/odbc_adapter/concerns/insert_attribute_stripper.rb +++ b/lib/odbc_adapter/concerns/insert_attribute_stripper.rb @@ -19,6 +19,12 @@ def save!(**options, &block) UNSAFE_INSERT_TYPES ||= %i(variant object array) def save_internal(base_function, **options, &block) + # Unless the validations are turned off or the hash is valid just run the save. This will trigger validation + # errors normally for an invalid record. We then disable validations during the initial save, because we'll + # often be saving a technically invalid record as we've stripped off required elements. + unless options[:validate] == false || valid? + return base_function.call(**options, &block) + end self.class.transaction do if new_record? stripped_attributes = {} @@ -31,7 +37,8 @@ def save_internal(base_function, **options, &block) else stripped_attributes = {} end - first_call_result = base_function.call(**options, &block) + temp_options = options.merge(validate: false) + first_call_result = base_function.call(**temp_options, &block) return false if first_call_result == false if stripped_attributes.any? restore_stripped_attributes(stripped_attributes) diff --git a/lib/odbc_adapter/type/object.rb b/lib/odbc_adapter/type/object.rb index d655260d..853cf4d1 100644 --- a/lib/odbc_adapter/type/object.rb +++ b/lib/odbc_adapter/type/object.rb @@ -15,6 +15,10 @@ def serialize(value) def changed_in_place?(raw_old_value, new_value) deserialize(raw_old_value) != new_value end + + def accessor + ActiveRecord::Store::StringKeyedHashAccessor + end end end end diff --git a/lib/odbc_adapter/type/variant.rb b/lib/odbc_adapter/type/variant.rb index 82b3e903..fda5b67e 100644 --- a/lib/odbc_adapter/type/variant.rb +++ b/lib/odbc_adapter/type/variant.rb @@ -19,6 +19,10 @@ def serialize(value) def changed_in_place?(raw_old_value, new_value) deserialize(raw_old_value) != new_value end + + def accessor + ActiveRecord::Store::StringKeyedHashAccessor + end end end end From dbe56e257e26df88a3c6c623e6e31f0db3e8ff92 Mon Sep 17 00:00:00 2001 From: Bob Fredenburg <71334116+bfredenburg@users.noreply.github.com> Date: Mon, 20 Feb 2023 09:48:04 -0500 Subject: [PATCH 43/49] [COR-3018] Support Rails 7 (#22) * Support add_column in migrations * Remove testing code --- lib/active_record/connection_adapters/odbc_adapter.rb | 4 ++-- lib/odbc_adapter/quoting.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 26dcc3b1..1d62958d 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -202,8 +202,8 @@ def translate_exception(exception, message:, sql:, binds:) # work with non-string keys, and in our case the keys are (almost) all # numeric def alias_type(map, new_type, old_type) - map.register_type(new_type) do |_, *args| - map.lookup(old_type, *args) + map.register_type(new_type) do |_| + map.lookup(old_type) end end diff --git a/lib/odbc_adapter/quoting.rb b/lib/odbc_adapter/quoting.rb index fe74a247..73cf2719 100644 --- a/lib/odbc_adapter/quoting.rb +++ b/lib/odbc_adapter/quoting.rb @@ -40,7 +40,7 @@ def quoted_date(value) end def lookup_cast_type_from_column(column) # :nodoc: - type_map.lookup(column.type, column) + type_map.lookup(column.type) end def quote_hash(hash:) From 91d9c135eb51ce71d8f54d332d7e1fbf1e62d1f6 Mon Sep 17 00:00:00 2001 From: Asher Cerka <83976948+ACerka-Springbuk@users.noreply.github.com> Date: Thu, 11 May 2023 16:33:00 -0400 Subject: [PATCH 44/49] Moj 574 speedup hr uploads processing (#23) * MOJ-574 Updates to the ODBC adapter for a new merge_all functionality. Similar functionality to active record's insert/upsert all, but uses the SQL merge syntax instead of insert with update capabilities. * MOJ-574 Fixes to the rails 7 changes and added the ability to prune duplicate records in merge_all * MOJ-574 Removed extra debug statement * MOJ-574 Optimization to reduce memory and/or cpu usage. Large merges are killing sidekiq. * MOJ-574 Cleaning up the delete code, but ultimately leaving it intentionally disabled and inaccessible. It'll get worked on/tested if/when it's needed. * MOJ-574 I forgot to remove the delete_key code from the persistence portion of the adapter... * MOJ-574 And forgot to correct delete_keys in updatable and insertable columns methods --- .../connection_adapters/odbc_adapter.rb | 43 ++-- lib/active_record/merge_all.rb | 192 ++++++++++++++++++ lib/active_record/merge_all_persistence.rb | 14 ++ lib/odbc_adapter.rb | 1 + lib/odbc_adapter/quoting.rb | 2 +- lib/odbc_adapter/schema_statements.rb | 14 +- 6 files changed, 251 insertions(+), 15 deletions(-) create mode 100644 lib/active_record/merge_all.rb create mode 100644 lib/active_record/merge_all_persistence.rb diff --git a/lib/active_record/connection_adapters/odbc_adapter.rb b/lib/active_record/connection_adapters/odbc_adapter.rb index 1d62958d..f869f46c 100644 --- a/lib/active_record/connection_adapters/odbc_adapter.rb +++ b/lib/active_record/connection_adapters/odbc_adapter.rb @@ -155,24 +155,41 @@ def next_sequence_value(table_name = nil) exec_query("SELECT #{table_name}.NEXTVAL as new_id").first["new_id"] end + def build_merge_sql(merge) # :nodoc: + <<~SQL + MERGE #{merge.into} AS TARGET USING (#{merge.values_list}) AS SOURCE ON #{merge.match} + #{merge.merge_delete} + #{merge.merge_update} + #{merge.merge_insert} + SQL + end + + def exec_merge_all(sql, name) # :nodoc: + exec_query(sql, name) + end + protected #Snowflake ODBC Adapter specific def initialize_type_map(map) - map.register_type :boolean, Type::Boolean.new - map.register_type :date, Type::Date.new - map.register_type :string, Type::String.new - map.register_type :datetime, Type::DateTime.new - map.register_type :time, Type::Time.new - map.register_type :binary, Type::Binary.new - map.register_type :float, Type::Float.new - map.register_type :integer, ::ODBCAdapter::Type::SnowflakeInteger.new - map.register_type(:decimal) do |_sql_type, column_data| - Type::Decimal.new(precision: column_data.precision, scale: column_data.scale) + map.register_type %r(boolean)i, Type::Boolean.new + map.register_type %r(date)i, Type::Date.new + map.register_type %r(varchar)i, Type::String.new + map.register_type %r(time)i, Type::Time.new + map.register_type %r(timestamp)i, Type::DateTime.new + map.register_type %r(binary)i, Type::Binary.new + map.register_type %r(double)i, Type::Float.new + map.register_type(%r(decimal)i) do |sql_type| + scale = extract_scale(sql_type) + if scale == 0 + ::ODBCAdapter::Type::SnowflakeInteger.new + else + Type::Decimal.new(precision: extract_precision(sql_type), scale: scale) + end end - map.register_type :object, ::ODBCAdapter::Type::SnowflakeObject.new - map.register_type :array, ::ODBCAdapter::Type::ArrayOfValues.new - map.register_type :variant, ::ODBCAdapter::Type::Variant.new + map.register_type %r(struct)i, ::ODBCAdapter::Type::SnowflakeObject.new + map.register_type %r(array)i, ::ODBCAdapter::Type::ArrayOfValues.new + map.register_type %r(variant)i, ::ODBCAdapter::Type::Variant.new end # Translate an exception from the native DBMS to something usable by diff --git a/lib/active_record/merge_all.rb b/lib/active_record/merge_all.rb new file mode 100644 index 00000000..634d40cc --- /dev/null +++ b/lib/active_record/merge_all.rb @@ -0,0 +1,192 @@ +# frozen_string_literal: true + +require "active_support/core_ext/enumerable" + +module ActiveRecord + class MergeAll # :nodoc: + attr_reader :model, :connection, :merges, :keys + attr_reader :perform_inserts, :perform_updates, :delete_key + + def initialize(model, merges, perform_inserts: true, perform_updates: true, prune_duplicates: false) + raise ArgumentError, "Empty list of attributes passed" if merges.blank? + + # TODO: Implement perform_deletes. Most of the code is here, but all completely untested. + @model, @connection, @merges, @keys = model, model.connection, merges, merges.first.keys.map(&:to_s) + @perform_inserts, @perform_updates, @delete_key = perform_inserts, perform_updates, nil + + if model.scope_attributes? + @scope_attributes = model.scope_attributes + @keys |= @scope_attributes.keys + end + @keys = @keys.to_set + + ensure_valid_options_for_connection! + + if prune_duplicates + do_prune_duplicates + end + end + + def execute + message = +"#{model} " + message << "Bulk " if merges.many? + message << "Merge" + connection.exec_merge_all to_sql, message + end + + def updatable_columns + keys - readonly_columns - [delete_key] + end + + def insertable_columns + keys - [delete_key] + end + + def insertable_non_primary_columns + insertable_columns - primary_keys + end + + def primary_keys + Array(connection.schema_cache.primary_keys(model.table_name)) + end + + def map_key_with_value + merges.map do |attributes| + attributes = attributes.stringify_keys + attributes.merge!(scope_attributes) if scope_attributes + + verify_attributes(attributes) + + keys.map do |key| + yield key, attributes[key] + end + end + end + + def perform_deletes + !delete_key.nil? + end + + private + attr_reader :scope_attributes + + def ensure_valid_options_for_connection! + + end + + def do_prune_duplicates + unless primary_keys.to_set.subset?(keys) + raise ArgumentError, "Pruning duplicates requires presense of all primary keys in the merges" + end + @merges = merges.reverse + merges.uniq! do |merge| + primary_keys.map { |key| merge[key] } + end + merges.reverse! + end + + def to_sql + connection.build_merge_sql(ActiveRecord::MergeAll::Builder.new(self)) + end + + def readonly_columns + primary_keys + model.readonly_attributes.to_a + end + + def verify_attributes(attributes) + if keys != attributes.keys.to_set + raise ArgumentError, "All objects being merged must have the same keys" + end + end + + class Builder # :nodoc: + attr_reader :model + + delegate :keys, to: :merge_all + + def initialize(merge_all) + @merge_all, @model, @connection = merge_all, merge_all.model, merge_all.connection + end + + def into + # "INTO #{model.quoted_table_name} (#{columns_list})" + "INTO #{model.quoted_table_name}" + end + + def values_list + types = extract_types_from_columns_on(model.table_name, keys: keys) + + values_list = merge_all.map_key_with_value do |key, value| + connection.with_yaml_fallback(types[key].serialize(value)) + end + + values = connection.visitor.compile(Arel::Nodes::ValuesList.new(values_list)) + + "SELECT * FROM (#{values}) AS v1 (#{columns_list})" + end + + def match + quote_columns(merge_all.primary_keys).map { |column| "SOURCE.#{column}=TARGET.#{column}" }.join(" AND ") + end + + def merge_delete + merge_all.perform_deletes ? "WHEN MATCHED AND SOURCE.#{quote_column(merge_all.delete_key)} = TRUE THEN DELETE" : "" + end + + def merge_update + merge_all.perform_updates ? "WHEN MATCHED THEN UPDATE SET #{updatable_columns.map { |column| "TARGET.#{column}=SOURCE.#{column}" }.join(",")}" : "" + end + + def merge_insert + if merge_all.perform_inserts + <<~SQL + WHEN NOT MATCHED AND #{quote_columns(merge_all.primary_keys).map { |column| "SOURCE.#{column} IS NOT NULL" }.join(" AND ")} THEN INSERT (#{insertable_columns_list}) VALUES (#{quote_columns(merge_all.insertable_columns).map { |column| "SOURCE.#{column}"}.join(",")}) + WHEN NOT MATCHED AND #{quote_columns(merge_all.primary_keys).map { |column| "SOURCE.#{column} IS NULL" }.join(" OR ")} THEN INSERT (#{insertable_non_primary_columns_list}) VALUES (#{quote_columns(merge_all.insertable_non_primary_columns).map { |column| "SOURCE.#{column}"}.join(",")}) + SQL + else + "" + end + end + + private + attr_reader :connection, :merge_all + + def columns_list + format_columns(merge_all.keys) + end + + def insertable_columns_list + format_columns(merge_all.insertable_columns) + end + + def insertable_non_primary_columns_list + format_columns(merge_all.insertable_non_primary_columns) + end + + def updatable_columns + quote_columns(merge_all.updatable_columns) + end + + def extract_types_from_columns_on(table_name, keys:) + columns = connection.schema_cache.columns_hash(table_name) + + unknown_column = (keys - columns.keys).first + raise UnknownAttributeError.new(model.new, unknown_column) if unknown_column + + keys.index_with { |key| model.type_for_attribute(key) } + end + + def format_columns(columns) + columns.respond_to?(:map) ? quote_columns(columns).join(",") : columns + end + + def quote_columns(columns) + columns.map(&method(:quote_column)) + end + + def quote_column(column) + connection.quote_column_name(column) + end + end + end +end diff --git a/lib/active_record/merge_all_persistence.rb b/lib/active_record/merge_all_persistence.rb new file mode 100644 index 00000000..72f671de --- /dev/null +++ b/lib/active_record/merge_all_persistence.rb @@ -0,0 +1,14 @@ +require 'active_record/merge_all' + +module ActiveRecord + # = Active Record \Persistence + module MergeAllPersistence + extend ActiveSupport::Concern + + module ClassMethods + def merge_all!(attributes, perform_inserts: true, perform_updates: true, prune_duplicates: false) + MergeAll.new(self, attributes, perform_inserts: perform_inserts, perform_updates: perform_updates, prune_duplicates: prune_duplicates).execute + end + end + end +end diff --git a/lib/odbc_adapter.rb b/lib/odbc_adapter.rb index 194fb562..838d80ed 100644 --- a/lib/odbc_adapter.rb +++ b/lib/odbc_adapter.rb @@ -1,2 +1,3 @@ # Requiring with this pattern to mirror ActiveRecord require 'active_record/connection_adapters/odbc_adapter' +require 'active_record/merge_all_persistence' \ No newline at end of file diff --git a/lib/odbc_adapter/quoting.rb b/lib/odbc_adapter/quoting.rb index 73cf2719..789cb693 100644 --- a/lib/odbc_adapter/quoting.rb +++ b/lib/odbc_adapter/quoting.rb @@ -40,7 +40,7 @@ def quoted_date(value) end def lookup_cast_type_from_column(column) # :nodoc: - type_map.lookup(column.type) + type_map.lookup(column.sql_type) end def quote_hash(hash:) diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index 90e8c755..be00f2aa 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -83,7 +83,7 @@ def columns(table_name, _name = nil) col_nullable = nullability(col_name, col[17], col[10]) # This section has been customized for Snowflake and will not work in general. - args = { sql_type: col_native_type, type: col_native_type, limit: col_limit } + args = { sql_type: construct_sql_type(col_native_type, col_limit, col_scale), type: col_native_type, limit: col_limit } args[:type] = case col_native_type when "BOOLEAN" then :boolean when "VARIANT" then :variant @@ -177,5 +177,17 @@ def name_regex(name) /^#{name}$/i end end + + # Changes in rails 7 mean that we need all of the type information in the sql_type column + # This reconstructs sql types using limit (which is precision) and scale + def construct_sql_type(native_type, limit, scale) + if scale > 0 + "#{native_type}(#{limit},#{scale})" + elsif limit > 0 + "#{native_type}(#{limit})" + else + native_type + end + end end end From 9ef47f6a2edc2e22ffadbb70725fd1bc7c4499c5 Mon Sep 17 00:00:00 2001 From: Brent S <95236117+mbscoggins@users.noreply.github.com> Date: Fri, 2 Jun 2023 10:09:59 -0500 Subject: [PATCH 45/49] [MOJ-578] Use new springbuk odbc-ruby gem. (#24) * Use new springbuk odbc-ruby gem. * Increment version. --- lib/odbc_adapter/version.rb | 2 +- odbc_adapter.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/odbc_adapter/version.rb b/lib/odbc_adapter/version.rb index 598c96ec..8390e98b 100644 --- a/lib/odbc_adapter/version.rb +++ b/lib/odbc_adapter/version.rb @@ -1,3 +1,3 @@ module ODBCAdapter - VERSION = '5.0.6'.freeze + VERSION = '5.0.7'.freeze end diff --git a/odbc_adapter.gemspec b/odbc_adapter.gemspec index 0eafa155..571a96c0 100644 --- a/odbc_adapter.gemspec +++ b/odbc_adapter.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.add_dependency 'activerecord', '>= 5.0.1' - spec.add_dependency 'ruby-odbc', '~> 0.9' + spec.add_dependency 'odbc-ruby', '~> 1.0.0' spec.add_development_dependency 'bundler', '>= 1.14' spec.add_development_dependency 'minitest', '~> 5.10' From 6be7bb6105d9c5afcb3dd426d3628b92f1b35a59 Mon Sep 17 00:00:00 2001 From: Brent S <95236117+mbscoggins@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:29:01 -0500 Subject: [PATCH 46/49] [MOJ-586] Update odbc-ruby gem. (#26) Update odbc-ruby gem. Update version number. --- lib/odbc_adapter/version.rb | 2 +- odbc_adapter.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/odbc_adapter/version.rb b/lib/odbc_adapter/version.rb index 8390e98b..33905ee1 100644 --- a/lib/odbc_adapter/version.rb +++ b/lib/odbc_adapter/version.rb @@ -1,3 +1,3 @@ module ODBCAdapter - VERSION = '5.0.7'.freeze + VERSION = '5.0.8'.freeze end diff --git a/odbc_adapter.gemspec b/odbc_adapter.gemspec index 571a96c0..8960dfbb 100644 --- a/odbc_adapter.gemspec +++ b/odbc_adapter.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.add_dependency 'activerecord', '>= 5.0.1' - spec.add_dependency 'odbc-ruby', '~> 1.0.0' + spec.add_dependency 'odbc-ruby', '~> 1.0.1' spec.add_development_dependency 'bundler', '>= 1.14' spec.add_development_dependency 'minitest', '~> 5.10' From 8ad3e62adecdc0c005a4ccf9c4f692c847241d42 Mon Sep 17 00:00:00 2001 From: Asher Cerka <83976948+ACerka-Springbuk@users.noreply.github.com> Date: Tue, 18 Jul 2023 11:38:57 -0400 Subject: [PATCH 47/49] Moj 519 plan years controller (#27) * MOJ-519 Rewrote the column retrieval to use the information schema instead of odbc calls so we can get default values * MOJ-519 Removed unnecessary check for empty string default value * MOJ-519 Removed the idea of the table store. May be worth revisiting later on, but with multiple instances of the connection I'm not confident that the small gains are worthwhile. * MOJ-519 Committing code with debug still in place * MOJ-519 Removed the debug code and made the private methods private * MOJ-519 Remove case changes from the columns query so it can be used with quoted table names --- lib/odbc_adapter/schema_statements.rb | 118 +++++++++++++++++++++----- 1 file changed, 96 insertions(+), 22 deletions(-) diff --git a/lib/odbc_adapter/schema_statements.rb b/lib/odbc_adapter/schema_statements.rb index be00f2aa..50238ca4 100644 --- a/lib/odbc_adapter/schema_statements.rb +++ b/lib/odbc_adapter/schema_statements.rb @@ -62,27 +62,44 @@ def indexes(table_name, _name = nil) end end + def retrieve_column_data(table_name) + column_query = "SHOW COLUMNS IN TABLE #{table_name}" + + # Temporarily disable debug logging so we don't spam the log with table column queries + query_results = ActiveRecord::Base.logger.silence do + exec_query(column_query) + end + + column_data = query_results.map do |query_result| + data_type_parsed = JSON.parse(query_result["data_type"]) + { + column_name: query_result["column_name"], + col_default: extract_default_from_snowflake(query_result["default"]), + col_native_type: extract_data_type_from_snowflake(data_type_parsed["type"]), + column_size: extract_column_size_from_snowflake(data_type_parsed), + numeric_scale: extract_scale_from_snowflake(data_type_parsed), + is_nullable: data_type_parsed["nullable"] + } + end + + column_data + end + + # Returns an array of Column objects for the table specified by # +table_name+. + # This entire function has been customized for Snowflake and will not work in general. def columns(table_name, _name = nil) - stmt = @connection.columns(native_case(table_name.to_s)) - result = stmt.fetch_all || [] - stmt.drop + result = retrieve_column_data(table_name) - db_regex = name_regex(current_database) - schema_regex = name_regex(current_schema) result.each_with_object([]) do |col, cols| - next unless col[0] =~ db_regex && col[1] =~ schema_regex - col_name = col[3] # SQLColumns: COLUMN_NAME - col_default = col[12] # SQLColumns: COLUMN_DEF - col_native_type = col[5] # SQLColumns: TYPE_NAME - col_limit = col[6] # SQLColumns: COLUMN_SIZE - col_scale = col[8] # SQLColumns: DECIMAL_DIGITS + col_name = col[:column_name] + col_default = col[:col_default] + col_native_type = col[:col_native_type] + col_limit = col[:column_size] + col_scale = col[:numeric_scale] + col_nullable = col[:is_nullable] - # SQLColumns: IS_NULLABLE, SQLColumns: NULLABLE - col_nullable = nullability(col_name, col[17], col[10]) - - # This section has been customized for Snowflake and will not work in general. args = { sql_type: construct_sql_type(col_native_type, col_limit, col_scale), type: col_native_type, limit: col_limit } args[:type] = case col_native_type when "BOOLEAN" then :boolean @@ -109,13 +126,6 @@ def columns(table_name, _name = nil) sql_type_metadata = ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new(**args) - # The @connection.columns function returns empty strings for column defaults. - # Even when the column has a default value. This is a call to the ODBC layer - # with only enough Ruby to make the call happen. Replacing the empty string - # with nil permits Rails to set the current datetime for created_at and - # updated_at on model creates and updates. - col_default = nil if col_default == "" - cols << new_column(format_case(col_name), col_default, sql_type_metadata, col_nullable, col_native_type) end end @@ -189,5 +199,69 @@ def construct_sql_type(native_type, limit, scale) native_type end end + + private + + # Extracts the value from a Snowflake column default definition. + def extract_default_from_snowflake(default) + case default + # null + when nil + nil + # Quoted strings + when /\A[(B]?'(.*)'\z/m + $1.gsub("''", "'").gsub("\\\\","\\") + # Boolean types + when "TRUE" + "true" + when "FALSE" + "false" + # Numeric types + when /\A(-?\d+(\.\d*)?)\z/ + $1 + else + nil + end + end + + def extract_data_type_from_snowflake(snowflake_data_type) + case snowflake_data_type + when "NUMBER" + "DECIMAL" + when /\ATIMESTAMP_.*/ + "TIMESTAMP" + when "TEXT" + "VARCHAR" + when "FLOAT" + "DOUBLE" + when "FIXED" + "DECIMAL" + when "REAL" + "DOUBLE" + else + snowflake_data_type + end + end + + def extract_column_size_from_snowflake(type_information) + case type_information["type"] + when /\ATIMESTAMP_.*/ + 35 + when "DATE" + 10 + when "FLOAT" + 38 + when "REAL" + 38 + when "BOOLEAN" + 1 + else + type_information["length"] || type_information["precision"] || 0 + end + end + + def extract_scale_from_snowflake(type_information) + type_information["scale"] || 0 + end end end From ecd6964fd8bd2ca38efb528e15c8fd309937d07f Mon Sep 17 00:00:00 2001 From: Asher Cerka <83976948+ACerka-Springbuk@users.noreply.github.com> Date: Mon, 24 Jul 2023 12:29:58 -0400 Subject: [PATCH 48/49] APP-3029 Fixes prune duplicates. All records with a nil primary key get their primary key replaced by a new object, which will never compare true to anything else. (#28) --- lib/active_record/merge_all.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/active_record/merge_all.rb b/lib/active_record/merge_all.rb index 634d40cc..352b9cb3 100644 --- a/lib/active_record/merge_all.rb +++ b/lib/active_record/merge_all.rb @@ -80,7 +80,10 @@ def do_prune_duplicates end @merges = merges.reverse merges.uniq! do |merge| - primary_keys.map { |key| merge[key] } + # Map the primary keys to determine uniqueness. If a primary key is nil, return a new empty object to + # guarantee a unique value. We don't ever want to throw out records that have a nil primary key as these are + # new records. + primary_keys.map { |key| merge[key].nil? ? Object.new : merge[key] } end merges.reverse! end From d3576fdd21b4c88cadadb60dee2cc20085487583 Mon Sep 17 00:00:00 2001 From: Rodrigo Kochenburger Date: Fri, 11 Aug 2023 11:00:31 -0300 Subject: [PATCH 49/49] Revert to ruby-odbc for now --- odbc_adapter.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odbc_adapter.gemspec b/odbc_adapter.gemspec index 8960dfbb..1dd9546a 100644 --- a/odbc_adapter.gemspec +++ b/odbc_adapter.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.add_dependency 'activerecord', '>= 5.0.1' - spec.add_dependency 'odbc-ruby', '~> 1.0.1' + spec.add_dependency 'ruby-odbc', '~> 0.99998' spec.add_development_dependency 'bundler', '>= 1.14' spec.add_development_dependency 'minitest', '~> 5.10'