Big refactoring including:

  * Fix issue with `time` type
  * Fix issue with negative numerics
  * Remove support for `send_at_once`, it is now synchronous by default
  * Use `copy_data` block from pg
  * Use `sync_put_copy_data` from pg and flush at the end
  * Add support for `geometry` and `geogrphy`
  * Add integrations tests using a Dummy rails App (See [spec/dummy](spec/dummy))
  * Remove the unused `use_tempfile` options from `EncodeForCopy`
  * Add explicit dependency to the [pg](https://rubygems.org/gems/pg/versions/1.3.4) gem, with a version minimum to 1.3.0 as we are using the `sync_put_copy_data` method

.env

@@ -0,0 +1 @@
+DATABASE_URL='postgis://postgres:postgres@localhost:5577'

.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+spec/dummy/log

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## Latest 2022-03-16
+
+* Big refactoring including:
+  * Support for mac address
+  * Fix issue with `time` type
+  * Fix issue with negative numerics
+  * Remove support for `send_at_once`, it is now synchronous by default
+  * Use `copy_data` block from pg
+  * Use `sync_put_copy_data` from pg and flush at the end
+  * Add support for `geometry` and `geogrphy`
+  * Add integrations tests using a Dummy rails App (See [spec/dummy](spec/dummy))
+  * Remove the unused `use_tempfile` options from `EncodeForCopy`
+  * Add explicit dependency to the [pg](https://rubygems.org/gems/pg/versions/1.3.4) gem, with a version minimum to 1.3.0 as we are using the `sync_put_copy_data` method
+
 ## 1.1.0       2018-05-24
 
 * Add support for range data types

README.md

@@ -43,6 +43,15 @@ MyModel.find_by(field_1: 'abc')
 Credits to [Pete Brumm](https://github.com/pbrumm) who wrote pg_data_encoder and
 which this library repurposes.
 
+## Developing
+
+```
+./start-local-db.sh
+cd spec/dummy
+bundle install
+DATABASE_URL='postgis://postgres:postgres@localhost:5577' rake db:create db:migrate db:test:prepare
+```
+
 ## LICENSE
 
 Copyright (c) 2018, Lukas Fittl <[email protected]><br>

activerecord-copy.gemspec

@@ -19,8 +19,14 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency('activerecord', '>= 3.1')
+  gem.add_dependency('pg', '>= 1.3.0')
 
   gem.add_development_dependency('rspec', '>= 2.12.0')
   gem.add_development_dependency('rspec-core', '>= 2.12.0')
+  gem.add_development_dependency('rspec-rails')
+
+  gem.add_development_dependency('rails', '>= 6.1.4.4')
+  gem.add_development_dependency('activerecord-postgis-adapter', '~> 7.1')
+  gem.add_development_dependency('dotenv-rails', '>= 2.7.6')
   gem.add_development_dependency('rgeo', '>= 2.4.0')
 end

docker-compose.yml

@@ -0,0 +1,11 @@
+version: "3"
+services:
+  db:
+    image: postgres/postgres
+    ports:
+      - "5577:5432"
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_HOST_AUTH_METHOD: "trust"
+

lib/activerecord-copy.rb

@@ -1,124 +1,31 @@
-require 'activerecord-copy/version'
+require 'json'
+require 'active_record'
 
+require 'activerecord-copy/version'
 require 'activerecord-copy/constants'
 require 'activerecord-copy/exception'
 require 'activerecord-copy/temp_buffer'
-
 require 'activerecord-copy/encode_for_copy'
-require 'activerecord-copy/decoder'
-
-require 'json'
-
-require 'active_record'
+require 'activerecord-copy/column_helper'
 
 module ActiveRecordCopy
   module CopyFromClient
     extend ActiveSupport::Concern
 
-    class CopyHandler
-      def initialize(columns:, model_class:, table_name:, send_at_once:)
-        @columns      = columns
-        @model_class  = model_class
-        @connection   = model_class.connection.raw_connection
-        @table_name   = table_name
-        @send_at_once = send_at_once
-        @column_types = columns.map do |c|
-          column = model_class.columns_hash[c.to_s]
-          raise format('Could not find column %s on %s', c, model_class.table_name) if column.nil?
-
-          if column.type == :integer
-            if column.limit == 8
-              :bigint
-            elsif column.limit == 2
-              :smallint
-            else
-              :integer
-            end
-          elsif column.sql_type == 'real'
-            :real
-          else
-            column.type
-          end
-        end
-      end
-
-      def <<(row)
-        start_copy_if_needed
-        @encoder.add row
-        unless @send_at_once
-          @connection.put_copy_data(@encoder.get_intermediate_io)
-        end
-      end
-
-      def copy(&_block)
-        reset
-
-        if @send_at_once
-          yield(self)
-          run_copy_at_once
-          return
-        end
-
-        begin
-          yield(self)
-        rescue Exception => err
-          if @copy_initialized
-            errmsg = format('%s while copy data: %s', err.class.name, err.message)
-            @connection.put_copy_end(errmsg)
-            @connection.get_result
-          end
-          raise
-        else
-          if @copy_initialized
-            @encoder.remove # writes the end marker
-            @connection.put_copy_end
-            @connection.get_last_result
-          end
-        end
-      end
-
-      private
-
-      def start_copy_if_needed
-        return if @copy_initialized || @send_at_once
-
-        @connection.exec(copy_sql)
-        @copy_initialized = true
-      end
-
-      def copy_sql
-        %{COPY #{@table_name}("#{@columns.join('","')}") FROM STDIN BINARY}
-      end
-
-      def run_copy_at_once
-        io = @encoder.get_io
-
-        @connection.copy_data(copy_sql) do
-          begin
-            while chunk = io.readpartial(10_240) # rubocop:disable Lint/AssignmentInCondition
-              @connection.put_copy_data chunk
-            end
-          rescue EOFError # rubocop:disable Lint/HandleExceptions
-          end
-        end
-
-        @encoder.remove
-
-        nil
-      end
-
-      def reset
-        @encoder = ActiveRecordCopy::EncodeForCopy.new column_types: @column_types
-        @copy_initialized = false
-      end
-    end
-
     class_methods do
-      def copy_from_client(columns, table_name: nil, send_at_once: false, &block)
+      def copy_from_client(columns, table_name: nil, &block)
         table_name ||= quoted_table_name
-        handler = CopyHandler.new(columns: columns, model_class: self, table_name: table_name, send_at_once: send_at_once)
-        handler.copy(&block)
-        true
+        connection = self.connection.raw_connection
+        column_types = columns.map do |c|
+          column = self.columns_hash[c.to_s]
+          raise format('Could not find column %s on %s', c, self.table_name) if column.nil?
+          ColumnHelper.find_column_type(column)
+        end
+        sql = %{COPY #{table_name}("#{columns.join('","')}") FROM STDIN BINARY}
+        encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: column_types, connection: connection)
+        connection.copy_data(sql)  do
+          encoder.process(&block)
+        end
       end
     end
   end

lib/activerecord-copy/column_helper.rb

@@ -0,0 +1,19 @@
+module ActiveRecordCopy
+  class ColumnHelper
+    def self.find_column_type(column)
+      if column.type == :integer
+        if column.limit == 8
+          :bigint
+        elsif column.limit == 2
+          :smallint
+        else
+          :integer
+        end
+      elsif column.sql_type == 'real'
+        :real
+      else
+        column.type
+      end
+    end
+  end
+end