add group_by method

.rspec

@@ -0,0 +1,5 @@
+--color
+--tty
+--format progress
+--order random
+--backtrace

Appraisals

@@ -0,0 +1,19 @@
+appraise 'rails3.1' do
+  gem 'activesupport', '~> 3.1.0'
+  gem 'activerecord', '~> 3.1.0'
+end
+
+appraise 'rails3.2' do
+  gem 'activesupport', '~> 3.2.0'
+  gem 'activerecord', '~> 3.2.0'
+end
+
+appraise 'rails4.0' do
+  gem 'activesupport', '~> 4.0.0'
+  gem 'activerecord', '~> 4.0.0'
+end
+
+appraise 'rails4.1' do
+  gem 'activesupport', '~> 4.1.0'
+  gem 'activerecord', '~> 4.1.0'
+end

Gemfile

@@ -1,4 +1,9 @@
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in redis_counters-dumpers.gemspec
+
+group :development, :test do
+  gem 'combustion', github: 'pat/combustion', ref: '7d0d24c3f36ce0eb336177fc493be0721bc26665'
+end
+
 gemspec

Makefile

@@ -0,0 +1,28 @@
+RAILS_ENV = test
+BUNDLE = RAILS_ENV=${RAILS_ENV} bundle
+BUNDLE_OPTIONS = -j 2
+RSPEC = rspec
+APPRAISAL = appraisal
+
+all: test
+
+test: config/database bundler/install appraisal/install
+	${BUNDLE} exec ${APPRAISAL} ${RSPEC} spec 2>&1
+
+config/database:
+	touch spec/internal/config/database.yml
+	echo 'test:' >> spec/internal/config/database.yml
+	echo '  adapter: postgresql' >> spec/internal/config/database.yml
+	echo '  database: docker' >> spec/internal/config/database.yml
+	echo '  username: docker' >> spec/internal/config/database.yml
+	echo '  host: localhost' >> spec/internal/config/database.yml
+	echo '  min_messages: warning' >> spec/internal/config/database.yml
+
+bundler/install:
+	if ! gem list bundler -i > /dev/null; then \
+	  gem install bundler; \
+	fi
+	${BUNDLE} install ${BUNDLE_OPTIONS}
+
+appraisal/install:
+	${BUNDLE} exec ${APPRAISAL} install

gemfiles/.gitignore

@@ -0,0 +1 @@
+*.gemfile.lock

gemfiles/rails3.1.gemfile

@@ -0,0 +1,12 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 3.1.0"
+gem "activerecord", "~> 3.1.0"
+
+group :development, :test do
+  gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
+end
+
+gemspec :path => "../"

gemfiles/rails3.2.gemfile

@@ -0,0 +1,12 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 3.2.0"
+gem "activerecord", "~> 3.2.0"
+
+group :development, :test do
+  gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
+end
+
+gemspec :path => "../"

gemfiles/rails4.0.gemfile

@@ -0,0 +1,12 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 4.0.0"
+gem "activerecord", "~> 4.0.0"
+
+group :development, :test do
+  gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
+end
+
+gemspec :path => "../"

gemfiles/rails4.1.gemfile

@@ -0,0 +1,12 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 4.1.0"
+gem "activerecord", "~> 4.1.0"
+
+group :development, :test do
+  gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
+end
+
+gemspec :path => "../"

lib/redis_counters/dumpers.rb

@@ -1,3 +1,4 @@
+require 'active_record'
 require 'redis_counters/dumpers/version'
 
 module RedisCounters

lib/redis_counters/dumpers/destination.rb

@@ -1,6 +1,7 @@
 # coding: utf-8
 require 'forwardable'
 require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/object/blank'
 require_relative 'dsl/destination'
 
 module RedisCounters
@@ -48,6 +49,9 @@ class Destination
       # а целевое поле :date, указывает на поле :start_month_date, дампера.
       attr_accessor :fields_map
 
+      # Список полей по которым будет группироваться таблицы с исходными данным, Array
+      attr_accessor :group_by
+
       # Список дополнительных условий, которые применяются при обновлении целевой таблицы, Array of String.
       # Каждое условие представляет собой строку - часть SQL выражения, которое может включать именованные
       # параметры из числа доступных в хеше оббщих параметров дампера: engine.common_params.
@@ -68,7 +72,8 @@ def merge
             source AS
             (
               SELECT #{selected_fields_expression}
-                FROM #{source_table}
+              FROM #{source_table}
+              #{group_by_expression}
             ),
             updated AS
             (
@@ -83,11 +88,11 @@ def merge
           INSERT INTO #{target_table} (#{target_fields})
             SELECT #{target_fields}
             FROM source
-          WHERE NOT EXISTS (
-            SELECT 1
+            WHERE NOT EXISTS (
+              SELECT 1
               FROM updated target
-            WHERE #{matching_expression}
-              #{extra_conditions}
+              WHERE #{matching_expression}
+                #{extra_conditions}
           )
         SQL
 
@@ -105,6 +110,11 @@ def selected_fields_expression
         full_fields_map.map { |target_field, source_field| "#{source_field} as #{target_field}" }.join(', ')
       end
 
+      def group_by_expression
+        return if group_by.blank?
+        'GROUP BY %s' % [group_by.join(', ')]
+      end
+
       def full_fields_map
         fields_map.reverse_merge(Hash[fields.zip(fields)])
       end

lib/redis_counters/dumpers/dsl/destination.rb

@@ -17,6 +17,7 @@ class Configuration < ::RedisCounters::Dumpers::Dsl::Base
           varags_setter :fields
           varags_setter :key_fields
           varags_setter :increment_fields
+          varags_setter :group_by
 
           alias_method :take, :fields
 

lib/redis_counters/dumpers/engine.rb

@@ -4,6 +4,7 @@
 require 'active_support/core_ext/hash/indifferent_access'
 require 'redis'
 require 'redis/namespace'
+require 'redis_counters'
 require_relative 'dsl/engine'
 
 module RedisCounters
@@ -179,6 +180,8 @@ def merge_data
         destinations.each { |dest| dest.merge }
 
         fire_callback(:on_after_merge, self, db_connection)
+
+        drop_temp_table
       end
 
       def fill_temp_table
@@ -245,6 +248,10 @@ def create_temp_table
         SQL
       end
 
+      def drop_temp_table
+        db_connection.execute "DROP TABLE #{temp_table_name}"
+      end
+
       def analyze_table
         db_connection.execute <<-SQL
           ANALYZE #{temp_table_name}

redis_counters-dumpers.gemspec

@@ -19,10 +19,19 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'activesupport', '>= 3.0'
   spec.add_dependency 'activerecord', '>= 3.0'
+  spec.add_dependency 'pg'
   spec.add_dependency 'redis', '>= 3.0'
   spec.add_dependency 'redis-namespace', '>= 1.3'
   spec.add_dependency 'callbacks_rb', '>= 0.0.1'
+  spec.add_dependency 'redis_counters', '>= 1.3'
+
 
   spec.add_development_dependency 'bundler', '>= 1.7'
   spec.add_development_dependency 'rake', '>= 10.0'
+  spec.add_development_dependency 'rspec', '>= 3.2'
+  spec.add_development_dependency 'rspec-rails', '>= 3.2'
+  spec.add_development_dependency 'rspec-given', '>= 3.5'
+  spec.add_development_dependency 'appraisal', '>= 1.0.2'
+  spec.add_development_dependency 'mock_redis'
+  spec.add_development_dependency 'apress-changelogger'
 end

spec/internal/app/models/stats_agg_total.rb

@@ -0,0 +1,2 @@
+class StatsAggTotal < ActiveRecord::Base
+end

spec/internal/app/models/stats_by_day.rb

@@ -0,0 +1,2 @@
+class StatsByDay < ActiveRecord::Base
+end

spec/internal/app/models/stats_total.rb

@@ -0,0 +1,2 @@
+class StatsTotal < ActiveRecord::Base
+end

spec/internal/config/.gitignore

@@ -0,0 +1 @@
+database.yml

spec/internal/db/schema.rb

@@ -0,0 +1,25 @@
+ActiveRecord::Schema.define do
+  create_table :stats_by_days do |t|
+    t.integer :record_id, null: false
+    t.integer :column_id, null: false
+    t.date :date, null: false
+    t.integer :hits, null: false, default: 0
+  end
+
+  add_index :stats_by_days, [:record_id, :column_id, :date], unique: true
+
+  create_table :stats_totals do |t|
+    t.integer :record_id, null: false
+    t.integer :column_id, null: false
+    t.integer :hits, null: false, default: 0
+  end
+
+  add_index :stats_totals, [:record_id, :column_id], unique: true
+
+  create_table :stats_agg_totals do |t|
+    t.integer :record_id, null: false
+    t.integer :hits, null: false, default: 0
+  end
+
+  add_index :stats_agg_totals, [:record_id], unique: true
+end

spec/internal/log/.gitignore

@@ -0,0 +1 @@
+*.log

spec/lib/redis_counters/dumpers/engine_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+describe RedisCounters::Dumpers::Engine do
+  let(:dumper) do
+    RedisCounters::Dumpers::Engine.build do
+      name :stats_totals
+      fields record_id: :integer,
+             column_id: :integer,
+             value: :integer,
+             date: :date
+
+      destination do
+        model StatsByDay
+        take :record_id, :column_id, :hits, :date
+        key_fields :record_id, :column_id, :date
+        increment_fields :hits
+        map :hits, to: :value
+        condition 'target.date = :date'
+      end
+
+      destination do
+        model StatsTotal
+        take :record_id, :column_id, :hits
+        key_fields :record_id, :column_id
+        increment_fields :hits
+        map :hits, to: :value
+      end
+
+      destination do
+        model StatsAggTotal
+        take :record_id, :hits
+        key_fields :record_id
+        increment_fields :hits
+        map :hits, to: 'sum(value)'
+        group_by :record_id
+      end
+
+      on_before_merge do |dumper, _connection|
+        dumper.common_params = {date: dumper.date.strftime('%Y-%m-%d')}
+      end
+    end
+  end
+
+  let(:prev_date) { Date.new(2015, 1, 19) }
+  let(:prev_date_s) { prev_date.strftime('%Y-%m-%d') }
+
+  let(:date) { Date.new(2015, 1, 20) }
+  let(:date_s) { date.strftime('%Y-%m-%d') }
+
+  let(:counter) do
+    RedisCounters.create_counter(Redis.current,
+      counter_class: RedisCounters::HashCounter,
+      counter_name: :record_hits_by_day,
+      group_keys: [:record_id, :column_id],
+      partition_keys: [:date]
+    )
+  end
+
+  before do
+    allow(dumper).to receive(:redis_session).and_return(MockRedis.new)
+  end
+
+  describe '#process!' do
+    before do
+      counter.increment(date: prev_date_s, record_id: 1, column_id: 100)
+      counter.increment(date: prev_date_s, record_id: 1, column_id: 200)
+      counter.increment(date: prev_date_s, record_id: 1, column_id: 200)
+      counter.increment(date: prev_date_s, record_id: 2, column_id: 100)
+
+      dumper.process!(counter, prev_date)
+
+      counter.increment(date: date_s, record_id: 1, column_id: 100)
+      counter.increment(date: date_s, record_id: 1, column_id: 200)
+      counter.increment(date: date_s, record_id: 1, column_id: 200)
+      counter.increment(date: date_s, record_id: 2, column_id: 100)
+
+      dumper.process!(counter, date)
+    end
+
+    Then { expect(StatsByDay.count).to eq 6 }
+    And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
+    And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: prev_date).first.hits).to eq 2 }
+    And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
+    And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
+    And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: date).first.hits).to eq 2 }
+    And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
+
+    And { expect(StatsTotal.count).to eq 3 }
+    And { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
+    And { expect(StatsTotal.where(record_id: 1, column_id: 200).first.hits).to eq 4 }
+    And { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
+
+    And { expect(StatsAggTotal.count).to eq 2 }
+    And { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 6 }
+    And { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
+  end
+end

spec/spec_helper.rb

@@ -0,0 +1,18 @@
+# coding: utf-8
+require 'bundler/setup'
+require 'redis_counters/dumpers'
+
+require 'combustion'
+Combustion.initialize! :active_record
+
+require 'rspec/rails'
+require 'rspec/given'
+
+require 'mock_redis'
+require 'redis'
+Redis.current = MockRedis.new
+
+RSpec.configure do |config|
+  config.use_transactional_fixtures = true
+  config.before { Redis.current.flushdb }
+end