diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..e2b16fb --- /dev/null +++ b/.rspec @@ -0,0 +1,5 @@ +--color +--tty +--format progress +--order random +--backtrace diff --git a/Appraisals b/Appraisals new file mode 100644 index 0000000..d8a2da0 --- /dev/null +++ b/Appraisals @@ -0,0 +1,19 @@ +appraise 'rails3.1' do + gem 'activesupport', '~> 3.1.0' + gem 'activerecord', '~> 3.1.0' +end + +appraise 'rails3.2' do + gem 'activesupport', '~> 3.2.0' + gem 'activerecord', '~> 3.2.0' +end + +appraise 'rails4.0' do + gem 'activesupport', '~> 4.0.0' + gem 'activerecord', '~> 4.0.0' +end + +appraise 'rails4.1' do + gem 'activesupport', '~> 4.1.0' + gem 'activerecord', '~> 4.1.0' +end diff --git a/Gemfile b/Gemfile index f7dbef9..903d37f 100644 --- a/Gemfile +++ b/Gemfile @@ -1,4 +1,9 @@ source 'https://rubygems.org' # Specify your gem's dependencies in redis_counters-dumpers.gemspec + +group :development, :test do + gem 'combustion', github: 'pat/combustion', ref: '7d0d24c3f36ce0eb336177fc493be0721bc26665' +end + gemspec diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6848953 --- /dev/null +++ b/Makefile @@ -0,0 +1,28 @@ +RAILS_ENV = test +BUNDLE = RAILS_ENV=${RAILS_ENV} bundle +BUNDLE_OPTIONS = -j 2 +RSPEC = rspec +APPRAISAL = appraisal + +all: test + +test: config/database bundler/install appraisal/install + ${BUNDLE} exec ${APPRAISAL} ${RSPEC} spec 2>&1 + +config/database: + touch spec/internal/config/database.yml + echo 'test:' >> spec/internal/config/database.yml + echo ' adapter: postgresql' >> spec/internal/config/database.yml + echo ' database: docker' >> spec/internal/config/database.yml + echo ' username: docker' >> spec/internal/config/database.yml + echo ' host: localhost' >> spec/internal/config/database.yml + echo ' min_messages: warning' >> spec/internal/config/database.yml + +bundler/install: + if ! gem list bundler -i > /dev/null; then \ + gem install bundler; \ + fi + ${BUNDLE} install ${BUNDLE_OPTIONS} + +appraisal/install: + ${BUNDLE} exec ${APPRAISAL} install diff --git a/gemfiles/.gitignore b/gemfiles/.gitignore new file mode 100644 index 0000000..71afd1c --- /dev/null +++ b/gemfiles/.gitignore @@ -0,0 +1 @@ +*.gemfile.lock diff --git a/gemfiles/rails3.1.gemfile b/gemfiles/rails3.1.gemfile new file mode 100644 index 0000000..9eaf294 --- /dev/null +++ b/gemfiles/rails3.1.gemfile @@ -0,0 +1,12 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "activesupport", "~> 3.1.0" +gem "activerecord", "~> 3.1.0" + +group :development, :test do + gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665" +end + +gemspec :path => "../" diff --git a/gemfiles/rails3.2.gemfile b/gemfiles/rails3.2.gemfile new file mode 100644 index 0000000..f2cb4be --- /dev/null +++ b/gemfiles/rails3.2.gemfile @@ -0,0 +1,12 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "activesupport", "~> 3.2.0" +gem "activerecord", "~> 3.2.0" + +group :development, :test do + gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665" +end + +gemspec :path => "../" diff --git a/gemfiles/rails4.0.gemfile b/gemfiles/rails4.0.gemfile new file mode 100644 index 0000000..f283e37 --- /dev/null +++ b/gemfiles/rails4.0.gemfile @@ -0,0 +1,12 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "activesupport", "~> 4.0.0" +gem "activerecord", "~> 4.0.0" + +group :development, :test do + gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665" +end + +gemspec :path => "../" diff --git a/gemfiles/rails4.1.gemfile b/gemfiles/rails4.1.gemfile new file mode 100644 index 0000000..4390573 --- /dev/null +++ b/gemfiles/rails4.1.gemfile @@ -0,0 +1,12 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "activesupport", "~> 4.1.0" +gem "activerecord", "~> 4.1.0" + +group :development, :test do + gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665" +end + +gemspec :path => "../" diff --git a/lib/redis_counters/dumpers.rb b/lib/redis_counters/dumpers.rb index 9ad69ab..8b02c3c 100644 --- a/lib/redis_counters/dumpers.rb +++ b/lib/redis_counters/dumpers.rb @@ -1,3 +1,4 @@ +require 'active_record' require 'redis_counters/dumpers/version' module RedisCounters diff --git a/lib/redis_counters/dumpers/destination.rb b/lib/redis_counters/dumpers/destination.rb index 6fd2d35..43465a3 100644 --- a/lib/redis_counters/dumpers/destination.rb +++ b/lib/redis_counters/dumpers/destination.rb @@ -1,6 +1,7 @@ # coding: utf-8 require 'forwardable' require 'active_support/core_ext/hash/indifferent_access' +require 'active_support/core_ext/object/blank' require_relative 'dsl/destination' module RedisCounters @@ -48,6 +49,9 @@ class Destination # а целевое поле :date, указывает на поле :start_month_date, дампера. attr_accessor :fields_map + # Список полей по которым будет группироваться таблицы с исходными данным, Array + attr_accessor :group_by + # Список дополнительных условий, которые применяются при обновлении целевой таблицы, Array of String. # Каждое условие представляет собой строку - часть SQL выражения, которое может включать именованные # параметры из числа доступных в хеше оббщих параметров дампера: engine.common_params. @@ -68,7 +72,8 @@ def merge source AS ( SELECT #{selected_fields_expression} - FROM #{source_table} + FROM #{source_table} + #{group_by_expression} ), updated AS ( @@ -83,11 +88,11 @@ def merge INSERT INTO #{target_table} (#{target_fields}) SELECT #{target_fields} FROM source - WHERE NOT EXISTS ( - SELECT 1 + WHERE NOT EXISTS ( + SELECT 1 FROM updated target - WHERE #{matching_expression} - #{extra_conditions} + WHERE #{matching_expression} + #{extra_conditions} ) SQL @@ -105,6 +110,11 @@ def selected_fields_expression full_fields_map.map { |target_field, source_field| "#{source_field} as #{target_field}" }.join(', ') end + def group_by_expression + return if group_by.blank? + 'GROUP BY %s' % [group_by.join(', ')] + end + def full_fields_map fields_map.reverse_merge(Hash[fields.zip(fields)]) end diff --git a/lib/redis_counters/dumpers/dsl/destination.rb b/lib/redis_counters/dumpers/dsl/destination.rb index 599733e..be2ad70 100644 --- a/lib/redis_counters/dumpers/dsl/destination.rb +++ b/lib/redis_counters/dumpers/dsl/destination.rb @@ -17,6 +17,7 @@ class Configuration < ::RedisCounters::Dumpers::Dsl::Base varags_setter :fields varags_setter :key_fields varags_setter :increment_fields + varags_setter :group_by alias_method :take, :fields diff --git a/lib/redis_counters/dumpers/engine.rb b/lib/redis_counters/dumpers/engine.rb index 9dc2585..3f54c6c 100644 --- a/lib/redis_counters/dumpers/engine.rb +++ b/lib/redis_counters/dumpers/engine.rb @@ -4,6 +4,7 @@ require 'active_support/core_ext/hash/indifferent_access' require 'redis' require 'redis/namespace' +require 'redis_counters' require_relative 'dsl/engine' module RedisCounters @@ -179,6 +180,8 @@ def merge_data destinations.each { |dest| dest.merge } fire_callback(:on_after_merge, self, db_connection) + + drop_temp_table end def fill_temp_table @@ -245,6 +248,10 @@ def create_temp_table SQL end + def drop_temp_table + db_connection.execute "DROP TABLE #{temp_table_name}" + end + def analyze_table db_connection.execute <<-SQL ANALYZE #{temp_table_name} diff --git a/redis_counters-dumpers.gemspec b/redis_counters-dumpers.gemspec index 79488e9..22ca275 100644 --- a/redis_counters-dumpers.gemspec +++ b/redis_counters-dumpers.gemspec @@ -19,10 +19,19 @@ Gem::Specification.new do |spec| spec.add_dependency 'activesupport', '>= 3.0' spec.add_dependency 'activerecord', '>= 3.0' + spec.add_dependency 'pg' spec.add_dependency 'redis', '>= 3.0' spec.add_dependency 'redis-namespace', '>= 1.3' spec.add_dependency 'callbacks_rb', '>= 0.0.1' + spec.add_dependency 'redis_counters', '>= 1.3' + spec.add_development_dependency 'bundler', '>= 1.7' spec.add_development_dependency 'rake', '>= 10.0' + spec.add_development_dependency 'rspec', '>= 3.2' + spec.add_development_dependency 'rspec-rails', '>= 3.2' + spec.add_development_dependency 'rspec-given', '>= 3.5' + spec.add_development_dependency 'appraisal', '>= 1.0.2' + spec.add_development_dependency 'mock_redis' + spec.add_development_dependency 'apress-changelogger' end diff --git a/spec/internal/app/models/stats_agg_total.rb b/spec/internal/app/models/stats_agg_total.rb new file mode 100644 index 0000000..34d2e70 --- /dev/null +++ b/spec/internal/app/models/stats_agg_total.rb @@ -0,0 +1,2 @@ +class StatsAggTotal < ActiveRecord::Base +end diff --git a/spec/internal/app/models/stats_by_day.rb b/spec/internal/app/models/stats_by_day.rb new file mode 100644 index 0000000..f0a71c1 --- /dev/null +++ b/spec/internal/app/models/stats_by_day.rb @@ -0,0 +1,2 @@ +class StatsByDay < ActiveRecord::Base +end diff --git a/spec/internal/app/models/stats_total.rb b/spec/internal/app/models/stats_total.rb new file mode 100644 index 0000000..2ada32c --- /dev/null +++ b/spec/internal/app/models/stats_total.rb @@ -0,0 +1,2 @@ +class StatsTotal < ActiveRecord::Base +end diff --git a/spec/internal/config/.gitignore b/spec/internal/config/.gitignore new file mode 100644 index 0000000..b5649dd --- /dev/null +++ b/spec/internal/config/.gitignore @@ -0,0 +1 @@ +database.yml diff --git a/spec/internal/db/schema.rb b/spec/internal/db/schema.rb new file mode 100644 index 0000000..90f408c --- /dev/null +++ b/spec/internal/db/schema.rb @@ -0,0 +1,25 @@ +ActiveRecord::Schema.define do + create_table :stats_by_days do |t| + t.integer :record_id, null: false + t.integer :column_id, null: false + t.date :date, null: false + t.integer :hits, null: false, default: 0 + end + + add_index :stats_by_days, [:record_id, :column_id, :date], unique: true + + create_table :stats_totals do |t| + t.integer :record_id, null: false + t.integer :column_id, null: false + t.integer :hits, null: false, default: 0 + end + + add_index :stats_totals, [:record_id, :column_id], unique: true + + create_table :stats_agg_totals do |t| + t.integer :record_id, null: false + t.integer :hits, null: false, default: 0 + end + + add_index :stats_agg_totals, [:record_id], unique: true +end diff --git a/spec/internal/log/.gitignore b/spec/internal/log/.gitignore new file mode 100644 index 0000000..397b4a7 --- /dev/null +++ b/spec/internal/log/.gitignore @@ -0,0 +1 @@ +*.log diff --git a/spec/lib/redis_counters/dumpers/engine_spec.rb b/spec/lib/redis_counters/dumpers/engine_spec.rb new file mode 100644 index 0000000..3859cd2 --- /dev/null +++ b/spec/lib/redis_counters/dumpers/engine_spec.rb @@ -0,0 +1,97 @@ +require 'spec_helper' + +describe RedisCounters::Dumpers::Engine do + let(:dumper) do + RedisCounters::Dumpers::Engine.build do + name :stats_totals + fields record_id: :integer, + column_id: :integer, + value: :integer, + date: :date + + destination do + model StatsByDay + take :record_id, :column_id, :hits, :date + key_fields :record_id, :column_id, :date + increment_fields :hits + map :hits, to: :value + condition 'target.date = :date' + end + + destination do + model StatsTotal + take :record_id, :column_id, :hits + key_fields :record_id, :column_id + increment_fields :hits + map :hits, to: :value + end + + destination do + model StatsAggTotal + take :record_id, :hits + key_fields :record_id + increment_fields :hits + map :hits, to: 'sum(value)' + group_by :record_id + end + + on_before_merge do |dumper, _connection| + dumper.common_params = {date: dumper.date.strftime('%Y-%m-%d')} + end + end + end + + let(:prev_date) { Date.new(2015, 1, 19) } + let(:prev_date_s) { prev_date.strftime('%Y-%m-%d') } + + let(:date) { Date.new(2015, 1, 20) } + let(:date_s) { date.strftime('%Y-%m-%d') } + + let(:counter) do + RedisCounters.create_counter(Redis.current, + counter_class: RedisCounters::HashCounter, + counter_name: :record_hits_by_day, + group_keys: [:record_id, :column_id], + partition_keys: [:date] + ) + end + + before do + allow(dumper).to receive(:redis_session).and_return(MockRedis.new) + end + + describe '#process!' do + before do + counter.increment(date: prev_date_s, record_id: 1, column_id: 100) + counter.increment(date: prev_date_s, record_id: 1, column_id: 200) + counter.increment(date: prev_date_s, record_id: 1, column_id: 200) + counter.increment(date: prev_date_s, record_id: 2, column_id: 100) + + dumper.process!(counter, prev_date) + + counter.increment(date: date_s, record_id: 1, column_id: 100) + counter.increment(date: date_s, record_id: 1, column_id: 200) + counter.increment(date: date_s, record_id: 1, column_id: 200) + counter.increment(date: date_s, record_id: 2, column_id: 100) + + dumper.process!(counter, date) + end + + Then { expect(StatsByDay.count).to eq 6 } + And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 } + And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: prev_date).first.hits).to eq 2 } + And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 } + And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 } + And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: date).first.hits).to eq 2 } + And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 } + + And { expect(StatsTotal.count).to eq 3 } + And { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 } + And { expect(StatsTotal.where(record_id: 1, column_id: 200).first.hits).to eq 4 } + And { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 } + + And { expect(StatsAggTotal.count).to eq 2 } + And { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 6 } + And { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 } + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..9ab4c25 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,18 @@ +# coding: utf-8 +require 'bundler/setup' +require 'redis_counters/dumpers' + +require 'combustion' +Combustion.initialize! :active_record + +require 'rspec/rails' +require 'rspec/given' + +require 'mock_redis' +require 'redis' +Redis.current = MockRedis.new + +RSpec.configure do |config| + config.use_transactional_fixtures = true + config.before { Redis.current.flushdb } +end