diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..949fa6d --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +*.gem +*.rbc +.bundle +.config +.yardoc +Gemfile.lock +InstalledFiles +_yardoc +coverage +doc/ +lib/bundler/man +pkg +rdoc +spec/reports +test/tmp +test/version_tmp +tmp +.idea/ +.rbx/ +gemfiles/ \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..e69de29 diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..a0b6856 --- /dev/null +++ b/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' +source 'http://apress:montalcino@gems.railsc.ru' + +gemspec \ No newline at end of file diff --git a/README.md b/README.md index e69de29..f96bbd9 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,180 @@ +# RedisCounters [![Code Climate](https://codeclimate.com/repos/522e9b497e00a46a0d01227c/badges/ae868ca76e52852ebc5a/gpa.png)](https://codeclimate.com/repos/522e9b497e00a46a0d01227c/feed) [![CircleCI](https://circleci.com/gh/abak-press/class_logger.png?circle-token=e4d0ed5c60a5ff795bf971229addb871552c2750)](https://circleci.com/gh/abak-press/redis_counters) + +Набор структур данных на базе Redis. + +## RedisCounters::HashCounter + +Счетчик на основе Hash, с ~~преферансом и тайками-близняшками~~ партиционированием и группировкой значений. + +Обязательные параметры: counter_name, field_name или group_keys. + +### Сложность + + инкремент - O(1). + +### Примеры использования + +Простой счетчик значений. +```ruby +counter = RedisCounters::HashCounter.new(redis, { + :counter_name => :pages_by_day, + :field_name => :pages +}) + +5.times { counter.increment } + +redis: + pages_by_day = { + pages => 5 + } +``` + +Счетчик посещенных страниц компании с партиционированием по дате. +```ruby +counter = RedisCounters::HashCounter.new(redis, { + :counter_name => :pages_by_day, + :group_keys => [:company_id], + :partition_keys => [:date] +}) + +2.times { counter.increment(:company_id = 1, :date => '2013-08-01') } +3.times { counter.increment(:company_id = 2, :date => '2013-08-01') } +1.times { counter.increment(:company_id = 3, :date => '2013-08-02') } + +redis: + pages_by_day:2013-08-01 = { + 1 => 2 + 2 => 3 + } + pages_by_day:2013-08-02 = { + 3 => 1 + } +``` + +Тоже самое, но партиция задается с помощью proc. +```ruby +counter = RedisCounters::HashCounter.new(redis, { + :counter_name => :pages_by_day, + :group_keys => [:company_id], + :partition_keys => proc { |params| params.fetch(:date) } +}) +``` + +Счетчик посещенных страниц компании с группировкой по городу посетителя и партиционированием по дате. +```ruby +counter = RedisCounters::HashCounter.new(redis, { + :counter_name => :pages_by_day, + :group_keys => [:company_id, city_id], + :partition_keys => [:date] +}) + +2.times { counter.increment(:company_id = 1, :city_id => 11, :date => '2013-08-01') } +1.times { counter.increment(:company_id = 1, :city_id => 12, :date => '2013-08-01') } +3.times { counter.increment(:company_id = 2, :city_id => 11, :date => '2013-08-01') } + +redis: + pages_by_day:2013-08-01 = { + 1:11 => 2, + 1:12 => 1, + 2_11 => 3 + } +``` + +## RedisCounters::UniqueValuesList + +Список уникальных значений, с возможностью группировки и партиционирования значений. +Помимо списка значений, ведет так же, список партиций, для каждой группы. + +Обязательные параметры: counter_name и value_keys. + +### Сложность + + добавление элемента - от O(1), при отсутствии партиционирования, до O(N), где N - кол-во партиций. + +### Примеры использования + +Простой список уникальных пользователей. +```ruby +counter = RedisCounters::UniqueValuesList.new(redis, { + :counter_name => :users, + :value_keys => [:user_id] +}) + +counter.increment(:user_id => 1) +counter.increment(:user_id => 2) +counter.increment(:user_id => 1) + +redis: + users = ['1', '2'] +``` + +Список уникальных пользователей, посетивших компаниию, за месяц, сгруппированный по суткам. +```ruby +counter = RedisCounters::UniqueValuesList.new(redis, { + :counter_name => :company_users_by_month, + :value_keys => [:company_id, :user_id], + :group_keys => [:start_month_date], + :partition_keys => [:date] +}) + +2.times { counter.add(:company_id = 1, :user_id => 11, :date => '2013-08-10', :start_month_date => '2013-08-01') } +3.times { counter.add(:company_id = 1, :user_id => 22, :date => '2013-08-10', :start_month_date => '2013-08-01') } +3.times { counter.add(:company_id = 1, :user_id => 22, :date => '2013-09-05', :start_month_date => '2013-09-01') } +3.times { counter.add(:company_id = 2, :user_id => 11, :date => '2013-08-10', :start_month_date => '2013-08-01') } +1.times { counter.add(:company_id = 2, :user_id => 22, :date => '2013-08-11', :start_month_date => '2013-08-01') } + +redis: + company_users_by_month:2013-08-01:partitions = ['2013-08-10', '2013-08-11'] + company_users_by_month:2013-08-01:2013-08-10 = ['1:11', '1:22', '2:11'] + company_users_by_month:2013-08-01:2013-08-11 = ['2:22'] + + company_users_by_month:2013-09-01:partitions = ['2013-09-05'] + company_users_by_month:2013-09-01:2013-09-05 = ['1:22'] +``` + +## RedisCounters::UniqueHashCounter + +Структура на основе двух предыдущих. +HashCounter, с возможностью подсчета только у уникальных событий. + +### Сложность + аналогично UniqueValuesList. + +### Примеры использования + +Счетчик уникальных пользователей, посетивших компаниию, за месяц, сгруппированный по суткам. +```ruby +counter = RedisCounters::UniqueHashCounter.new(redis, { + :counter_name => :company_users_by_month, + :group_keys => [:company_id], + :partition_keys => [:date], + :unique_list => { + :value_keys => [:company_id, :user_id], + :group_keys => [:start_month_date], + :partition_keys => [:date] + } +}) + +2.times { counter.increment(:company_id = 1, :user_id => 11, :date => '2013-08-10', :start_month_date => '2013-08-01') } +3.times { counter.increment(:company_id = 1, :user_id => 22, :date => '2013-08-10', :start_month_date => '2013-08-01') } +3.times { counter.increment(:company_id = 1, :user_id => 22, :date => '2013-09-05', :start_month_date => '2013-09-01') } +3.times { counter.increment(:company_id = 2, :user_id => 11, :date => '2013-08-10', :start_month_date => '2013-08-01') } +1.times { counter.increment(:company_id = 2, :user_id => 22, :date => '2013-08-11', :start_month_date => '2013-08-01') } + +redis: + company_users_by_month:2013-08-10 = { + 1 = 2, + 2 = 1 + } + company_users_by_month:2013-08-11 = { + 2 = 1 + } + company_users_by_month:2013-09-05 = { + 1 = 1 + } + + company_users_by_month_uq:2013-08-01:partitions = ['2013-08-10', '2013-08-11'] + company_users_by_month_uq:2013-08-01:2013-08-10 = ['1:11', '1:22', '2:11'] + company_users_by_month_uq:2013-08-01:2013-08-11 = ['2:22'] + + company_users_by_month_uq:2013-09-01:partitions = ['2013-09-05'] + company_users_by_month_uq:2013-09-01:2013-09-05 = ['1:22'] +``` \ No newline at end of file diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..7599eea --- /dev/null +++ b/Rakefile @@ -0,0 +1,14 @@ +# coding: utf-8 +# load everything from tasks/ directory +Dir[File.join(File.dirname(__FILE__), 'tasks', '*.{rb,rake}')].each { |f| load(f) } + +task :build => [:check] +task :tag => :build + +desc 'Check if all projects are ready for build process' +task :check => [:audit, :quality, :coverage] + +require 'rspec/core/rake_task' + +# setup `spec` task +RSpec::Core::RakeTask.new(:spec) \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..309ec4f --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.0beta1 \ No newline at end of file diff --git a/lib/redis_counters.rb b/lib/redis_counters.rb new file mode 100644 index 0000000..cfe1d6c --- /dev/null +++ b/lib/redis_counters.rb @@ -0,0 +1,17 @@ +# encoding: utf-8 +require 'redis_counters/version' +require 'redis_counters/base_counter' +require 'redis_counters/hash_counter' +require 'redis_counters/unique_hash_counter' +require 'redis_counters/unique_values_list' + +require 'active_support/core_ext' + +module RedisCounters + + def create_counter(redis, opts) + BaseCounter.create(redis, opts) + end + + module_function :create_counter +end \ No newline at end of file diff --git a/lib/redis_counters/base_counter.rb b/lib/redis_counters/base_counter.rb new file mode 100644 index 0000000..bdafbae --- /dev/null +++ b/lib/redis_counters/base_counter.rb @@ -0,0 +1,44 @@ +# coding: utf-8 +require 'forwardable' + +module RedisCounters + + class BaseCounter + extend Forwardable + + KEY_DELIMITER = ':'.freeze + + attr_reader :redis + attr_reader :options + attr_reader :params + + def self.create(redis, opts) + counter_class = opts.fetch(:counter_class).to_s.constantize + counter_class.new(redis, opts) + end + + def initialize(redis, opts) + @redis = redis + @options = opts + init + end + + def process(params = {}, &block) + @params = params + process_value(&block) + end + + protected + + def init + counter_name.present? + end + + def counter_name + @counter_name ||= options.fetch(:counter_name) + end + + def_delegator :redis, :multi, :transaction + end + +end \ No newline at end of file diff --git a/lib/redis_counters/hash_counter.rb b/lib/redis_counters/hash_counter.rb new file mode 100644 index 0000000..677007c --- /dev/null +++ b/lib/redis_counters/hash_counter.rb @@ -0,0 +1,50 @@ +# coding: utf-8 +require 'redis_counters/base_counter' + +module RedisCounters + + class HashCounter < BaseCounter + alias_method :increment, :process + + def init + super + return if field_name.present? || group_keys.present? + raise ArgumentError, 'field_name or group_keys required!' + end + + protected + + def process_value + redis.hincrby(key, field, 1) + end + + def key + [counter_name, partition].flatten.join(KEY_DELIMITER) + end + + def partition + partition_keys.map do |key| + key.respond_to?(:call) ? key.call(params) : params.fetch(key) + end + end + + def field + group_params = group_keys.map { |key| params.fetch(key) } + group_params << field_name if field_name.present? + group_params.join(KEY_DELIMITER) + end + + def field_name + @field_name ||= options[:field_name] + end + + def group_keys + @group_keys ||= Array.wrap(options.fetch(:group_keys, [])) + end + + def partition_keys + @partition_keys ||= Array.wrap(options.fetch(:partition_keys, [])) + end + end + +end \ No newline at end of file diff --git a/lib/redis_counters/unique_hash_counter.rb b/lib/redis_counters/unique_hash_counter.rb new file mode 100644 index 0000000..8d9e6f6 --- /dev/null +++ b/lib/redis_counters/unique_hash_counter.rb @@ -0,0 +1,35 @@ +# coding: utf-8 +require 'redis_counters/hash_counter' +require 'redis_counters/unique_values_list' + +module RedisCounters + + class UniqueHashCounter < HashCounter + UNIQUE_LIST_POSTFIX = 'uq'.freeze + + protected + + def process_value + unique_values_list.add(params) { super } + end + + attr_reader :unique_values_list + + def init + super + @unique_values_list = UniqueValuesList.new( + redis, + unique_values_list_options + ) + end + + def unique_values_list_options + options.fetch(:unique_list).merge!(:counter_name => unique_values_list_name) + end + + def unique_values_list_name + [counter_name, UNIQUE_LIST_POSTFIX].join(KEY_DELIMITER) + end + end + +end \ No newline at end of file diff --git a/lib/redis_counters/unique_values_list.rb b/lib/redis_counters/unique_values_list.rb new file mode 100644 index 0000000..a1beee3 --- /dev/null +++ b/lib/redis_counters/unique_values_list.rb @@ -0,0 +1,122 @@ +# coding: utf-8 +require 'redis_counters/base_counter' + +module RedisCounters + + class UniqueValuesList < BaseCounter + PARTITIONS_LIST_POSTFIX = :partitions + + alias_method :add, :process + + protected + + def process_value + loop do + reset_partitions_cache + + watch_partitions_list + watch_all_partitions + + if value_already_exists? + redis.unwatch + return false + end + + result = transaction do + add_value + add_partition + yield redis if block_given? + end + + return true if result.present? + end + end + + def reset_partitions_cache + @partitions = nil + end + + def watch_partitions_list + return if use_default_partition? + redis.watch(partitions_list_key) + end + + def watch_all_partitions + partitions.each do |partition| + redis.watch(key(partition)) + end + end + + def value_already_exists? + partitions.reverse.any? do |partition| + redis.sismember(key(partition), value) + end + end + + def add_value + redis.sadd(key, value) + end + + def key(partition = partition_params) + [counter_name, group_params, partition].flatten.compact.join(KEY_DELIMITER) + end + + def group_params + group_keys.map { |key| params.fetch(key) } + end + + def partition_params + partition_keys.map { |key| params.fetch(key) } + end + + def value + value_params = value_keys.map { |key| params.fetch(key) } + value_params.join(KEY_DELIMITER) + end + + def partitions + return @partitions unless @partitions.nil? + return (@partitions = [nil]) if use_default_partition? + + @partitions = redis.smembers(partitions_list_key).map do |partition| + partition.split(KEY_DELIMITER) + end + .delete_if(&:empty?) + end + + def add_partition + return if use_default_partition? + return unless new_partition? + redis.sadd(partitions_list_key, current_partition) + end + + def partitions_list_key + [counter_name, group_params, PARTITIONS_LIST_POSTFIX].flatten.join(KEY_DELIMITER) + end + + def current_partition + partition_params.flatten.join(KEY_DELIMITER) + end + + def new_partition? + !partitions.include?(current_partition.split(KEY_DELIMITER)) + end + + def use_default_partition? + partition_keys.empty? + end + + def value_keys + @value_keys ||= Array.wrap(options.fetch(:value_keys)) + end + + def partition_keys + @partition_keys ||= Array.wrap(options.fetch(:partition_keys, [])) + end + + def group_keys + @group_keys ||= Array.wrap(options.fetch(:group_keys, [])) + end + end + +end \ No newline at end of file diff --git a/lib/redis_counters/version.rb b/lib/redis_counters/version.rb new file mode 100644 index 0000000..03ff4e4 --- /dev/null +++ b/lib/redis_counters/version.rb @@ -0,0 +1,3 @@ +module RedisCounters + VERSION = '1.0.0beta1' +end \ No newline at end of file diff --git a/redis_counters.gemspec b/redis_counters.gemspec new file mode 100644 index 0000000..233ec8f --- /dev/null +++ b/redis_counters.gemspec @@ -0,0 +1,41 @@ +# coding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'redis_counters/version' + +Gem::Specification.new do |gem| + gem.name = 'redis_counters' + gem.version = RedisCounters::VERSION + gem.authors = ['Artem Napolskih'] + gem.email = %w(napolskih@gmail.com) + gem.summary = %q{Redis Counters} + gem.homepage = 'https://github.com/abak-press/redis_counters' + + gem.files = `git ls-files`.split($/) + gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) } + gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) + gem.require_paths = ['lib'] + + gem.add_dependency 'activesupport', '>= 3.0' + + gem.add_development_dependency 'bundler' + gem.add_development_dependency 'rake' + gem.add_development_dependency 'rspec', '~> 2.14.0' + gem.add_development_dependency 'mock_redis' + gem.add_development_dependency 'timecop' + + # test coverage tool + gem.add_development_dependency 'simplecov' + + # code quality check + gem.add_development_dependency 'cane', '>= 2.6.0' + + # dependencies security tool + gem.add_development_dependency 'bundler-audit' + + # automatic changelog builder + gem.add_development_dependency 'changelogger' + + # a tool for uploading files to private gem repo + gem.add_development_dependency 'multipart-post' +end \ No newline at end of file diff --git a/spec/redis_counters/base_spec.rb b/spec/redis_counters/base_spec.rb new file mode 100644 index 0000000..8b56ce3 --- /dev/null +++ b/spec/redis_counters/base_spec.rb @@ -0,0 +1,29 @@ +require 'spec_helper' + +describe RedisCounters::BaseCounter do + let(:redis) { MockRedis.new } + + let(:options) { { + :counter_class => RedisCounters::HashCounter, + :counter_name => :counter_name, + :field_name => :field_name + } } + + let(:counter) { described_class.new(redis, options) } + + context '.create' do + it { expect(described_class.create(redis, options)).to be_a RedisCounters::HashCounter } + end + + context '#process' do + it { expect(described_class.create(redis, options)).to respond_to :process } + end + + context 'when counter_name not given' do + let(:options) { { + :field_name => :field_name + } } + + it { expect { counter }.to raise_error KeyError } + end +end \ No newline at end of file diff --git a/spec/redis_counters/hash_counter_spec.rb b/spec/redis_counters/hash_counter_spec.rb new file mode 100644 index 0000000..18fb60d --- /dev/null +++ b/spec/redis_counters/hash_counter_spec.rb @@ -0,0 +1,139 @@ +require 'spec_helper' + +describe RedisCounters::HashCounter do + let(:redis) { MockRedis.new } + let(:value) { rand(10) + 1 } + let(:options) { { :counter_name => :test_counter, :field_name => :test_field } } + let(:counter) { described_class.new(redis, options) } + + it { expect(counter).to respond_to :process } + it { expect(counter).to respond_to :increment } + + context 'when field_name and group_keys not given' do + let(:options) { { :counter_name => :test_counter } } + + it { expect { counter.process }.to raise_error ArgumentError } + end + + context 'when only field_name given' do + let(:options) { { + :counter_name => :test_counter, + :field_name => :test_field + } } + + before { value.times { counter.process } } + + it { expect(redis.keys('*')).to have(1).key } + it { expect(redis.keys('*').first).to eq 'test_counter' } + it { expect(redis.hexists('test_counter', 'test_field')).to be_true } + it { expect(redis.hget('test_counter', 'test_field')).to eq value.to_s } + end + + context 'when group_keys given' do + context 'when field_name not given' do + let(:options) { { + :counter_name => :test_counter, + :group_keys => [:param1, :param2] + } } + + before { value.times { counter.process(:param1 => 11, :param2 => 22, :param3 => 33) } } + + it { expect(redis.keys('*')).to have(1).key } + it { expect(redis.keys('*').first).to eq 'test_counter' } + it { expect(redis.hexists('test_counter', '11:22')).to be_true } + it { expect(redis.hget('test_counter', '11:22')).to eq value.to_s } + end + + context 'when exists group_keys given' do + let(:options) { { + :counter_name => :test_counter, + :field_name => :test_field, + :group_keys => [:param1, :param2] + } } + + before { value.times { counter.process(:param1 => 11, :param2 => 22, :param3 => 33) } } + before { 2.times { counter.process(:param1 => 12, :param2 => 22, :param3 => 33) } } + + it { expect(redis.keys('*')).to have(1).key } + it { expect(redis.keys('*').first).to eq 'test_counter' } + it { expect(redis.hexists('test_counter', '11:22:test_field')).to be_true } + it { expect(redis.hget('test_counter', '11:22:test_field')).to eq value.to_s } + it { expect(redis.hexists('test_counter', '12:22:test_field')).to be_true } + it { expect(redis.hget('test_counter', '12:22:test_field')).to eq 2.to_s } + end + + context 'when not exists group_keys given' do + let(:options) { { + :counter_name => :test_counter, + :field_name => :test_field, + :group_keys => [:param1, :param4] + } } + + it { expect { counter.process }.to raise_error KeyError } + end + end + + context 'when use partition' do + context 'when all partition_keys is Symbols' do + let(:options) { { + :counter_name => :test_counter, + :field_name => :test_field, + :partition_keys => [:param1, :param2] + } } + + before { value.times { counter.process(:param1 => 11, :param2 => 22, :param3 => 33) } } + before { 3.times { counter.process(:param1 => 21, :param2 => 22, :param3 => 33) } } + + it { expect(redis.keys('*')).to have(2).key } + it { expect(redis.keys('*').first).to eq 'test_counter:11:22' } + it { expect(redis.keys('*').last).to eq 'test_counter:21:22' } + it { expect(redis.hexists('test_counter:11:22', 'test_field')).to be_true } + it { expect(redis.hget('test_counter:11:22', 'test_field')).to eq value.to_s } + it { expect(redis.hexists('test_counter:21:22', 'test_field')).to be_true } + it { expect(redis.hget('test_counter:21:22', 'test_field')).to eq 3.to_s } + end + + context 'when all partition_keys is Proc' do + let(:options) { { + :counter_name => :test_counter, + :field_name => :test_field, + :partition_keys => proc { |params| params[:param1].odd?.to_s } + } } + + before { 2.times { counter.process(:param1 => 1, :param2 => 2) } } + before { 3.times { counter.process(:param1 => 2, :param2 => 2) } } + + it { expect(redis.keys('*')).to have(2).key } + it { expect(redis.keys('*').first).to eq 'test_counter:true' } + it { expect(redis.keys('*').last).to eq 'test_counter:false' } + it { expect(redis.hexists('test_counter:true', 'test_field')).to be_true } + it { expect(redis.hget('test_counter:true', 'test_field')).to eq 2.to_s } + it { expect(redis.hexists('test_counter:false', 'test_field')).to be_true } + it { expect(redis.hget('test_counter:false', 'test_field')).to eq 3.to_s } + end + + context 'when partition_keys consists of mixed types' do + let(:options) { { + :counter_name => :test_counter, + :field_name => :test_field, + :partition_keys => [:date, proc { |params| params[:param1].odd?.to_s }] + } } + + before { 2.times { counter.process(:param1 => 1, :param2 => 2, :date => '2013-04-27') } } + before { 1.times { counter.process(:param1 => 3, :param2 => 2, :date => '2013-04-27') } } + before { 4.times { counter.process(:param1 => 2, :param2 => 2, :date => '2013-04-27') } } + before { 1.times { counter.process(:param1 => 2, :param2 => 2, :date => '2013-04-28') } } + + it { expect(redis.keys('*')).to have(3).key } + it { expect(redis.keys('*').first).to eq 'test_counter:2013-04-27:true' } + it { expect(redis.keys('*').second).to eq 'test_counter:2013-04-27:false' } + it { expect(redis.keys('*').third).to eq 'test_counter:2013-04-28:false' } + it { expect(redis.hexists('test_counter:2013-04-27:true', 'test_field')).to be_true } + it { expect(redis.hget('test_counter:2013-04-27:true', 'test_field')).to eq 3.to_s } + it { expect(redis.hexists('test_counter:2013-04-27:false', 'test_field')).to be_true } + it { expect(redis.hget('test_counter:2013-04-27:false', 'test_field')).to eq 4.to_s } + it { expect(redis.hexists('test_counter:2013-04-27:false', 'test_field')).to be_true } + it { expect(redis.hget('test_counter:2013-04-28:false', 'test_field')).to eq 1.to_s } + end + end +end diff --git a/spec/redis_counters/unique_hash_counter_spec.rb b/spec/redis_counters/unique_hash_counter_spec.rb new file mode 100644 index 0000000..c9944f4 --- /dev/null +++ b/spec/redis_counters/unique_hash_counter_spec.rb @@ -0,0 +1,62 @@ +require 'spec_helper' + +describe RedisCounters::UniqueHashCounter do + let(:redis) { MockRedis.new } + let(:unique_list_postfix) { described_class.const_get(:UNIQUE_LIST_POSTFIX) } + + let(:options) { { + :counter_name => :test_counter, + :field_name => :test_field, + :unique_list => {} + } } + let(:counter) { described_class.new(redis, options) } + + it { expect(counter).to be_a_kind_of RedisCounters::HashCounter } + + context 'when unique_list not given' do + let(:options) { { + :counter_name => :test_counter, + :field_name => :test_field + } } + + it { expect { counter.process }.to raise_error KeyError } + end + + context 'when only partition_keys and partition_keys given' do + let(:options) { { + :counter_name => :test_counter, + :field_name => :test_field, + :group_keys => [:param1], + :partition_keys => [:date], + :unique_list => { + :value_keys => [:sid], + :group_keys => [:param2], + :partition_keys => [:date] + } + } } + + before { 2.times { counter.process(:param1 => 1, :param2 => 2, :date => '2013-04-27', :sid => 1) } } + before { 2.times { counter.process(:param1 => 1, :param2 => 2, :date => '2013-04-27', :sid => 2) } } + before { 2.times { counter.process(:param1 => 2, :param2 => 2, :date => '2013-04-27', :sid => 3) } } + before { 2.times { counter.process(:param1 => 2, :param2 => 1, :date => '2013-04-28', :sid => 1) } } + before { 2.times { counter.process(:param1 => 2, :param2 => 1, :date => '2013-04-28', :sid => 5) } } + before { 2.times { counter.process(:param1 => 2, :param2 => 1, :date => '2013-04-27', :sid => 4) } } + before { 2.times { counter.process(:param1 => 2, :param2 => 1, :date => '2013-04-27', :sid => 1) } } + before { 2.times { counter.process(:param1 => 2, :param2 => 1, :date => '2013-04-28', :sid => 4) } } + before { 2.times { counter.process(:param1 => 2, :param2 => 1, :date => '2013-04-28', :sid => 4) } } + before { 2.times { counter.process(:param1 => 2, :param2 => 1, :date => '2013-04-27', :sid => 5) } } + + it { expect(redis.keys('*')).to have(7).key } + + it { expect(redis.keys('*')).to include 'test_counter:2013-04-27' } + it { expect(redis.hget('test_counter:2013-04-27', '1:test_field')).to eq 2.to_s } + it { expect(redis.hget('test_counter:2013-04-27', '2:test_field')).to eq 2.to_s } + it { expect(redis.hget('test_counter:2013-04-28', '2:test_field')).to eq 2.to_s } + + it { expect(redis.smembers('test_counter:uq:1:partitions')).to eq ['2013-04-27', '2013-04-28'] } + it { expect(redis.smembers('test_counter:uq:2:partitions')).to eq ['2013-04-27'] } + it { expect(redis.smembers('test_counter:uq:1:2013-04-27')).to eq ['4'] } + it { expect(redis.smembers('test_counter:uq:2:2013-04-27')).to eq ['3', '2', '1'] } + it { expect(redis.smembers('test_counter:uq:1:2013-04-28')).to eq ['5', '1'] } + end +end \ No newline at end of file diff --git a/spec/redis_counters/unique_values_list_spec.rb b/spec/redis_counters/unique_values_list_spec.rb new file mode 100644 index 0000000..b31284e --- /dev/null +++ b/spec/redis_counters/unique_values_list_spec.rb @@ -0,0 +1,196 @@ +require 'spec_helper' + +describe RedisCounters::UniqueValuesList do + let(:redis) { MockRedis.new } + let(:values) { rand(10) + 1 } + let(:partitions_list_postfix) { described_class.const_get(:PARTITIONS_LIST_POSTFIX) } + + let(:counter) { described_class.new(redis, options) } + + context 'when value_keys not given' do + let(:options) { { + :counter_name => :test_counter + } } + + it { expect { counter.add }.to raise_error KeyError } + end + + context 'when unknown value_key given' do + let(:options) { { + :counter_name => :test_counter, + :value_keys => [:param0, :param1] + } } + + it { expect { counter.add(:param1 => 1) }.to raise_error KeyError } + end + + context 'when unknown group_key given' do + let(:options) { { + :counter_name => :test_counter, + :value_keys => [:param0], + :group_keys => [:param1, :param2], + } } + + it { expect { counter.add(:param0 => 1, :param1 => 2) }.to raise_error KeyError } + end + + context 'when unknown partition_key given' do + let(:options) { { + :counter_name => :test_counter, + :value_keys => [:param0], + :partition_keys => [:param1, :param2], + } } + + it { expect { counter.add(:param0 => 1, :param1 => 2) }.to raise_error KeyError } + end + + context 'when group and partition keys given' do + let(:options) { { + :counter_name => :test_counter, + :value_keys => [:param0, :param1], + :group_keys => [:param2], + :partition_keys => [:param3, :param4] + } } + + before { values.times { counter.process(:param0 => 1, :param1 => 2, :param2 => :group1, :param3 => :part1, :param4 => :part2) } } + before { values.times { counter.process(:param0 => 2, :param1 => 1, :param2 => :group1, :param3 => :part1, :param4 => :part2) } } + before { values.times { counter.process(:param0 => 3, :param1 => 2, :param2 => :group1, :param3 => :part2, :param4 => :part2) } } + before { values.times { counter.process(:param0 => 4, :param1 => 5, :param2 => :group2, :param3 => :part1, :param4 => :part2) } } + + it { expect(redis.keys('*')).to have(5).key } + + context 'when check partitions' do + it { expect(redis.exists("test_counter:group1:#{partitions_list_postfix}")).to be_true } + it { expect(redis.exists("test_counter:group2:#{partitions_list_postfix}")).to be_true } + + it { expect(redis.smembers("test_counter:group1:#{partitions_list_postfix}")).to have(2).keys } + it { expect(redis.smembers("test_counter:group2:#{partitions_list_postfix}")).to have(1).keys } + + it { expect(redis.smembers("test_counter:group1:#{partitions_list_postfix}")).to include 'part1:part2' } + it { expect(redis.smembers("test_counter:group1:#{partitions_list_postfix}")).to include 'part2:part2' } + it { expect(redis.smembers("test_counter:group2:#{partitions_list_postfix}")).to include 'part1:part2' } + end + + context 'when check values' do + it { expect(redis.exists("test_counter:group1:part1:part2")).to be_true } + it { expect(redis.exists("test_counter:group1:part2:part2")).to be_true } + it { expect(redis.exists("test_counter:group2:part1:part2")).to be_true } + + it { expect(redis.smembers("test_counter:group1:part1:part2")).to have(2).keys } + it { expect(redis.smembers("test_counter:group1:part2:part2")).to have(1).keys } + it { expect(redis.smembers("test_counter:group2:part1:part2")).to have(1).keys } + + it { expect(redis.smembers("test_counter:group1:part1:part2")).to include '1:2' } + it { expect(redis.smembers("test_counter:group1:part1:part2")).to include '2:1' } + it { expect(redis.smembers("test_counter:group1:part2:part2")).to include '3:2' } + it { expect(redis.smembers("test_counter:group2:part1:part2")).to include '4:5' } + end + end + + context 'when group and partition keys no given' do + let(:options) { { + :counter_name => :test_counter, + :value_keys => [:param0, :param1] + } } + + before { values.times { counter.process(:param0 => 1, :param1 => 2) } } + before { values.times { counter.process(:param0 => 1, :param1 => 2) } } + before { values.times { counter.process(:param0 => 2, :param1 => 1) } } + before { values.times { counter.process(:param0 => 3, :param1 => 2) } } + + it { expect(redis.keys('*')).to have(1).key } + + context 'when check values' do + it { expect(redis.exists("test_counter")).to be_true } + it { expect(redis.smembers("test_counter")).to have(3).keys } + + it { expect(redis.smembers("test_counter")).to include '1:2' } + it { expect(redis.smembers("test_counter")).to include '2:1' } + it { expect(redis.smembers("test_counter")).to include '3:2' } + end + end + + context 'when no group keys given, but partition keys given' do + let(:options) { { + :counter_name => :test_counter, + :value_keys => [:param0, :param1], + :partition_keys => [:param3, :param4] + } } + + before { values.times { counter.process(:param0 => 1, :param1 => 2, :param3 => :part1, :param4 => :part2) } } + before { values.times { counter.process(:param0 => 2, :param1 => 1, :param3 => :part1, :param4 => :part2) } } + before { values.times { counter.process(:param0 => 3, :param1 => 2, :param3 => :part2, :param4 => :part2) } } + before { values.times { counter.process(:param0 => 4, :param1 => 5, :param3 => :part1, :param4 => :part2) } } + + it { expect(redis.keys('*')).to have(3).key } + + context 'when check partitions' do + it { expect(redis.exists("test_counter:#{partitions_list_postfix}")).to be_true } + + it { expect(redis.smembers("test_counter:#{partitions_list_postfix}")).to have(2).keys } + + it { expect(redis.smembers("test_counter:#{partitions_list_postfix}")).to include 'part1:part2' } + it { expect(redis.smembers("test_counter:#{partitions_list_postfix}")).to include 'part2:part2' } + end + + context 'when check values' do + it { expect(redis.exists("test_counter:part1:part2")).to be_true } + it { expect(redis.exists("test_counter:part2:part2")).to be_true } + + it { expect(redis.smembers("test_counter:part1:part2")).to have(3).keys } + it { expect(redis.smembers("test_counter:part2:part2")).to have(1).keys } + + it { expect(redis.smembers("test_counter:part1:part2")).to include '1:2' } + it { expect(redis.smembers("test_counter:part1:part2")).to include '2:1' } + it { expect(redis.smembers("test_counter:part2:part2")).to include '3:2' } + it { expect(redis.smembers("test_counter:part1:part2")).to include '4:5' } + end + end + + context 'when group keys given, but partition keys not given' do + let(:options) { { + :counter_name => :test_counter, + :value_keys => [:param0, :param1], + :group_keys => [:param2] + } } + + before { values.times { counter.process(:param0 => 1, :param1 => 2, :param2 => :group1) } } + before { values.times { counter.process(:param0 => 2, :param1 => 1, :param2 => :group1) } } + before { values.times { counter.process(:param0 => 3, :param1 => 2, :param2 => :group1) } } + before { values.times { counter.process(:param0 => 4, :param1 => 5, :param2 => :group2) } } + + it { expect(redis.keys('*')).to have(2).key } + + context 'when check values' do + it { expect(redis.exists("test_counter:group1")).to be_true } + it { expect(redis.exists("test_counter:group2")).to be_true } + + it { expect(redis.smembers("test_counter:group1")).to have(3).keys } + it { expect(redis.smembers("test_counter:group2")).to have(1).keys } + + it { expect(redis.smembers("test_counter:group1")).to include '1:2' } + it { expect(redis.smembers("test_counter:group1")).to include '2:1' } + it { expect(redis.smembers("test_counter:group1")).to include '3:2' } + it { expect(redis.smembers("test_counter:group2")).to include '4:5' } + end + end + + context 'when block given' do + let(:options) { { + :counter_name => :test_counter, + :value_keys => [:param0] + } } + + context 'when item added' do + it { expect { |b| counter.process(:param0 => 1, &b) }.to yield_with_args(redis) } + it { expect(counter.process(:param0 => 1)).to be_true } + end + + context 'when item not added' do + before { counter.process(:param0 => 1) } + + it { expect { |b| counter.process(:param0 => 1, &b) }.to_not yield_with_args(redis) } + it { expect(counter.process(:param0 => 1)).to be_false } + end + end +end \ No newline at end of file diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..8c5984e --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,20 @@ +require 'rubygems' +require 'bundler/setup' +require 'rspec' +require 'simplecov' +require 'mock_redis' + +SimpleCov.start('test_frameworks') + +require 'redis_counters' + +# require helpers +support_dir = File.expand_path(File.join('..', 'support'), __FILE__) +Dir[File.join(support_dir, '**', '*.rb')].each { |f| require f } + +RSpec.configure do |config| + config.backtrace_exclusion_patterns = [/lib\/rspec\/(core|expectations|matchers|mocks)/] + config.color_enabled = true + config.formatter = 'documentation' + config.order = 'random' +end \ No newline at end of file diff --git a/tasks/audit.rake b/tasks/audit.rake new file mode 100644 index 0000000..0483dbd --- /dev/null +++ b/tasks/audit.rake @@ -0,0 +1,6 @@ +# coding: utf-8 + +desc 'Audit current gemset' +task :audit do + spawn 'bundle exec bundle-audit' +end \ No newline at end of file diff --git a/tasks/cane.rake b/tasks/cane.rake new file mode 100644 index 0000000..4248b23 --- /dev/null +++ b/tasks/cane.rake @@ -0,0 +1,12 @@ +# coding: utf-8 + +require 'cane/rake_task' + +desc 'Run cane to check quality metrics' +Cane::RakeTask.new(:quality) do |cane| + cane.abc_max = 15 + cane.abc_glob = cane.style_glob = cane.doc_glob = '*/{lib,bin}/**/*.rb' + cane.style_measure = 120 + cane.parallel = false + cane.no_doc = true +end \ No newline at end of file diff --git a/tasks/coverage.rake b/tasks/coverage.rake new file mode 100644 index 0000000..ffd34f0 --- /dev/null +++ b/tasks/coverage.rake @@ -0,0 +1,21 @@ +# coding: utf-8 + +MINIMUM_COVERAGE = 85 + +desc "Check if test coverage is equal or greater than %.2f%%" % MINIMUM_COVERAGE +task :coverage => :spec do + require 'simplecov' + require 'simplecov/exit_codes' + + covered_percent = SimpleCov.result.covered_percent.round(2) + if covered_percent < MINIMUM_COVERAGE + $stderr.puts "Coverage (%.2f%%) is below the expected minimum coverage (%.2f%%)." % \ + [covered_percent, MINIMUM_COVERAGE] + + exit(SimpleCov::ExitCodes::MINIMUM_COVERAGE) + end +end + +task :clean do + FileUtils.rm_rf 'coverage' +end \ No newline at end of file diff --git a/tasks/gem.rake b/tasks/gem.rake new file mode 100644 index 0000000..83c2239 --- /dev/null +++ b/tasks/gem.rake @@ -0,0 +1,92 @@ +# coding: utf-8 + +desc 'Release gem (build and upload to gem repo)' +task :release => [ + :ensure_master, + :build, + 'version:release', + :tag, + :push, + :upload +] + +desc 'Build project into pkg directory' +task :build do + FileUtils.mkdir_p('pkg') + + gemspec = "#{project}.gemspec" + spawn("gem build -V #{gemspec}") + built_gem_path = Dir["#{project}-*.gem"].sort_by{|f| File.mtime(f)}.last + + FileUtils.mv(built_gem_path, 'pkg') +end + +desc 'Mark project as stable with version tag' +task :tag do + tag_name = "v#{current_version}" + + spawn("git tag -a -m \"Version #{current_version}\" #{tag_name}") + puts "Tag #{tag_name} created" +end + +task :push do + spawn 'git push' + spawn 'git push --tags' +end + +# upload built tarballs to repo +task :upload do + require 'uri' + require 'net/http/post/multipart' + + repo = gems_sources.grep(/railsc/).first + uri = URI.parse(repo) + + tarball_name = "#{project}-#{current_version}.gem" + upload_gem(uri.dup, tarball_name) +end + +task :ensure_master do + `git rev-parse --abbrev-ref HEAD`.chomp.strip == 'master' || abort("Can be released only from `master` branch") +end + +def upload_gem(repo_uri, tarball_name) + require 'net/http/post/multipart' + repo_uri.path = '/upload' + + tarball_path = File.join('pkg', tarball_name) + + File.open(tarball_path) do |gem| + req = Net::HTTP::Post::Multipart.new repo_uri.path, + "file" => UploadIO.new(gem, "application/x-tar", tarball_name) + + req.basic_auth(repo_uri.user, repo_uri.password) if repo_uri.user + + res = Net::HTTP.start(repo_uri.host, repo_uri.port) do |http| + http.request(req) + end + + if [200, 302].include?(res.code.to_i) + puts "#{tarball_name} uploaded successfully" + else + $stderr.puts "Cannot upload #{tarball_name}. Response status: #{res.code}" + exit(1) + end + end # File.open +end + +task :clean do + FileUtils.rm_f 'Gemfile.lock' +end + +def gems_sources + Bundler. + setup. # get bundler runtime + specs. # for each spec + map(&:source). # get its sources + select { |v| Bundler::Source::Rubygems === v }. # fetch only rubygems-like repos + map(&:remotes). # get all remotes + flatten. + uniq. + map(&:to_s) +end \ No newline at end of file diff --git a/tasks/support.rb b/tasks/support.rb new file mode 100644 index 0000000..4507304 --- /dev/null +++ b/tasks/support.rb @@ -0,0 +1,40 @@ +# coding: utf-8 +# Helpers for root Rakefile + +require 'pty' + +ROOT = File.expand_path(File.join('..', '..'), __FILE__) + +# run +cmd+ in subprocess, redirect its stdout to parent's stdout +def spawn(cmd) + puts ">> #{cmd}" + + cmd += ' 2>&1' + PTY.spawn cmd do |r, w, pid| + begin + r.sync + r.each_char { |chr| STDOUT.write(chr) } + rescue Errno::EIO => e + # simply ignoring this + ensure + ::Process.wait pid + end + end + abort "#{cmd} failed" unless $? && $?.exitstatus == 0 +end + +def project + 'redis_counters' +end + +# get current version from VERSION file +def current_version + File.read(File.join(ROOT, 'VERSION')).strip.chomp +end + +# get released version from git +def released_version + /\Av([\d\.]+)\z/ === `git describe --tags --abbrev=0 2>/dev/null || echo 'v0.0.0'`.chomp.strip + + $1 +end \ No newline at end of file diff --git a/tasks/version.rake b/tasks/version.rake new file mode 100644 index 0000000..125238e --- /dev/null +++ b/tasks/version.rake @@ -0,0 +1,35 @@ +# coding: utf-8 +namespace :version do + task :current do + puts current_version + end + + desc 'Write a version from VERSION file to project lib/**/version.rb' + task :update do + Dir['lib/**/version.rb'].each do |file| + contents = File.read(file) + contents.gsub!(/VERSION\s*=\s*(['"])(.*?)\1/m, "VERSION = '#{current_version}'") + File.write(file, contents) + end + end + + desc 'Put version files to repo' + task :commit do + Dir['lib/**/version.rb'].each do |file| + spawn "git add #{file}" + end + # git diff --exit-code returns 0 if nothing was changed and 1 otherwise + spawn "git diff --cached --exit-code > /dev/null || git commit -m \"Release #{current_version}\" || echo -n" + end + + desc 'Release new version' + task :release => [:changelog, :update, :commit] + + desc 'Generate CHANGELOG file' + task :changelog do + changelog = File.join(ROOT, 'CHANGELOG') + + spawn "changelogger changelog '#{ROOT}' --top_version='v#{current_version}' > '#{changelog}'" + spawn "git add '#{changelog}'" + end +end \ No newline at end of file