From 8a07f2a3919e7384a7bda704db6f8eef8219794e Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Mon, 2 Nov 2020 13:02:41 +0300 Subject: [PATCH 01/20] first draft --- lib/table_sync/publishing/batch_pub.rb | 62 ++++++++++++++++ lib/table_sync/publishing/data/attributes.rb | 26 +++++++ lib/table_sync/publishing/data/base.rb | 61 ++++++++++++++++ lib/table_sync/publishing/data/batch.rb | 19 +++++ lib/table_sync/publishing/data/object.rb | 21 ++++++ lib/table_sync/publishing/data/raw.rb | 6 ++ lib/table_sync/publishing/message.rb | 63 ++++++++++++++++ lib/table_sync/publishing/message/base.rb | 51 +++++++++++++ lib/table_sync/publishing/message/batch.rb | 20 ++++++ .../publishing/message/find_objects.rb | 46 ++++++++++++ lib/table_sync/publishing/message/object.rb | 19 +++++ lib/table_sync/publishing/message/raw.rb | 33 +++++++++ lib/table_sync/publishing/params/base.rb | 71 +++++++++++++++++++ lib/table_sync/publishing/params/batch.rb | 18 +++++ lib/table_sync/publishing/params/object.rb | 31 ++++++++ lib/table_sync/publishing/pub.rb | 25 +++++++ 16 files changed, 572 insertions(+) create mode 100644 lib/table_sync/publishing/batch_pub.rb create mode 100644 lib/table_sync/publishing/data/attributes.rb create mode 100644 lib/table_sync/publishing/data/base.rb create mode 100644 lib/table_sync/publishing/data/batch.rb create mode 100644 lib/table_sync/publishing/data/object.rb create mode 100644 lib/table_sync/publishing/data/raw.rb create mode 100644 lib/table_sync/publishing/message.rb create mode 100644 lib/table_sync/publishing/message/base.rb create mode 100644 lib/table_sync/publishing/message/batch.rb create mode 100644 lib/table_sync/publishing/message/find_objects.rb create mode 100644 lib/table_sync/publishing/message/object.rb create mode 100644 lib/table_sync/publishing/message/raw.rb create mode 100644 lib/table_sync/publishing/params/base.rb create mode 100644 lib/table_sync/publishing/params/batch.rb create mode 100644 lib/table_sync/publishing/params/object.rb create mode 100644 lib/table_sync/publishing/pub.rb diff --git a/lib/table_sync/publishing/batch_pub.rb b/lib/table_sync/publishing/batch_pub.rb new file mode 100644 index 0000000..2e4fd84 --- /dev/null +++ b/lib/table_sync/publishing/batch_pub.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Publisher + include Tainbox + + attribute :klass + attribute :attrs + attribute :state + attribute :routing_key + attribute :headers + + attribute :raw, default: false + + # how necessary is serialization check? + def publish + job.perform_later(attributes) + end + + def publish_now + message.publish + end + + private + + # MESSAGE + + def message + raw ? raw_message : batch_message + end + + def batch_message + TableSync::Publishing::Message::Batch.new(**message_params) + end + + def raw_message + TableSync::Publishing::Message::Raw.new(**message_params) + end + + def message_params + attributes.slice( + :klass, :attrs, :state, :routing_key, :headers + ) + end + + # JOB + + def job + job_callable ? job_callable.call : raise job_callable_error_message + end + + def job_callable + TableSync.batch_publishing_job_class_callable + end + + def job_callable_error_message + "Can't publish, set TableSync.batch_publishing_job_class_callable" + end +end + +# Насколько нужно проверять сриализацию? Никто не пихает туда сложные объекты. + +# Не надо конфёрм. \ No newline at end of file diff --git a/lib/table_sync/publishing/data/attributes.rb b/lib/table_sync/publishing/data/attributes.rb new file mode 100644 index 0000000..b8f044e --- /dev/null +++ b/lib/table_sync/publishing/data/attributes.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Data::Attributes + include Tainbox + + attribute :object + attribute :destroy + + # Can't find object when destruction! + + def for_sync + destroy ? attributes_for_destroy : attributes_for_update + end + + private + + def attributes_for_destroy + object.try(:table_sync_destroy_attributes) || + TableSync.publishing_adapter.primary_key(object) + end + + def attributes_for_update + object.try(:attributes_for_sync) || + TableSync.publishing_adapter.attributes(object) + end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/data/base.rb b/lib/table_sync/publishing/data/base.rb new file mode 100644 index 0000000..0d4ab2b --- /dev/null +++ b/lib/table_sync/publishing/data/base.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Data::Base + include Tainbox + + attribute :state, default: :updated + + def construct + { + model: model, + attributes: attributes_for_sync, + version: version, + event: event, + metadata: metadata, + } + end + + private + + # MISC + + def model + klass.try(:table_sync_model_name) || klass.name + end + + def version + Time.current.to_f + end + + def metadata + {} + end + + # STATE, EVENT + + def destroyed? + state == :destroyed + end + + def created? + state == :created + end + + def event + destroyed? ? :destroy : :update + end + + # NOT IMPLEMENTED + + def klass + raise NotImplementedError + end + + def attributes_for_sync + raise NotImplementedError + end +end + +# def validate_state +# raise "Unknown state: #{state.inspect}" unless %i[created updated destroyed].include?(state) +# end \ No newline at end of file diff --git a/lib/table_sync/publishing/data/batch.rb b/lib/table_sync/publishing/data/batch.rb new file mode 100644 index 0000000..d231144 --- /dev/null +++ b/lib/table_sync/publishing/data/batch.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Data::Batch + attribute :objects + + private + + def klass + objects.first.class + end + + def attributes_for_sync + objects.map do |object| + TableSync::Publishing::Data::Attributes.new( + object: object, destroy: destroyed? + ).construct + end + end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/data/object.rb b/lib/table_sync/publishing/data/object.rb new file mode 100644 index 0000000..5f19679 --- /dev/null +++ b/lib/table_sync/publishing/data/object.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Data::Object + attribute :object + + private + + def attributes_for_sync + TableSync::Publishing::Data::Attributes.new( + object: object, destroy: destroyed? + ).construct + end + + def klass + object.class + end + + def metadata + { created: created? } + end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/data/raw.rb b/lib/table_sync/publishing/data/raw.rb new file mode 100644 index 0000000..0d83653 --- /dev/null +++ b/lib/table_sync/publishing/data/raw.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Data::Raw + attribute :klass + attribute :attributes_for_sync +end diff --git a/lib/table_sync/publishing/message.rb b/lib/table_sync/publishing/message.rb new file mode 100644 index 0000000..e5b0cd3 --- /dev/null +++ b/lib/table_sync/publishing/message.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Message + include Tainbox + + attribute :klass + attribute :primary_keys + attribute :options + + attr_accessor :params, :publishing_data + + def initialize(params) + super(params) + + init_klass + + wrap_and_symbolize_primary_keys + validate_primary_keys + + init_params + init_publishing_data + end + + def publish + Rabbit.publish(message_params) + end + + def message_params + params.merge(data: publishing_data) + end + + # INITIALIZATION + + def init_klass + self.klass = klass.constantize # Object.const_get(klass) + end + + def wrap_and_symbolize_primary_keys + self.primary_keys = Array.wrap(primary_keys).map(&:symbolize_keys) + end + + def validate_primary_keys + TableSync::Publishing::Message::Validate.new(klass, primary_keys).call! + end + + def init_params + self.params = if batch? + TableSync::Publishing::Message::Batch::Params.new(options).call + else + TableSync::Publishing::Message::Params.new(object).call + end + end + + def init_publishing_data + self.publishing_data = TableSync::Publishing::Message::Params.new( + klass: klass, primary_keys: primary_keys, + ).call + end + + def batch? + primary_keys.size >1 + end +end diff --git a/lib/table_sync/publishing/message/base.rb b/lib/table_sync/publishing/message/base.rb new file mode 100644 index 0000000..96ee393 --- /dev/null +++ b/lib/table_sync/publishing/message/base.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Message::Base + include Tainbox + + attr_reader :objects + + attribute :klass + attribute :attrs + attribute :state + + def initialize(**params) + super(**params) + + @objects = find_objects + + raise "Synced objects not found!" if objects.empty? + end + + def publish + Rabbit.publish(message_params) + end + + def notify + # notify stuff + end + + private + + # find if update|create and new if destruction? + + def find_objects + TableSync::Publishing::Message::FindObjects.new( + klass: klass, attrs: attrs + ).list + end + + # MESSAGE PARAMS + + def message_params + params.merge(data: data) + end + + def data + raise NotImplementedError + end + + def params + raise NotImplementedError + end +end diff --git a/lib/table_sync/publishing/message/batch.rb b/lib/table_sync/publishing/message/batch.rb new file mode 100644 index 0000000..1030c13 --- /dev/null +++ b/lib/table_sync/publishing/message/batch.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Message::Batch < TableSync::Publishing::Message::Base + attribute :headers + attribute :routing_key + + private + + def data + TableSync::Publishing::Data::Batch.new( + objects: objects, state: state + ).construct + end + + def params + TableSync::Publishing::Params::Batch.new( + klass: klass, headers: headers, routing_key: routing_key + ).construct + end +end diff --git a/lib/table_sync/publishing/message/find_objects.rb b/lib/table_sync/publishing/message/find_objects.rb new file mode 100644 index 0000000..757fad1 --- /dev/null +++ b/lib/table_sync/publishing/message/find_objects.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Message::FindObjects + include Tainbox + include Memery + + attribute :klass + attribute :attrs + + def initialize(**params) + super(**params) + + self.klass = klass.constantize + self.attrs = Array.wrap(attrs).map(&:deep_symbolize_keys) + + raise "Contains incomplete primary keys!" unless valid? + end + + def list + needles.map { |needle| find_object(needle) } + end + + private + + def needles + attrs.map { |attrs| attrs.slice(*primary_key_columns) } + end + + def find_object(needle) + TableSync.publishing_adapter.find(klass, needle) + end + + memoize def primary_key_columns + TableSync.publishing_adapter.primary_key_columns(klass) + end + + # VALIDATION + + def valid? + attrs.map(&:keys).all? { |keys| contains_pk?(keys) } + end + + def contains_pk?(keys) + (primary_key_columns - keys).empty? + end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/message/object.rb b/lib/table_sync/publishing/message/object.rb new file mode 100644 index 0000000..f70ef88 --- /dev/null +++ b/lib/table_sync/publishing/message/object.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Message::Object < TableSync::Publishing::Message::Base + private + + def object + objects.first + end + + def data + TableSync::Publishing::Data::Object.new( + object: object, state: state + ).construct + end + + def params + TableSync::Publishing::Params::Object.new(object: object).construct + end +end diff --git a/lib/table_sync/publishing/message/raw.rb b/lib/table_sync/publishing/message/raw.rb new file mode 100644 index 0000000..84d039b --- /dev/null +++ b/lib/table_sync/publishing/message/raw.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Message::Raw + include Tainbox + + attribute :klass + attribute :attrs + attribute :state + attribute :routing_key + attribute :headers + + def publish + Rabbit.publish(message_params) + end + + private + + def message_params + params.merge(data: data) + end + + def data + TableSync::Publishing::Data::Raw.new( + attributes_for_sync: attrs, state: state + ).construct + end + + def params + TableSync::Publishing::Params::Batch.new( + klass: klass, routing_key: routing_key, headers: headers, + ).construct + end +end diff --git a/lib/table_sync/publishing/params/base.rb b/lib/table_sync/publishing/params/base.rb new file mode 100644 index 0000000..2002efe --- /dev/null +++ b/lib/table_sync/publishing/params/base.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Params::Base + include Tainbox + + DEFAULT_PARAMS = { + confirm_select: true, + realtime: true, + event: :table_sync, + }.freeze + + def construct + DEFAULT_PARAMS.merge( + routing_key: routing_key, + headers: headers, + exchange_name: exchange_name, + ) + end + + private + + # EXCHANGE + + # only set if exists in original, what if simply nil? + def exchange_name + TableSync.exchange_name + end + + # ROUTING KEY + + def calculated_routing_key + if TableSync.routing_key_callable + TableSync.routing_key_callable.call(klass, attrs_for_routing_key) + else + raise "Can't publish, set TableSync.routing_key_callable!" + end + end + + # HEADERS + + def calculated_headers + if TableSync.headers_callable + TableSync.headers_callable.call(klass, attrs_for_routing_key) + else + raise "Can't publish, set TableSync.headers_callable!" + end + end + + # NOT IMPLEMENTED + + # name of the model being synced in the string format + def klass + raise NotImplementedError + end + + def routing_key + raise NotImplementedError + end + + def headers + raise NotImplementedError + end + + def attrs_for_routing_key + raise NotImplementedError + end + + def attrs_for_headers + raise NotImplementedError + end +end diff --git a/lib/table_sync/publishing/params/batch.rb b/lib/table_sync/publishing/params/batch.rb new file mode 100644 index 0000000..63cfeb3 --- /dev/null +++ b/lib/table_sync/publishing/params/batch.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Params::Batch + attribute :klass + + attribute :routing_key, default: -> { calculated_routing_key } + attribute :headers, default: -> { calculated_headers } + + private + + def attrs_for_routing_key + {} + end + + def attrs_for_headers + {} + end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/params/object.rb b/lib/table_sync/publishing/params/object.rb new file mode 100644 index 0000000..ec04a0b --- /dev/null +++ b/lib/table_sync/publishing/params/object.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Params::Object + attribute :object + + private + + def klass + object.class.name + end + + # ROUTING KEY + + def routing_key + calculated_routing_key + end + + def attrs_for_routing_key + object.try(:attrs_for_routing_key) || super + end + + # HEADERS + + def headers + calculated_headers + end + + def attrs_for_headers + object.try(:attrs_for_headers) || super + end +end diff --git a/lib/table_sync/publishing/pub.rb b/lib/table_sync/publishing/pub.rb new file mode 100644 index 0000000..07f17b2 --- /dev/null +++ b/lib/table_sync/publishing/pub.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Publisher + include Tainbox + + attribute :klass + attribute :attrs + attribute :state + + attribute :debounce_time, default: 60 + + def publish + message.publish + end + + private + + def message + TableSync::Publishing::Message::Object.new( + klass: klass, attrs: attrs, state: state + ) + end + + # debounces, queues +end From 4a43d101c8aacea9997d250c72fa66eae6137eb3 Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Wed, 24 Mar 2021 15:29:25 -0400 Subject: [PATCH 02/20] refactor publishing --- lib/table_sync.rb | 25 +++- lib/table_sync/errors.rb | 9 ++ lib/table_sync/orm_adapter/active_record.rb | 19 +++ lib/table_sync/orm_adapter/sequel.rb | 60 ++++++++ lib/table_sync/publishing/base_publisher.rb | 114 ---------------- lib/table_sync/publishing/batch.rb | 43 ++++++ lib/table_sync/publishing/batch_pub.rb | 62 --------- lib/table_sync/publishing/batch_publisher.rb | 109 --------------- lib/table_sync/publishing/data/attributes.rb | 26 ---- lib/table_sync/publishing/data/base.rb | 61 --------- lib/table_sync/publishing/data/batch.rb | 19 --- lib/table_sync/publishing/data/object.rb | 21 --- lib/table_sync/publishing/data/objects.rb | 56 ++++++++ lib/table_sync/publishing/data/raw.rb | 2 +- .../publishing/helpers/attributes.rb | 13 ++ .../publishing/helpers/debounce_time.rb | 9 ++ lib/table_sync/publishing/helpers/objects.rb | 35 +++++ lib/table_sync/publishing/message.rb | 63 --------- lib/table_sync/publishing/message/base.rb | 34 +++-- lib/table_sync/publishing/message/batch.rb | 8 +- .../publishing/message/find_objects.rb | 46 ------- lib/table_sync/publishing/message/object.rb | 19 --- lib/table_sync/publishing/message/raw.rb | 13 +- lib/table_sync/publishing/message/single.rb | 9 ++ .../publishing/orm_adapter/active_record.rb | 32 ----- .../publishing/orm_adapter/sequel.rb | 57 -------- lib/table_sync/publishing/params/base.rb | 29 ++-- lib/table_sync/publishing/params/batch.rb | 21 +-- lib/table_sync/publishing/params/object.rb | 31 ----- lib/table_sync/publishing/params/raw.rb | 5 + lib/table_sync/publishing/params/single.rb | 29 ++++ lib/table_sync/publishing/pub.rb | 25 ---- lib/table_sync/publishing/publisher.rb | 129 ------------------ lib/table_sync/publishing/raw.rb | 36 +++++ lib/table_sync/publishing/single.rb | 41 ++++++ lib/table_sync/setup/active_record.rb | 18 +++ lib/table_sync/setup/base.rb | 67 +++++++++ lib/table_sync/setup/sequel.rb | 20 +++ lib/table_sync/version.rb | 2 +- spec/instrument/publish_spec.rb | 8 +- spec/publishing/sequel/adapter_spec.rb | 2 +- 41 files changed, 543 insertions(+), 884 deletions(-) create mode 100644 lib/table_sync/orm_adapter/active_record.rb create mode 100644 lib/table_sync/orm_adapter/sequel.rb delete mode 100644 lib/table_sync/publishing/base_publisher.rb create mode 100644 lib/table_sync/publishing/batch.rb delete mode 100644 lib/table_sync/publishing/batch_pub.rb delete mode 100644 lib/table_sync/publishing/batch_publisher.rb delete mode 100644 lib/table_sync/publishing/data/attributes.rb delete mode 100644 lib/table_sync/publishing/data/base.rb delete mode 100644 lib/table_sync/publishing/data/batch.rb delete mode 100644 lib/table_sync/publishing/data/object.rb create mode 100644 lib/table_sync/publishing/data/objects.rb create mode 100644 lib/table_sync/publishing/helpers/attributes.rb create mode 100644 lib/table_sync/publishing/helpers/debounce_time.rb create mode 100644 lib/table_sync/publishing/helpers/objects.rb delete mode 100644 lib/table_sync/publishing/message.rb delete mode 100644 lib/table_sync/publishing/message/find_objects.rb delete mode 100644 lib/table_sync/publishing/message/object.rb create mode 100644 lib/table_sync/publishing/message/single.rb delete mode 100644 lib/table_sync/publishing/orm_adapter/active_record.rb delete mode 100644 lib/table_sync/publishing/orm_adapter/sequel.rb delete mode 100644 lib/table_sync/publishing/params/object.rb create mode 100644 lib/table_sync/publishing/params/raw.rb create mode 100644 lib/table_sync/publishing/params/single.rb delete mode 100644 lib/table_sync/publishing/pub.rb delete mode 100644 lib/table_sync/publishing/publisher.rb create mode 100644 lib/table_sync/publishing/raw.rb create mode 100644 lib/table_sync/publishing/single.rb create mode 100644 lib/table_sync/setup/active_record.rb create mode 100644 lib/table_sync/setup/base.rb create mode 100644 lib/table_sync/setup/sequel.rb diff --git a/lib/table_sync.rb b/lib/table_sync.rb index 5dcbcbd..db15ee3 100644 --- a/lib/table_sync.rb +++ b/lib/table_sync.rb @@ -17,6 +17,9 @@ module TableSync require_relative "table_sync/naming_resolver/sequel" require_relative "table_sync/receiving" require_relative "table_sync/publishing" + require_relative "table_sync/setup/base" + require_relative "table_sync/setup/active_record" + require_relative "table_sync/setup/sequel" class << self attr_accessor :publishing_job_class_callable @@ -25,22 +28,32 @@ class << self attr_accessor :exchange_name attr_accessor :routing_metadata_callable attr_accessor :notifier + attr_reader :orm attr_reader :publishing_adapter attr_reader :receiving_model + attr_reader :setup - def sync(klass, **opts) - publishing_adapter.setup_sync(klass, opts) + def sync(object_class, on: nil, if_condition: nil, unless_condition: nil, debounce_time: nil) + setup.new( + object_class: object_class, + on: on, + if_condition: if_condition, + unless_condition: unless_condition, + debounce_time: debounce_time, + ).register_callbacks end def orm=(val) case val when :active_record - @publishing_adapter = Publishing::ORMAdapter::ActiveRecord - @receiving_model = Receiving::Model::ActiveRecord + @publishing_adapter = TableSync::ORMAdapter::ActiveRecord + @receiving_model = Receiving::Model::ActiveRecord + @setup = TableSync::Setup::ActiveRecord when :sequel - @publishing_adapter = Publishing::ORMAdapter::Sequel - @receiving_model = Receiving::Model::Sequel + @publishing_adapter = TableSync::ORMAdapter::Sequel + @receiving_model = Receiving::Model::Sequel + @setup = TableSync::Setup::Sequel else raise ORMNotSupported.new(val.inspect) end diff --git a/lib/table_sync/errors.rb b/lib/table_sync/errors.rb index 78abb61..17a88ab 100644 --- a/lib/table_sync/errors.rb +++ b/lib/table_sync/errors.rb @@ -3,6 +3,15 @@ module TableSync Error = Class.new(StandardError) + class NoJobClassError < Error + def initialize(type) + super(<<~MSG) + Can't find job class for publishing! + Please initialize TableSync.#{type}_publishing_job with the required job class! + MSG + end + end + class UpsertError < Error def initialize(data:, target_keys:, result:) super("data: #{data.inspect}, target_keys: #{target_keys.inspect}, result: #{result.inspect}") diff --git a/lib/table_sync/orm_adapter/active_record.rb b/lib/table_sync/orm_adapter/active_record.rb new file mode 100644 index 0000000..e6ed194 --- /dev/null +++ b/lib/table_sync/orm_adapter/active_record.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module TableSync::ORMAdapter + module ActiveRecord + module_function + + def model_naming(object) + ::TableSync::NamingResolver::ActiveRecord.new(table_name: object.table_name) + end + + def find(dataset, conditions) + dataset.find_by(conditions) + end + + def attributes(object) + object.attributes + end + end +end diff --git a/lib/table_sync/orm_adapter/sequel.rb b/lib/table_sync/orm_adapter/sequel.rb new file mode 100644 index 0000000..cc7ebc2 --- /dev/null +++ b/lib/table_sync/orm_adapter/sequel.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +class TableSync::ORMAdapter::Sequel + attr_reader :object, :object_class, :object_data + + def initialize(object_class, object_data) + @object_class = object_class + @object_data = object_data + + validate! + end + + def init + @object = object_class.new(object_data) + end + + def find + @object = object_class.find(needle) + end + + def needle + object_data.slice(*primary_key_columns) + end + + def validate! + if (primary_key_columns - object_data.keys).any? + raise NoPrimaryKeyError.new(object_class, object_data, primary_key_columns) + end + end + + def primary_key_columns + Array.wrap(object_class.primary_key) + end + + # ? + def primary_key + object.primary_key + end + + def attributes + object.values + end + + def attributes_for_update + if object.method_defined?(:attributes_for_sync) + object.attributes_for_sync + else + attributes + end + end + + def attributes_for_destroy + if object_class.method_defined?(:table_sync_destroy_attributes) + object_class.table_sync_destroy_attributes(attributes) + else + primary_key + end + end + end +end diff --git a/lib/table_sync/publishing/base_publisher.rb b/lib/table_sync/publishing/base_publisher.rb deleted file mode 100644 index 4787573..0000000 --- a/lib/table_sync/publishing/base_publisher.rb +++ /dev/null @@ -1,114 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::BasePublisher - include Memery - - BASE_SAFE_JSON_TYPES = [NilClass, String, TrueClass, FalseClass, Numeric, Symbol].freeze - NOT_MAPPED = Object.new - - private - - attr_accessor :object_class - - # @!method job_callable - # @!method job_callable_error_message - # @!method attrs_for_callables - # @!method attrs_for_routing_key - # @!method attrs_for_metadata - # @!method attributes_for_sync - - memoize def current_time - Time.current - end - - memoize def primary_keys - Array(object_class.primary_key).map(&:to_sym) - end - - memoize def attributes_for_sync_defined? - object_class.method_defined?(:attributes_for_sync) - end - - memoize def attrs_for_routing_key_defined? - object_class.method_defined?(:attrs_for_routing_key) - end - - memoize def attrs_for_metadata_defined? - object_class.method_defined?(:attrs_for_metadata) - end - - def resolve_routing_key - routing_key_callable.call(object_class.name, attrs_for_routing_key) - end - - def metadata - TableSync.routing_metadata_callable&.call(object_class.name, attrs_for_metadata) - end - - def confirm? - @confirm - end - - def routing_key_callable - return TableSync.routing_key_callable if TableSync.routing_key_callable - raise "Can't publish, set TableSync.routing_key_callable" - end - - def filter_safe_for_serialization(object) - case object - when Array - object.map(&method(:filter_safe_for_serialization)).select(&method(:object_mapped?)) - when Hash - object - .transform_keys(&method(:filter_safe_for_serialization)) - .transform_values(&method(:filter_safe_hash_values)) - .select { |*objects| objects.all?(&method(:object_mapped?)) } - when Float::INFINITY - NOT_MAPPED - when *BASE_SAFE_JSON_TYPES - object - else - NOT_MAPPED - end - end - - def filter_safe_hash_values(value) - case value - when Symbol - value.to_s - else - filter_safe_for_serialization(value) - end - end - - def object_mapped?(object) - object != NOT_MAPPED - end - - def job_class - job_callable ? job_callable.call : raise(job_callable_error_message) - end - - def publishing_data - { - model: object_class.try(:table_sync_model_name) || object_class.name, - attributes: attributes_for_sync, - version: current_time.to_f, - } - end - - def params - params = { - event: :table_sync, - data: publishing_data, - confirm_select: confirm?, - routing_key: routing_key, - realtime: true, - headers: metadata, - } - - params[:exchange_name] = TableSync.exchange_name if TableSync.exchange_name - - params - end -end diff --git a/lib/table_sync/publishing/batch.rb b/lib/table_sync/publishing/batch.rb new file mode 100644 index 0000000..7f3d32c --- /dev/null +++ b/lib/table_sync/publishing/batch.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Batch + include Tainbox + + attribute :object_class + attribute :original_attributes + + attribute :routing_key + attribute :headers + + attribute :event + + def publish_later + job.perform_later(job_attributes) + end + + def publish_now + TableSync::Publishing::Message::Batch.new(attributes).publish + end + + private + + # JOB + + def job + TableSync.batch_publishing_job || raise NoJobClassError.new("batch") + end + + def job_attributes + attributes.merge( + original_attributes: serialized_original_attributes, + ) + end + + def serialized_original_attributes + original_attributes.map do |set_of_attributes| + TableSync::Publishing::Helpers::Attributes + .new(set_of_attributes) + .serialize + end + end +end diff --git a/lib/table_sync/publishing/batch_pub.rb b/lib/table_sync/publishing/batch_pub.rb deleted file mode 100644 index 2e4fd84..0000000 --- a/lib/table_sync/publishing/batch_pub.rb +++ /dev/null @@ -1,62 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Publisher - include Tainbox - - attribute :klass - attribute :attrs - attribute :state - attribute :routing_key - attribute :headers - - attribute :raw, default: false - - # how necessary is serialization check? - def publish - job.perform_later(attributes) - end - - def publish_now - message.publish - end - - private - - # MESSAGE - - def message - raw ? raw_message : batch_message - end - - def batch_message - TableSync::Publishing::Message::Batch.new(**message_params) - end - - def raw_message - TableSync::Publishing::Message::Raw.new(**message_params) - end - - def message_params - attributes.slice( - :klass, :attrs, :state, :routing_key, :headers - ) - end - - # JOB - - def job - job_callable ? job_callable.call : raise job_callable_error_message - end - - def job_callable - TableSync.batch_publishing_job_class_callable - end - - def job_callable_error_message - "Can't publish, set TableSync.batch_publishing_job_class_callable" - end -end - -# Насколько нужно проверять сриализацию? Никто не пихает туда сложные объекты. - -# Не надо конфёрм. \ No newline at end of file diff --git a/lib/table_sync/publishing/batch_publisher.rb b/lib/table_sync/publishing/batch_publisher.rb deleted file mode 100644 index 600452e..0000000 --- a/lib/table_sync/publishing/batch_publisher.rb +++ /dev/null @@ -1,109 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::BatchPublisher < TableSync::Publishing::BasePublisher - def initialize(object_class, original_attributes_array, **options) - @original_attributes_array = original_attributes_array.map do |hash| - filter_safe_for_serialization(hash.deep_symbolize_keys) - end - - @object_class = object_class.constantize - @confirm = options[:confirm] || true - @routing_key = options[:routing_key] || resolve_routing_key - @push_original_attributes = options[:push_original_attributes] || false - @headers = options[:headers] - @event = options[:event] || :update - end - - def publish - enqueue_job - end - - def publish_now - return unless need_publish? - Rabbit.publish(params) - - model_naming = TableSync.publishing_adapter.model_naming(object_class) - TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema, - event: event, - count: publishing_data[:attributes].size, direction: :publish - end - - private - - attr_reader :original_attributes_array, :routing_key, :headers, :event - - def push_original_attributes? - @push_original_attributes - end - - def need_publish? - (push_original_attributes? && original_attributes_array.present?) || objects.present? - end - - memoize def objects - needles.map { |needle| TableSync.publishing_adapter.find(object_class, needle) }.compact - end - - def job_callable - TableSync.batch_publishing_job_class_callable - end - - def job_callable_error_message - "Can't publish, set TableSync.batch_publishing_job_class_callable" - end - - def attrs_for_callables - {} - end - - def attrs_for_routing_key - {} - end - - def attrs_for_metadata - {} - end - - def params - { - **super, - headers: headers, - } - end - - def needles - original_attributes_array.map { |original_attributes| original_attributes.slice(*primary_keys) } - end - - def publishing_data - { - **super, - event: event, - metadata: {}, - } - end - - def attributes_for_sync - return original_attributes_array if push_original_attributes? - - objects.map do |object| - if attributes_for_sync_defined? - object.attributes_for_sync - else - TableSync.publishing_adapter.attributes(object) - end - end - end - - def enqueue_job - job_class.perform_later( - object_class.name, - original_attributes_array, - enqueue_additional_options, - ) - end - - def enqueue_additional_options - { confirm: confirm?, push_original_attributes: push_original_attributes? } - end -end diff --git a/lib/table_sync/publishing/data/attributes.rb b/lib/table_sync/publishing/data/attributes.rb deleted file mode 100644 index b8f044e..0000000 --- a/lib/table_sync/publishing/data/attributes.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Data::Attributes - include Tainbox - - attribute :object - attribute :destroy - - # Can't find object when destruction! - - def for_sync - destroy ? attributes_for_destroy : attributes_for_update - end - - private - - def attributes_for_destroy - object.try(:table_sync_destroy_attributes) || - TableSync.publishing_adapter.primary_key(object) - end - - def attributes_for_update - object.try(:attributes_for_sync) || - TableSync.publishing_adapter.attributes(object) - end -end \ No newline at end of file diff --git a/lib/table_sync/publishing/data/base.rb b/lib/table_sync/publishing/data/base.rb deleted file mode 100644 index 0d4ab2b..0000000 --- a/lib/table_sync/publishing/data/base.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Data::Base - include Tainbox - - attribute :state, default: :updated - - def construct - { - model: model, - attributes: attributes_for_sync, - version: version, - event: event, - metadata: metadata, - } - end - - private - - # MISC - - def model - klass.try(:table_sync_model_name) || klass.name - end - - def version - Time.current.to_f - end - - def metadata - {} - end - - # STATE, EVENT - - def destroyed? - state == :destroyed - end - - def created? - state == :created - end - - def event - destroyed? ? :destroy : :update - end - - # NOT IMPLEMENTED - - def klass - raise NotImplementedError - end - - def attributes_for_sync - raise NotImplementedError - end -end - -# def validate_state -# raise "Unknown state: #{state.inspect}" unless %i[created updated destroyed].include?(state) -# end \ No newline at end of file diff --git a/lib/table_sync/publishing/data/batch.rb b/lib/table_sync/publishing/data/batch.rb deleted file mode 100644 index d231144..0000000 --- a/lib/table_sync/publishing/data/batch.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Data::Batch - attribute :objects - - private - - def klass - objects.first.class - end - - def attributes_for_sync - objects.map do |object| - TableSync::Publishing::Data::Attributes.new( - object: object, destroy: destroyed? - ).construct - end - end -end \ No newline at end of file diff --git a/lib/table_sync/publishing/data/object.rb b/lib/table_sync/publishing/data/object.rb deleted file mode 100644 index 5f19679..0000000 --- a/lib/table_sync/publishing/data/object.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Data::Object - attribute :object - - private - - def attributes_for_sync - TableSync::Publishing::Data::Attributes.new( - object: object, destroy: destroyed? - ).construct - end - - def klass - object.class - end - - def metadata - { created: created? } - end -end \ No newline at end of file diff --git a/lib/table_sync/publishing/data/objects.rb b/lib/table_sync/publishing/data/objects.rb new file mode 100644 index 0000000..27d969d --- /dev/null +++ b/lib/table_sync/publishing/data/objects.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Data::Objects + attr_reader :objects, :event + + def initialize(objects:, event:) + @objects = objects + @event = event + end + + def construct + { + model: model, + attributes: attributes_for_sync, + version: version, + event: event, + metadata: metadata, + } + end + + private + + def model + if object_class.method_defined?(:table_sync_model_name) + object_class.table_sync_model_name + else + object_class.name + end + end + + def version + Time.current.to_f + end + + def metadata + { created: event == :create } # remove? who needs this? + end + + def object_class + objects.first.class + end + + def attributes_for_sync + objects.map do |object| + if destruction? + object.attributes_for_destroy + else + object.attributes_for_update + end + end + end + + def destruction? + event == :destroy + end +end diff --git a/lib/table_sync/publishing/data/raw.rb b/lib/table_sync/publishing/data/raw.rb index 0d83653..4e56065 100644 --- a/lib/table_sync/publishing/data/raw.rb +++ b/lib/table_sync/publishing/data/raw.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true class TableSync::Publishing::Data::Raw - attribute :klass + attribute :object_class attribute :attributes_for_sync end diff --git a/lib/table_sync/publishing/helpers/attributes.rb b/lib/table_sync/publishing/helpers/attributes.rb new file mode 100644 index 0000000..6ef9347 --- /dev/null +++ b/lib/table_sync/publishing/helpers/attributes.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Helpers::Attributes + attr_reader :attributes + + def initialize(attributes) + @attributes = attributes.deep_symbolize_keys + end + + def serialize + attributes + end +end diff --git a/lib/table_sync/publishing/helpers/debounce_time.rb b/lib/table_sync/publishing/helpers/debounce_time.rb new file mode 100644 index 0000000..6896a1d --- /dev/null +++ b/lib/table_sync/publishing/helpers/debounce_time.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Helpers::DebounceTime + attr_reader :time + + def initialize(time) + @time = time + end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/helpers/objects.rb b/lib/table_sync/publishing/helpers/objects.rb new file mode 100644 index 0000000..249e5fc --- /dev/null +++ b/lib/table_sync/publishing/helpers/objects.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Helpers::Objects + attr_reader :object_class, :original_attributes, :event + + def initialize(object_class:, original_attributes:, event:) + self.event = event + self.object_class = object_class.constantize + self.original_attributes = Array.wrap(original_attributes).map(&:deep_symbolize_keys) + end + + def construct_list + destruction? ? init_objects : find_objects + end + + private + + def init_objects + original_attributes.each do |attrs| + TableSync.publishing_adapter.new(object_class, attrs).init + end + end + + def find_objects + original_attributes.each do |attrs| + TableSync.publishing_adapter.new(object_class, attrs).find + end + end + + end + + def destruction? + event == :destroy + end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/message.rb b/lib/table_sync/publishing/message.rb deleted file mode 100644 index e5b0cd3..0000000 --- a/lib/table_sync/publishing/message.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Message - include Tainbox - - attribute :klass - attribute :primary_keys - attribute :options - - attr_accessor :params, :publishing_data - - def initialize(params) - super(params) - - init_klass - - wrap_and_symbolize_primary_keys - validate_primary_keys - - init_params - init_publishing_data - end - - def publish - Rabbit.publish(message_params) - end - - def message_params - params.merge(data: publishing_data) - end - - # INITIALIZATION - - def init_klass - self.klass = klass.constantize # Object.const_get(klass) - end - - def wrap_and_symbolize_primary_keys - self.primary_keys = Array.wrap(primary_keys).map(&:symbolize_keys) - end - - def validate_primary_keys - TableSync::Publishing::Message::Validate.new(klass, primary_keys).call! - end - - def init_params - self.params = if batch? - TableSync::Publishing::Message::Batch::Params.new(options).call - else - TableSync::Publishing::Message::Params.new(object).call - end - end - - def init_publishing_data - self.publishing_data = TableSync::Publishing::Message::Params.new( - klass: klass, primary_keys: primary_keys, - ).call - end - - def batch? - primary_keys.size >1 - end -end diff --git a/lib/table_sync/publishing/message/base.rb b/lib/table_sync/publishing/message/base.rb index 96ee393..82ec82d 100644 --- a/lib/table_sync/publishing/message/base.rb +++ b/lib/table_sync/publishing/message/base.rb @@ -2,19 +2,21 @@ class TableSync::Publishing::Message::Base include Tainbox + + NO_OBJECTS_FOR_SYNC = Class.new(StandardError) attr_reader :objects - attribute :klass - attribute :attrs - attribute :state + attribute :object_class + attribute :original_attributes + attribute :event - def initialize(**params) - super(**params) + def initialize(params) + super(params) - @objects = find_objects + @objects = find_or_init_objects - raise "Synced objects not found!" if objects.empty? + raise NO_OBJECTS_FOR_SYNC if objects.empty? end def publish @@ -27,12 +29,16 @@ def notify private - # find if update|create and new if destruction? + def find_or_init_objects + TableSync::Publishing::Helpers::Objects.new( + object_class: object_class, original_attributes: original_attributes, event: event, + ).construct_list + end - def find_objects - TableSync::Publishing::Message::FindObjects.new( - klass: klass, attrs: attrs - ).list + def data + TableSync::Publishing::Data::Objects.new( + objects: objects, event: event + ).construct end # MESSAGE PARAMS @@ -42,7 +48,9 @@ def message_params end def data - raise NotImplementedError + TableSync::Publishing::Data::Objects.new( + objects: objects, event: event + ).construct end def params diff --git a/lib/table_sync/publishing/message/batch.rb b/lib/table_sync/publishing/message/batch.rb index 1030c13..54769bf 100644 --- a/lib/table_sync/publishing/message/batch.rb +++ b/lib/table_sync/publishing/message/batch.rb @@ -6,15 +6,9 @@ class TableSync::Publishing::Message::Batch < TableSync::Publishing::Message::Ba private - def data - TableSync::Publishing::Data::Batch.new( - objects: objects, state: state - ).construct - end - def params TableSync::Publishing::Params::Batch.new( - klass: klass, headers: headers, routing_key: routing_key + object_class: object_class, headers: headers, routing_key: routing_key ).construct end end diff --git a/lib/table_sync/publishing/message/find_objects.rb b/lib/table_sync/publishing/message/find_objects.rb deleted file mode 100644 index 757fad1..0000000 --- a/lib/table_sync/publishing/message/find_objects.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Message::FindObjects - include Tainbox - include Memery - - attribute :klass - attribute :attrs - - def initialize(**params) - super(**params) - - self.klass = klass.constantize - self.attrs = Array.wrap(attrs).map(&:deep_symbolize_keys) - - raise "Contains incomplete primary keys!" unless valid? - end - - def list - needles.map { |needle| find_object(needle) } - end - - private - - def needles - attrs.map { |attrs| attrs.slice(*primary_key_columns) } - end - - def find_object(needle) - TableSync.publishing_adapter.find(klass, needle) - end - - memoize def primary_key_columns - TableSync.publishing_adapter.primary_key_columns(klass) - end - - # VALIDATION - - def valid? - attrs.map(&:keys).all? { |keys| contains_pk?(keys) } - end - - def contains_pk?(keys) - (primary_key_columns - keys).empty? - end -end \ No newline at end of file diff --git a/lib/table_sync/publishing/message/object.rb b/lib/table_sync/publishing/message/object.rb deleted file mode 100644 index f70ef88..0000000 --- a/lib/table_sync/publishing/message/object.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Message::Object < TableSync::Publishing::Message::Base - private - - def object - objects.first - end - - def data - TableSync::Publishing::Data::Object.new( - object: object, state: state - ).construct - end - - def params - TableSync::Publishing::Params::Object.new(object: object).construct - end -end diff --git a/lib/table_sync/publishing/message/raw.rb b/lib/table_sync/publishing/message/raw.rb index 84d039b..c5fc19d 100644 --- a/lib/table_sync/publishing/message/raw.rb +++ b/lib/table_sync/publishing/message/raw.rb @@ -3,12 +3,13 @@ class TableSync::Publishing::Message::Raw include Tainbox - attribute :klass - attribute :attrs - attribute :state + attribute :object_class + attribute :original_attributes attribute :routing_key attribute :headers + attribute :event + def publish Rabbit.publish(message_params) end @@ -21,13 +22,13 @@ def message_params def data TableSync::Publishing::Data::Raw.new( - attributes_for_sync: attrs, state: state + attributes_for_sync: original_attributes, event: event, ).construct end def params - TableSync::Publishing::Params::Batch.new( - klass: klass, routing_key: routing_key, headers: headers, + TableSync::Publishing::Params::Raw.new( + object_class: object_class, routing_key: routing_key, headers: headers, ).construct end end diff --git a/lib/table_sync/publishing/message/single.rb b/lib/table_sync/publishing/message/single.rb new file mode 100644 index 0000000..487d407 --- /dev/null +++ b/lib/table_sync/publishing/message/single.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Message::Single < TableSync::Publishing::Message::Base + private + + def params + TableSync::Publishing::Params::Single.new(object: object).construct + end +end diff --git a/lib/table_sync/publishing/orm_adapter/active_record.rb b/lib/table_sync/publishing/orm_adapter/active_record.rb deleted file mode 100644 index a2a65cd..0000000 --- a/lib/table_sync/publishing/orm_adapter/active_record.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module TableSync::Publishing::ORMAdapter - module ActiveRecord - module_function - - def model_naming(object) - ::TableSync::NamingResolver::ActiveRecord.new(table_name: object.table_name) - end - - def find(dataset, conditions) - dataset.find_by(conditions) - end - - def attributes(object) - object.attributes - end - - def setup_sync(klass, opts) - debounce_time = opts.delete(:debounce_time) - - klass.instance_exec do - { create: :created, update: :updated, destroy: :destroyed }.each do |event, state| - after_commit(on: event, **opts) do - TableSync::Publishing::Publisher.new(self.class.name, attributes, - state: state, debounce_time: debounce_time).publish - end - end - end - end - end -end diff --git a/lib/table_sync/publishing/orm_adapter/sequel.rb b/lib/table_sync/publishing/orm_adapter/sequel.rb deleted file mode 100644 index 767298c..0000000 --- a/lib/table_sync/publishing/orm_adapter/sequel.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module TableSync::Publishing::ORMAdapter - module Sequel - module_function - - def model_naming(object) - ::TableSync::NamingResolver::Sequel.new(table_name: object.table_name, db: object.db) - end - - def find(dataset, conditions) - dataset.find(conditions) - end - - def attributes(object) - object.values - end - - def setup_sync(klass, opts) - if_predicate = to_predicate(opts.delete(:if), true) - unless_predicate = to_predicate(opts.delete(:unless), false) - debounce_time = opts.delete(:debounce_time) - - if opts.any? - raise "Only :if, :skip_debounce and :unless options are currently " \ - "supported for Sequel hooks" - end - - register_callbacks(klass, if_predicate, unless_predicate, debounce_time) - end - - def to_predicate(val, default) - return val.to_proc if val.respond_to?(:to_proc) - - -> (*) { default } - end - - def register_callbacks(klass, if_predicate, unless_predicate, debounce_time) - { create: :created, update: :updated, destroy: :destroyed }.each do |event, state| - klass.send(:define_method, :"after_#{event}") do - if instance_eval(&if_predicate) && !instance_eval(&unless_predicate) - db.after_commit do - TableSync::Publishing::Publisher.new( - self.class.name, - values, - state: state, - debounce_time: debounce_time, - ).publish - end - end - - super() - end - end - end - end -end diff --git a/lib/table_sync/publishing/params/base.rb b/lib/table_sync/publishing/params/base.rb index 2002efe..bfcff73 100644 --- a/lib/table_sync/publishing/params/base.rb +++ b/lib/table_sync/publishing/params/base.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class TableSync::Publishing::Params::Base - include Tainbox - DEFAULT_PARAMS = { confirm_select: true, realtime: true, @@ -19,37 +17,38 @@ def construct private - # EXCHANGE - - # only set if exists in original, what if simply nil? - def exchange_name - TableSync.exchange_name - end - # ROUTING KEY def calculated_routing_key if TableSync.routing_key_callable - TableSync.routing_key_callable.call(klass, attrs_for_routing_key) + TableSync.routing_key_callable.call(object_class, attrs_for_routing_key) else raise "Can't publish, set TableSync.routing_key_callable!" end end + def attrs_for_routing_key + {} + end + # HEADERS def calculated_headers if TableSync.headers_callable - TableSync.headers_callable.call(klass, attrs_for_routing_key) + TableSync.headers_callable.call(object_class, attrs_for_headers) else raise "Can't publish, set TableSync.headers_callable!" end end + def attrs_for_headers + {} + end + # NOT IMPLEMENTED # name of the model being synced in the string format - def klass + def object_class raise NotImplementedError end @@ -61,11 +60,7 @@ def headers raise NotImplementedError end - def attrs_for_routing_key - raise NotImplementedError - end - - def attrs_for_headers + def exchange_name raise NotImplementedError end end diff --git a/lib/table_sync/publishing/params/batch.rb b/lib/table_sync/publishing/params/batch.rb index 63cfeb3..c30bcbf 100644 --- a/lib/table_sync/publishing/params/batch.rb +++ b/lib/table_sync/publishing/params/batch.rb @@ -1,18 +1,11 @@ # frozen_string_literal: true -class TableSync::Publishing::Params::Batch - attribute :klass +class TableSync::Publishing::Params::Batch < TableSync::Publishing::Params::Base + include Tainbox - attribute :routing_key, default: -> { calculated_routing_key } - attribute :headers, default: -> { calculated_headers } + attribute :object_class - private - - def attrs_for_routing_key - {} - end - - def attrs_for_headers - {} - end -end \ No newline at end of file + attribute :exchange_name, default: -> { TableSync.exchange_name } + attribute :routing_key, default: -> { calculated_routing_key } + attribute :headers, default: -> { calculated_headers } +end diff --git a/lib/table_sync/publishing/params/object.rb b/lib/table_sync/publishing/params/object.rb deleted file mode 100644 index ec04a0b..0000000 --- a/lib/table_sync/publishing/params/object.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Params::Object - attribute :object - - private - - def klass - object.class.name - end - - # ROUTING KEY - - def routing_key - calculated_routing_key - end - - def attrs_for_routing_key - object.try(:attrs_for_routing_key) || super - end - - # HEADERS - - def headers - calculated_headers - end - - def attrs_for_headers - object.try(:attrs_for_headers) || super - end -end diff --git a/lib/table_sync/publishing/params/raw.rb b/lib/table_sync/publishing/params/raw.rb new file mode 100644 index 0000000..325f13b --- /dev/null +++ b/lib/table_sync/publishing/params/raw.rb @@ -0,0 +1,5 @@ +# frozen_strinf_literal: true + +class TableSync::Publishing::Params::Raw < TableSync::Publishing::Params::Batch + # FOR NAMING CONSISTENCY +end diff --git a/lib/table_sync/publishing/params/single.rb b/lib/table_sync/publishing/params/single.rb new file mode 100644 index 0000000..6619d17 --- /dev/null +++ b/lib/table_sync/publishing/params/single.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Params::Single < TableSync::Publishing::Params::Base + attr_reader :object, :routing_key, :headers + + def initialize(object) + @object = object + @routing_key = calculated_routing_key + @headers = calculated_headers + end + + private + + def object_class + object.class.name + end + + def attrs_for_routing_key + object.try(:attrs_for_routing_key) || super + end + + def attrs_for_headers + object.try(:attrs_for_headers) || super + end + + def exchange_name + TableSync.exchange_name + end +end diff --git a/lib/table_sync/publishing/pub.rb b/lib/table_sync/publishing/pub.rb deleted file mode 100644 index 07f17b2..0000000 --- a/lib/table_sync/publishing/pub.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Publisher - include Tainbox - - attribute :klass - attribute :attrs - attribute :state - - attribute :debounce_time, default: 60 - - def publish - message.publish - end - - private - - def message - TableSync::Publishing::Message::Object.new( - klass: klass, attrs: attrs, state: state - ) - end - - # debounces, queues -end diff --git a/lib/table_sync/publishing/publisher.rb b/lib/table_sync/publishing/publisher.rb deleted file mode 100644 index 7ef1c3f..0000000 --- a/lib/table_sync/publishing/publisher.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Publisher < TableSync::Publishing::BasePublisher - DEBOUNCE_TIME = 1.minute - - # 'original_attributes' are not published, they are used to resolve the routing key - def initialize(object_class, original_attributes, **opts) - @object_class = object_class.constantize - @original_attributes = filter_safe_for_serialization(original_attributes.deep_symbolize_keys) - @confirm = opts.fetch(:confirm, true) - @debounce_time = opts[:debounce_time]&.seconds || DEBOUNCE_TIME - - if opts[:destroyed].nil? - @state = opts.fetch(:state, :updated).to_sym - validate_state - else - # TODO Legacy job support, remove - @state = opts[:destroyed] ? :destroyed : :updated - end - end - - def publish - return enqueue_job if destroyed? || debounce_time.zero? - - sync_time = Rails.cache.read(cache_key) || current_time - debounce_time - 1.second - return if sync_time > current_time - - next_sync_time = sync_time + debounce_time - next_sync_time <= current_time ? enqueue_job : enqueue_job(next_sync_time) - end - - def publish_now - # Update request and object does not exist -> skip publishing - return if !object && !destroyed? - - Rabbit.publish(params) - model_naming = TableSync.publishing_adapter.model_naming(object_class) - TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema, - event: event, direction: :publish - end - - private - - attr_reader :original_attributes - attr_reader :state - attr_reader :debounce_time - - def attrs_for_callables - original_attributes - end - - def attrs_for_routing_key - return object.attrs_for_routing_key if attrs_for_routing_key_defined? - attrs_for_callables - end - - def attrs_for_metadata - return object.attrs_for_metadata if attrs_for_metadata_defined? - attrs_for_callables - end - - def job_callable - TableSync.publishing_job_class_callable - end - - def job_callable_error_message - "Can't publish, set TableSync.publishing_job_class_callable" - end - - def enqueue_job(perform_at = current_time) - job = job_class.set(wait_until: perform_at) - job.perform_later(object_class.name, original_attributes, state: state.to_s, confirm: confirm?) - Rails.cache.write(cache_key, perform_at) - end - - def routing_key - resolve_routing_key - end - - def publishing_data - { - **super, - event: event, - metadata: { created: created? }, - } - end - - def attributes_for_sync - if destroyed? - if object_class.respond_to?(:table_sync_destroy_attributes) - object_class.table_sync_destroy_attributes(original_attributes) - else - needle - end - elsif attributes_for_sync_defined? - object.attributes_for_sync - else - TableSync.publishing_adapter.attributes(object) - end - end - - memoize def object - TableSync.publishing_adapter.find(object_class, needle) - end - - def event - destroyed? ? :destroy : :update - end - - def needle - original_attributes.slice(*primary_keys) - end - - def cache_key - "#{object_class}/#{needle}_table_sync_time".delete(" ") - end - - def destroyed? - state == :destroyed - end - - def created? - state == :created - end - - def validate_state - raise "Unknown state: #{state.inspect}" unless %i[created updated destroyed].include?(state) - end -end diff --git a/lib/table_sync/publishing/raw.rb b/lib/table_sync/publishing/raw.rb new file mode 100644 index 0000000..2b8ba5b --- /dev/null +++ b/lib/table_sync/publishing/raw.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Raw + include Tainbox + + attribute :object_class + attribute :original_attributes + + attribute :routing_key + attribute :headers + + attribute :event + + def publish_now + TableSync::Publishing::Message::Raw.new(attributes).publish + end +end + +# event + +# debounce +# serialization +# def jobs +# enqueue + +# publishers + +# specs + +# docs + +# cases + +# changes + +# add validations? \ No newline at end of file diff --git a/lib/table_sync/publishing/single.rb b/lib/table_sync/publishing/single.rb new file mode 100644 index 0000000..e29ab24 --- /dev/null +++ b/lib/table_sync/publishing/single.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class TableSync::Publishing::Single + include Tainbox + + attribute :object_class + attribute :original_attributes + attribute :event + + attribute :debounce_time, default: 60 + + def publish_later + job.perform_later(job_attributes) + end + + def publish_now + TableSync::Publishing::Message::Single.new(attributes).publish + end + + private + + # DEBOUNCE + + # JOB + + def job + TableSync.single_publishing_job || raise NoJobClassError.new("single") + end + + def job_attributes + attributes.merge( + original_attributes: serialized_original_attributes, + ) + end + + def serialized_original_attributes + TableSync::Publishing::Helpers::Attributes + .new(original_attributes) + .serialize + end +end diff --git a/lib/table_sync/setup/active_record.rb b/lib/table_sync/setup/active_record.rb new file mode 100644 index 0000000..55615fb --- /dev/null +++ b/lib/table_sync/setup/active_record.rb @@ -0,0 +1,18 @@ +# frozen-string_literal: true + +class TableSync::Setup::ActiveRecord < TableSync::Setup::Base + private + + def define_after_commit_on(event) + after_commit(on: event) do + return if not if_condition.call(self) + return if unless_condition.call(self) + + enqueue_message(self) + end + end + + def adapte + TableSync::ORMAdapter::ActiveRecord + end +end diff --git a/lib/table_sync/setup/base.rb b/lib/table_sync/setup/base.rb new file mode 100644 index 0000000..8cc48d3 --- /dev/null +++ b/lib/table_sync/setup/base.rb @@ -0,0 +1,67 @@ +# frozen-string_literal: true + +class TableSync::Setup::Base + include Tainbox + + EVENTS = %i[create update destroy].freeze + INVALID_EVENTS = Class.new(StandardError) + INVALID_CONDITION = Class.new(StandardError) + + attribute :object_class + attribute :debounce_time + attribute :on, default: [] + attribute :if_condition, default: -> { Proc.new {} } + attribute :unless_condition, default: -> { Proc.new {} } + + def initialize(attrs) + super(attrs) + + self.on = Array.wrap(on).map(:to_sym) + + raise INVALID_EVENTS unless valid_events? + raise INVALID_CONDITIONS unless valid_conditions? + end + + def register_callbacks + applicable_events.each do |event| + object_class.instance_exec(&define_after_commit_on(event)) + end + end + + private + + # VALIDATION + + def valid_events? + on.all? { |event| event.in?(EVENTS) } + end + + def valid_conditions? + if_condition.is_a?(Proc) && unless_condition.is_a?(Proc) + end + + # EVENTS + + def applicable_events + on.presence || EVENTS + end + + # CREATING HOOKS + + def define_after_commit_on(event) + raise NotImplementedError + end + + def enqueue_message(object) + TableSync::Publishing::Single.new( + object_class: object.class.name, + original_attributes: adapter.attributes(object), + event: event, + debounce_time: debounce_time, + ).publish_later + end + + def adapter + raise NotImplementedError + end +end diff --git a/lib/table_sync/setup/sequel.rb b/lib/table_sync/setup/sequel.rb new file mode 100644 index 0000000..7109fe0 --- /dev/null +++ b/lib/table_sync/setup/sequel.rb @@ -0,0 +1,20 @@ +# frozen-string_literal: true + +class TableSync::Setup::Sequel < TableSync::Setup::Base + private + + def define_after_commit_on(event) + define_method("after_#{event}".to_sym) do + return if not if_condition.call(self) + return if unless_condition.call(self) + + enqueue_message(self) + + super() + end + end + + def adapter + TableSync::ORMAdapter::Sequel + end +end diff --git a/lib/table_sync/version.rb b/lib/table_sync/version.rb index 33e8dea..58792dd 100644 --- a/lib/table_sync/version.rb +++ b/lib/table_sync/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module TableSync - VERSION = "4.2.2" + VERSION = "6.0" end diff --git a/spec/instrument/publish_spec.rb b/spec/instrument/publish_spec.rb index 70a7a20..0bcb6e4 100644 --- a/spec/instrument/publish_spec.rb +++ b/spec/instrument/publish_spec.rb @@ -28,8 +28,8 @@ def db; end end [ - TableSync::Publishing::ORMAdapter::ActiveRecord, - TableSync::Publishing::ORMAdapter::Sequel, + TableSync::ORMAdapter::ActiveRecord, + TableSync::ORMAdapter::Sequel, ].each do |publishing_adapter| describe TableSync::Instrument do before do @@ -62,9 +62,9 @@ def db; end shared_context "custom schema" do before do - if publishing_adapter == TableSync::Publishing::ORMAdapter::Sequel + if publishing_adapter == TableSync::ORMAdapter::Sequel table_name = Sequel[:custom_schema][:players] - elsif publishing_adapter == TableSync::Publishing::ORMAdapter::ActiveRecord + elsif publishing_adapter == TableSync::ORMAdapter::ActiveRecord table_name = "custom_schema.players" end diff --git a/spec/publishing/sequel/adapter_spec.rb b/spec/publishing/sequel/adapter_spec.rb index 7fb05f4..19cad91 100644 --- a/spec/publishing/sequel/adapter_spec.rb +++ b/spec/publishing/sequel/adapter_spec.rb @@ -23,7 +23,7 @@ class ItemWithBothPredicates < Sequel::Model(:items) TableSync.orm = :active_record -RSpec.describe TableSync::Publishing::ORMAdapter::Sequel, :sequel do +RSpec.describe TableSync::ORMAdapter::Sequel, :sequel do let(:item_class) { ItemWithoutPredicate } let(:attrs) { { name: "An item", price: 10 } } let(:queue_adapter) { ActiveJob::Base.queue_adapter } From f5b16532a4a91b08353482f05a0e9d938edd2b90 Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Sun, 28 Mar 2021 07:13:44 -0400 Subject: [PATCH 03/20] debounce, req fixes --- Gemfile.lock | 14 ++- lib/table_sync.rb | 8 ++ lib/table_sync/orm_adapter/active_record.rb | 94 +++++++++++++-- lib/table_sync/orm_adapter/base.rb | 68 +++++++++++ lib/table_sync/orm_adapter/sequel.rb | 53 +-------- lib/table_sync/publishing.rb | 25 +++- lib/table_sync/publishing/base_publisher.rb | 110 ------------------ lib/table_sync/publishing/batch.rb | 4 +- lib/table_sync/publishing/data/objects.rb | 82 ++++++------- lib/table_sync/publishing/data/raw.rb | 11 +- .../publishing/helpers/attributes.rb | 64 ++++++++-- lib/table_sync/publishing/helpers/debounce.rb | 60 ++++++++++ .../publishing/helpers/debounce_time.rb | 9 -- lib/table_sync/publishing/helpers/objects.rb | 50 ++++---- lib/table_sync/publishing/message/base.rb | 90 +++++++------- lib/table_sync/publishing/message/batch.rb | 20 ++-- lib/table_sync/publishing/message/raw.rb | 64 +++++----- lib/table_sync/publishing/message/single.rb | 12 +- lib/table_sync/publishing/params/base.rb | 98 ++++++++-------- lib/table_sync/publishing/params/batch.rb | 14 ++- lib/table_sync/publishing/params/raw.rb | 6 +- lib/table_sync/publishing/params/single.rb | 46 ++++---- lib/table_sync/publishing/raw.rb | 2 +- lib/table_sync/publishing/single.rb | 7 +- lib/table_sync/setup/active_record.rb | 26 ++--- lib/table_sync/setup/base.rb | 92 +++++++-------- lib/table_sync/setup/sequel.rb | 26 ++--- 27 files changed, 646 insertions(+), 509 deletions(-) create mode 100644 lib/table_sync/orm_adapter/base.rb delete mode 100644 lib/table_sync/publishing/base_publisher.rb create mode 100644 lib/table_sync/publishing/helpers/debounce.rb delete mode 100644 lib/table_sync/publishing/helpers/debounce_time.rb diff --git a/Gemfile.lock b/Gemfile.lock index 2625d24..9841b97 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - table_sync (5.0.0) + table_sync (6.0) memery rabbit_messaging rails @@ -99,15 +99,17 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.3) mimemagic (~> 0.3.2) - memery (1.3.0) + memery (1.4.0) ruby2_keywords (~> 0.0.2) method_source (1.0.0) - mimemagic (0.3.5) - mini_mime (1.0.2) + mimemagic (0.3.10) + nokogiri (~> 1) + rake + mini_mime (1.0.3) mini_portile2 (2.5.0) minitest (5.14.4) nio4r (2.5.7) - nokogiri (1.11.1) + nokogiri (1.11.2) mini_portile2 (~> 2.5.0) racc (~> 1.4) parallel (1.20.1) @@ -117,7 +119,7 @@ GEM pry (0.14.0) coderay (~> 1.1) method_source (~> 1.0) - rabbit_messaging (0.9.0) + rabbit_messaging (0.10.0) bunny (~> 2.0) exception_notification lamian diff --git a/lib/table_sync.rb b/lib/table_sync.rb index db15ee3..276d580 100644 --- a/lib/table_sync.rb +++ b/lib/table_sync.rb @@ -11,12 +11,20 @@ module TableSync require_relative "table_sync/utils" require_relative "table_sync/version" require_relative "table_sync/errors" + require_relative "table_sync/instrument" require_relative "table_sync/instrument_adapter/active_support" + require_relative "table_sync/naming_resolver/active_record" require_relative "table_sync/naming_resolver/sequel" + + require_relative "table_sync/orm_adapter/base" + require_relative "table_sync/orm_adapter/active_record" + require_relative "table_sync/orm_adapter/sequel" + require_relative "table_sync/receiving" require_relative "table_sync/publishing" + require_relative "table_sync/setup/base" require_relative "table_sync/setup/active_record" require_relative "table_sync/setup/sequel" diff --git a/lib/table_sync/orm_adapter/active_record.rb b/lib/table_sync/orm_adapter/active_record.rb index e6ed194..6b760c1 100644 --- a/lib/table_sync/orm_adapter/active_record.rb +++ b/lib/table_sync/orm_adapter/active_record.rb @@ -1,19 +1,93 @@ # frozen_string_literal: true module TableSync::ORMAdapter - module ActiveRecord - module_function - - def model_naming(object) - ::TableSync::NamingResolver::ActiveRecord.new(table_name: object.table_name) - end - - def find(dataset, conditions) - dataset.find_by(conditions) + class ActiveRecord < Base + def primary_key + object.pk_hash end - def attributes(object) + def attributes object.attributes end end end + + +# module TableSync::ORMAdapter +# module ActiveRecord +# module_function + +# def model_naming(object) +# ::TableSync::NamingResolver::ActiveRecord.new(table_name: object.table_name) +# end + +# def find(dataset, conditions) +# dataset.find_by(conditions) +# end + +# def attributes(object) +# object.attributes +# end +# end +# end + + +# frozen_string_literal: true + +# class TableSync::ORMAdapter::Sequel +# attr_reader :object, :object_class, :object_data + +# def initialize(object_class, object_data) +# @object_class = object_class +# @object_data = object_data + +# validate! +# end + +# def init +# @object = object_class.new(object_data) +# end + +# def find +# @object = object_class.find(needle) +# end + +# def needle +# object_data.slice(*primary_key_columns) +# end + +# def validate! +# if (primary_key_columns - object_data.keys).any? +# raise NoPrimaryKeyError.new(object_class, object_data, primary_key_columns) +# end +# end + +# def primary_key_columns +# Array.wrap(object_class.primary_key) +# end + +# def primary_key +# object.pk_hash +# end + +# def attributes +# object.values +# end + +# def attributes_for_update +# if object.respond_to?(:attributes_for_sync) +# object.attributes_for_sync +# else +# attributes +# end +# end + +# def attributes_for_destroy +# if object_class.respond_to?(:table_sync_destroy_attributes) +# object_class.table_sync_destroy_attributes(attributes) +# else +# primary_key +# end +# end +# end +# end diff --git a/lib/table_sync/orm_adapter/base.rb b/lib/table_sync/orm_adapter/base.rb new file mode 100644 index 0000000..ce3365c --- /dev/null +++ b/lib/table_sync/orm_adapter/base.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module TableSync::ORMAdapter + class Base + attr_reader :object, :object_class, :object_data + + def initialize(object_class, object_data) + @object_class = object_class + @object_data = object_data + + validate! + end + + # VALIDATE + + def validate! + if (primary_key_columns - object_data.keys).any? + raise NoPrimaryKeyError.new(object_class, object_data, primary_key_columns) + end + end + + # FIND OR INIT OBJECT + + def init + @object = object_class.new(object_data) + end + + def find + @object = object_class.find(needle) + end + + def needle + object_data.slice(*primary_key_columns) + end + + def primary_key_columns + Array.wrap(object_class.primary_key) + end + + # ATTRIBUTES + + def attributes_for_update + if object.respond_to?(:attributes_for_sync) + object.attributes_for_sync + else + attributes + end + end + + def attributes_for_destroy + if object_class.respond_to?(:table_sync_destroy_attributes) + object_class.table_sync_destroy_attributes(attributes) + else + primary_key + end + end + + # NOT IMPLEMENTED + + def primary_key + raise NotImplementedError + end + + def attributes + raise NotImplementedError + end + end +end diff --git a/lib/table_sync/orm_adapter/sequel.rb b/lib/table_sync/orm_adapter/sequel.rb index cc7ebc2..a8ce5ae 100644 --- a/lib/table_sync/orm_adapter/sequel.rb +++ b/lib/table_sync/orm_adapter/sequel.rb @@ -1,60 +1,13 @@ # frozen_string_literal: true -class TableSync::ORMAdapter::Sequel - attr_reader :object, :object_class, :object_data - - def initialize(object_class, object_data) - @object_class = object_class - @object_data = object_data - - validate! - end - - def init - @object = object_class.new(object_data) - end - - def find - @object = object_class.find(needle) - end - - def needle - object_data.slice(*primary_key_columns) - end - - def validate! - if (primary_key_columns - object_data.keys).any? - raise NoPrimaryKeyError.new(object_class, object_data, primary_key_columns) - end - end - - def primary_key_columns - Array.wrap(object_class.primary_key) - end - - # ? +module TableSync::ORMAdapter + class Sequel < Base def primary_key - object.primary_key + object.pk_hash end def attributes object.values end - - def attributes_for_update - if object.method_defined?(:attributes_for_sync) - object.attributes_for_sync - else - attributes - end - end - - def attributes_for_destroy - if object_class.method_defined?(:table_sync_destroy_attributes) - object_class.table_sync_destroy_attributes(attributes) - else - primary_key - end - end end end diff --git a/lib/table_sync/publishing.rb b/lib/table_sync/publishing.rb index 8bdb214..49c898a 100644 --- a/lib/table_sync/publishing.rb +++ b/lib/table_sync/publishing.rb @@ -2,10 +2,25 @@ module TableSync module Publishing - require_relative "publishing/base_publisher" - require_relative "publishing/publisher" - require_relative "publishing/batch_publisher" - require_relative "publishing/orm_adapter/active_record" - require_relative "publishing/orm_adapter/sequel" + require_relative "publishing/data/objects" + require_relative "publishing/data/raw" + + require_relative "publishing/helpers/attributes" + require_relative "publishing/helpers/debounce" + require_relative "publishing/helpers/objects" + + require_relative "publishing/params/base" + require_relative "publishing/params/batch" + require_relative "publishing/params/raw" + require_relative "publishing/params/single" + + require_relative "publishing/message/base" + require_relative "publishing/message/batch" + require_relative "publishing/message/raw" + require_relative "publishing/message/single" + + require_relative "publishing/batch" + require_relative "publishing/raw" + require_relative "publishing/single" end end diff --git a/lib/table_sync/publishing/base_publisher.rb b/lib/table_sync/publishing/base_publisher.rb deleted file mode 100644 index 7018df4..0000000 --- a/lib/table_sync/publishing/base_publisher.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::BasePublisher - include Memery - - BASE_SAFE_JSON_TYPES = [NilClass, String, TrueClass, FalseClass, Numeric, Symbol].freeze - NOT_MAPPED = Object.new - - private - - attr_accessor :object_class - - # @!method job_callable - # @!method job_callable_error_message - # @!method attrs_for_callables - # @!method attrs_for_routing_key - # @!method attrs_for_metadata - # @!method attributes_for_sync - - memoize def current_time - Time.current - end - - memoize def primary_keys - Array(object_class.primary_key).map(&:to_sym) - end - - memoize def attributes_for_sync_defined? - object_class.method_defined?(:attributes_for_sync) - end - - def resolve_routing_key - routing_key_callable.call(object_class.name, attrs_for_routing_key) - end - - def metadata - TableSync.routing_metadata_callable&.call(object_class.name, attrs_for_metadata) - end - - def confirm? - @confirm - end - - def routing_key_callable - return TableSync.routing_key_callable if TableSync.routing_key_callable - raise "Can't publish, set TableSync.routing_key_callable" - end - - def filter_safe_for_serialization(object) - case object - when Array - object.each_with_object([]) do |value, memo| - value = filter_safe_for_serialization(value) - memo << value if object_mapped?(value) - end - when Hash - object.each_with_object({}) do |(key, value), memo| - key = filter_safe_for_serialization(key) - value = filter_safe_hash_values(value) - memo[key] = value if object_mapped?(key) && object_mapped?(value) - end - when Float::INFINITY - NOT_MAPPED - when *BASE_SAFE_JSON_TYPES - object - else # rubocop:disable Lint/DuplicateBranch - NOT_MAPPED - end - end - - def filter_safe_hash_values(value) - case value - when Symbol - value.to_s - else - filter_safe_for_serialization(value) - end - end - - def object_mapped?(object) - object != NOT_MAPPED - end - - def job_class - job_callable ? job_callable.call : raise(job_callable_error_message) - end - - def publishing_data - { - model: object_class.try(:table_sync_model_name) || object_class.name, - attributes: attributes_for_sync, - version: current_time.to_f, - } - end - - def params - params = { - event: :table_sync, - data: publishing_data, - confirm_select: confirm?, - routing_key: routing_key, - realtime: true, - headers: metadata, - } - - params[:exchange_name] = TableSync.exchange_name if TableSync.exchange_name - - params - end -end diff --git a/lib/table_sync/publishing/batch.rb b/lib/table_sync/publishing/batch.rb index 7f3d32c..8a7cc34 100644 --- a/lib/table_sync/publishing/batch.rb +++ b/lib/table_sync/publishing/batch.rb @@ -9,7 +9,7 @@ class TableSync::Publishing::Batch attribute :routing_key attribute :headers - attribute :event + attribute :event, default: :update def publish_later job.perform_later(job_attributes) @@ -24,7 +24,7 @@ def publish_now # JOB def job - TableSync.batch_publishing_job || raise NoJobClassError.new("batch") + TableSync.batch_publishing_job # || raise NoJobClassError.new("batch") end def job_attributes diff --git a/lib/table_sync/publishing/data/objects.rb b/lib/table_sync/publishing/data/objects.rb index 27d969d..92e68ee 100644 --- a/lib/table_sync/publishing/data/objects.rb +++ b/lib/table_sync/publishing/data/objects.rb @@ -1,56 +1,58 @@ # frozen_string_literal: true -class TableSync::Publishing::Data::Objects - attr_reader :objects, :event +module TableSync::Publishing::Data + class Objects + attr_reader :objects, :event - def initialize(objects:, event:) - @objects = objects - @event = event - end + def initialize(objects:, event:) + @objects = objects + @event = event + end - def construct - { - model: model, - attributes: attributes_for_sync, - version: version, - event: event, - metadata: metadata, - } - end + def construct + { + model: model, + attributes: attributes_for_sync, + version: version, + event: event, + metadata: metadata, + } + end - private + private - def model - if object_class.method_defined?(:table_sync_model_name) - object_class.table_sync_model_name - else - object_class.name + def model + if object_class.method_defined?(:table_sync_model_name) + object_class.table_sync_model_name + else + object_class.name + end end - end - def version - Time.current.to_f - end + def version + Time.current.to_f + end - def metadata - { created: event == :create } # remove? who needs this? - end + def metadata + { created: event == :create } # remove? who needs this? + end - def object_class - objects.first.class - end + def object_class + objects.first.class + end - def attributes_for_sync - objects.map do |object| - if destruction? - object.attributes_for_destroy - else - object.attributes_for_update + def attributes_for_sync + objects.map do |object| + if destruction? + object.attributes_for_destroy + else + object.attributes_for_update + end end end - end - def destruction? - event == :destroy + def destruction? + event == :destroy + end end end diff --git a/lib/table_sync/publishing/data/raw.rb b/lib/table_sync/publishing/data/raw.rb index 4e56065..a7909de 100644 --- a/lib/table_sync/publishing/data/raw.rb +++ b/lib/table_sync/publishing/data/raw.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true -class TableSync::Publishing::Data::Raw - attribute :object_class - attribute :attributes_for_sync +# check if works! +module TableSync::Publishing::Data + class Raw + include Tainbox + + attribute :object_class + attribute :attributes_for_sync + end end diff --git a/lib/table_sync/publishing/helpers/attributes.rb b/lib/table_sync/publishing/helpers/attributes.rb index 6ef9347..7415506 100644 --- a/lib/table_sync/publishing/helpers/attributes.rb +++ b/lib/table_sync/publishing/helpers/attributes.rb @@ -1,13 +1,61 @@ # frozen_string_literal: true -class TableSync::Publishing::Helpers::Attributes - attr_reader :attributes +module TableSync::Publishing::Helpers + class Attributes + BASE_SAFE_JSON_TYPES = [ + NilClass, + String, + TrueClass, + FalseClass, + Numeric, + Symbol, + ].freeze - def initialize(attributes) - @attributes = attributes.deep_symbolize_keys - end + NOT_MAPPED = Object.new + + attr_reader :attributes + + def initialize(attributes) + @attributes = attributes.deep_symbolize_keys + end + + def serialize + filter_safe_for_serialization(attributes) + end + + def filter_safe_for_serialization(object) + case object + when Array + object.each_with_object([]) do |value, memo| + value = filter_safe_for_serialization(value) + memo << value if object_mapped?(value) + end + when Hash + object.each_with_object({}) do |(key, value), memo| + key = filter_safe_for_serialization(key) + value = filter_safe_hash_values(value) + memo[key] = value if object_mapped?(key) && object_mapped?(value) + end + when Float::INFINITY + NOT_MAPPED + when *BASE_SAFE_JSON_TYPES + object + else # rubocop:disable Lint/DuplicateBranch + NOT_MAPPED + end + end + + def filter_safe_hash_values(value) + case value + when Symbol + value.to_s + else + filter_safe_for_serialization(value) + end + end - def serialize - attributes + def object_mapped?(object) + object != NOT_MAPPED + end end -end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/helpers/debounce.rb b/lib/table_sync/publishing/helpers/debounce.rb new file mode 100644 index 0000000..ae5d7c3 --- /dev/null +++ b/lib/table_sync/publishing/helpers/debounce.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module TableSync::Publishing::Helpers + class Debounce + include Memery + + attr_reader :debounce_time, :object_class, :needle + + def initialize(object_class:, needle:, debounce_time: nil) + @debounce_time = debounce_time + @object_class = object_class + @needle = needle + end + + def sync_time? + no_last_sync_time? || past_next_sync_time? + end + + # No sync before, no need for debounce + def no_last_sync_time? + last_sync_time.nil? + end + + def past_next_sync_time? + + end + + memoize def last_sync_time + Rails.cache.read(cache_key) + end + + memoize def current_time + Time.current + end + + def save_next_sync_time(time) + Rails.cache.write(cache_key, next_sync_time) + end + + def cache_key + "#{object_class}/#{needle.values.join}_table_sync_time".delete(" ") + end + end +end + + # def publish + # return enqueue_job if destroyed? || debounce_time.zero? + + # sync_time = Rails.cache.read(cache_key) || current_time - debounce_time - 1.second + # return if sync_time > current_time + + # next_sync_time = sync_time + debounce_time + # next_sync_time <= current_time ? enqueue_job : enqueue_job(next_sync_time) + # end + + # def enqueue_job(perform_at = current_time) + # job = job_class.set(wait_until: perform_at) + # job.perform_later(object_class.name, original_attributes, state: state.to_s, confirm: confirm?) + # Rails.cache.write(cache_key, perform_at) + # end diff --git a/lib/table_sync/publishing/helpers/debounce_time.rb b/lib/table_sync/publishing/helpers/debounce_time.rb deleted file mode 100644 index 6896a1d..0000000 --- a/lib/table_sync/publishing/helpers/debounce_time.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Helpers::DebounceTime - attr_reader :time - - def initialize(time) - @time = time - end -end \ No newline at end of file diff --git a/lib/table_sync/publishing/helpers/objects.rb b/lib/table_sync/publishing/helpers/objects.rb index 249e5fc..2d646a7 100644 --- a/lib/table_sync/publishing/helpers/objects.rb +++ b/lib/table_sync/publishing/helpers/objects.rb @@ -1,35 +1,35 @@ # frozen_string_literal: true -class TableSync::Publishing::Helpers::Objects - attr_reader :object_class, :original_attributes, :event - - def initialize(object_class:, original_attributes:, event:) - self.event = event - self.object_class = object_class.constantize - self.original_attributes = Array.wrap(original_attributes).map(&:deep_symbolize_keys) - end +module TableSync::Publishing::Helpers + class Objects + attr_reader :object_class, :original_attributes, :event + + def initialize(object_class:, original_attributes:, event:) + self.event = event + self.object_class = object_class.constantize + self.original_attributes = Array.wrap(original_attributes) + end - def construct_list - destruction? ? init_objects : find_objects - end + def construct_list + destruction? ? init_objects : find_objects + end - private + private - def init_objects - original_attributes.each do |attrs| - TableSync.publishing_adapter.new(object_class, attrs).init + def init_objects + original_attributes.each do |attrs| + TableSync.publishing_adapter.new(object_class, attrs).init + end end - end - def find_objects - original_attributes.each do |attrs| - TableSync.publishing_adapter.new(object_class, attrs).find + def find_objects + original_attributes.each do |attrs| + TableSync.publishing_adapter.new(object_class, attrs).find + end end - end + def destruction? + event == :destroy + end end - - def destruction? - event == :destroy - end -end \ No newline at end of file +end diff --git a/lib/table_sync/publishing/message/base.rb b/lib/table_sync/publishing/message/base.rb index 82ec82d..02ca0fd 100644 --- a/lib/table_sync/publishing/message/base.rb +++ b/lib/table_sync/publishing/message/base.rb @@ -1,59 +1,65 @@ # frozen_string_literal: true -class TableSync::Publishing::Message::Base - include Tainbox - - NO_OBJECTS_FOR_SYNC = Class.new(StandardError) +module TableSync::Publishing::Message + class Base + include Tainbox + + NO_OBJECTS_FOR_SYNC = Class.new(StandardError) - attr_reader :objects + attr_reader :objects - attribute :object_class - attribute :original_attributes - attribute :event + attribute :object_class + attribute :original_attributes + attribute :event - def initialize(params) - super(params) + def initialize(params) + super(params) - @objects = find_or_init_objects + @objects = find_or_init_objects - raise NO_OBJECTS_FOR_SYNC if objects.empty? - end + raise NO_OBJECTS_FOR_SYNC if objects.empty? + end - def publish - Rabbit.publish(message_params) - end + def publish + Rabbit.publish(message_params) - def notify - # notify stuff - end + notify! + end - private + def notify! + # model_naming = TableSync.publishing_adapter.model_naming(object_class) + # TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema, + # event: event, direction: :publish + end - def find_or_init_objects - TableSync::Publishing::Helpers::Objects.new( - object_class: object_class, original_attributes: original_attributes, event: event, - ).construct_list - end + private - def data - TableSync::Publishing::Data::Objects.new( - objects: objects, event: event - ).construct - end + def find_or_init_objects + TableSync::Publishing::Helpers::Objects.new( + object_class: object_class, original_attributes: original_attributes, event: event, + ).construct_list + end - # MESSAGE PARAMS + def data + TableSync::Publishing::Data::Objects.new( + objects: objects, event: event + ).construct + end - def message_params - params.merge(data: data) - end + # MESSAGE PARAMS - def data - TableSync::Publishing::Data::Objects.new( - objects: objects, event: event - ).construct - end + def message_params + params.merge(data: data) + end + + def data + TableSync::Publishing::Data::Objects.new( + objects: objects, event: event + ).construct + end - def params - raise NotImplementedError + def params + raise NotImplementedError + end end -end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/message/batch.rb b/lib/table_sync/publishing/message/batch.rb index 54769bf..bed746f 100644 --- a/lib/table_sync/publishing/message/batch.rb +++ b/lib/table_sync/publishing/message/batch.rb @@ -1,14 +1,16 @@ # frozen_string_literal: true -class TableSync::Publishing::Message::Batch < TableSync::Publishing::Message::Base - attribute :headers - attribute :routing_key +module TableSync::Publishing::Message + class Batch < Base + attribute :headers + attribute :routing_key - private + private - def params - TableSync::Publishing::Params::Batch.new( - object_class: object_class, headers: headers, routing_key: routing_key - ).construct - end + def params + TableSync::Publishing::Params::Batch.new( + object_class: object_class, headers: headers, routing_key: routing_key + ).construct + end + end end diff --git a/lib/table_sync/publishing/message/raw.rb b/lib/table_sync/publishing/message/raw.rb index c5fc19d..52aa0d3 100644 --- a/lib/table_sync/publishing/message/raw.rb +++ b/lib/table_sync/publishing/message/raw.rb @@ -1,34 +1,36 @@ # frozen_string_literal: true -class TableSync::Publishing::Message::Raw - include Tainbox - - attribute :object_class - attribute :original_attributes - attribute :routing_key - attribute :headers - - attribute :event - - def publish - Rabbit.publish(message_params) - end - - private - - def message_params - params.merge(data: data) - end - - def data - TableSync::Publishing::Data::Raw.new( - attributes_for_sync: original_attributes, event: event, - ).construct - end - - def params - TableSync::Publishing::Params::Raw.new( - object_class: object_class, routing_key: routing_key, headers: headers, - ).construct +module TableSync::Publishing::Message + class Raw + include Tainbox + + attribute :object_class + attribute :original_attributes + attribute :routing_key + attribute :headers + + attribute :event + + def publish + Rabbit.publish(message_params) + end + + private + + def message_params + params.merge(data: data) + end + + def data + TableSync::Publishing::Data::Raw.new( + attributes_for_sync: original_attributes, event: event, + ).construct + end + + def params + TableSync::Publishing::Params::Raw.new( + object_class: object_class, routing_key: routing_key, headers: headers, + ).construct + end end -end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/message/single.rb b/lib/table_sync/publishing/message/single.rb index 487d407..5dbe7a5 100644 --- a/lib/table_sync/publishing/message/single.rb +++ b/lib/table_sync/publishing/message/single.rb @@ -1,9 +1,11 @@ # frozen_string_literal: true -class TableSync::Publishing::Message::Single < TableSync::Publishing::Message::Base - private +module TableSync::Publishing::Message + class Single < Base + private - def params - TableSync::Publishing::Params::Single.new(object: object).construct - end + def params + TableSync::Publishing::Params::Single.new(object: object).construct + end + end end diff --git a/lib/table_sync/publishing/params/base.rb b/lib/table_sync/publishing/params/base.rb index bfcff73..b745fc8 100644 --- a/lib/table_sync/publishing/params/base.rb +++ b/lib/table_sync/publishing/params/base.rb @@ -1,66 +1,68 @@ # frozen_string_literal: true -class TableSync::Publishing::Params::Base - DEFAULT_PARAMS = { - confirm_select: true, - realtime: true, - event: :table_sync, - }.freeze +module TableSync::Publishing::Params + class Base + DEFAULT_PARAMS = { + confirm_select: true, + realtime: true, + event: :table_sync, + }.freeze - def construct - DEFAULT_PARAMS.merge( - routing_key: routing_key, - headers: headers, - exchange_name: exchange_name, - ) - end + def construct + DEFAULT_PARAMS.merge( + routing_key: routing_key, + headers: headers, + exchange_name: exchange_name, + ) + end - private + private - # ROUTING KEY + # ROUTING KEY - def calculated_routing_key - if TableSync.routing_key_callable - TableSync.routing_key_callable.call(object_class, attrs_for_routing_key) - else - raise "Can't publish, set TableSync.routing_key_callable!" + def calculated_routing_key + if TableSync.routing_key_callable + TableSync.routing_key_callable.call(object_class, attrs_for_routing_key) + else + raise "Can't publish, set TableSync.routing_key_callable!" + end end - end - def attrs_for_routing_key - {} - end + def attrs_for_routing_key + {} + end - # HEADERS + # HEADERS - def calculated_headers - if TableSync.headers_callable - TableSync.headers_callable.call(object_class, attrs_for_headers) - else - raise "Can't publish, set TableSync.headers_callable!" + def calculated_headers + if TableSync.headers_callable + TableSync.headers_callable.call(object_class, attrs_for_headers) + else + raise "Can't publish, set TableSync.headers_callable!" + end end - end - def attrs_for_headers - {} - end + def attrs_for_headers + {} + end - # NOT IMPLEMENTED + # NOT IMPLEMENTED - # name of the model being synced in the string format - def object_class - raise NotImplementedError - end + # name of the model being synced in the string format + def object_class + raise NotImplementedError + end - def routing_key - raise NotImplementedError - end + def routing_key + raise NotImplementedError + end - def headers - raise NotImplementedError - end + def headers + raise NotImplementedError + end - def exchange_name - raise NotImplementedError + def exchange_name + raise NotImplementedError + end end -end +end \ No newline at end of file diff --git a/lib/table_sync/publishing/params/batch.rb b/lib/table_sync/publishing/params/batch.rb index c30bcbf..e6a9b2b 100644 --- a/lib/table_sync/publishing/params/batch.rb +++ b/lib/table_sync/publishing/params/batch.rb @@ -1,11 +1,13 @@ # frozen_string_literal: true -class TableSync::Publishing::Params::Batch < TableSync::Publishing::Params::Base - include Tainbox +module TableSync::Publishing::Params + class Batch < Base + include Tainbox - attribute :object_class + attribute :object_class - attribute :exchange_name, default: -> { TableSync.exchange_name } - attribute :routing_key, default: -> { calculated_routing_key } - attribute :headers, default: -> { calculated_headers } + attribute :exchange_name, default: -> { TableSync.exchange_name } + attribute :routing_key, default: -> { calculated_routing_key } + attribute :headers, default: -> { calculated_headers } + end end diff --git a/lib/table_sync/publishing/params/raw.rb b/lib/table_sync/publishing/params/raw.rb index 325f13b..2cfff50 100644 --- a/lib/table_sync/publishing/params/raw.rb +++ b/lib/table_sync/publishing/params/raw.rb @@ -1,5 +1,7 @@ # frozen_strinf_literal: true -class TableSync::Publishing::Params::Raw < TableSync::Publishing::Params::Batch - # FOR NAMING CONSISTENCY +module TableSync::Publishing::Params + class Raw < Batch + # FOR NAMING CONSISTENCY + end end diff --git a/lib/table_sync/publishing/params/single.rb b/lib/table_sync/publishing/params/single.rb index 6619d17..c6d51e3 100644 --- a/lib/table_sync/publishing/params/single.rb +++ b/lib/table_sync/publishing/params/single.rb @@ -1,29 +1,35 @@ # frozen_string_literal: true -class TableSync::Publishing::Params::Single < TableSync::Publishing::Params::Base - attr_reader :object, :routing_key, :headers - - def initialize(object) - @object = object - @routing_key = calculated_routing_key - @headers = calculated_headers - end +module TableSync::Publishing::Params + class Single < Base + attr_reader :object, :routing_key, :headers - private + def initialize(object) + @object = object + @routing_key = calculated_routing_key + @headers = calculated_headers + end - def object_class - object.class.name - end + private - def attrs_for_routing_key - object.try(:attrs_for_routing_key) || super - end + def object_class + object.class.name + end - def attrs_for_headers - object.try(:attrs_for_headers) || super - end + def attrs_for_routing_key + return object.attrs_for_routing_key if object.respond_to?(:attrs_for_routing_key) + + super + end + + def attrs_for_headers + return object.attrs_for_headers if object.respond_to?(:attrs_for_headers) + + super + end - def exchange_name - TableSync.exchange_name + def exchange_name + TableSync.exchange_name + end end end diff --git a/lib/table_sync/publishing/raw.rb b/lib/table_sync/publishing/raw.rb index 2b8ba5b..ae5e3dd 100644 --- a/lib/table_sync/publishing/raw.rb +++ b/lib/table_sync/publishing/raw.rb @@ -9,7 +9,7 @@ class TableSync::Publishing::Raw attribute :routing_key attribute :headers - attribute :event + attribute :event, default: :update def publish_now TableSync::Publishing::Message::Raw.new(attributes).publish diff --git a/lib/table_sync/publishing/single.rb b/lib/table_sync/publishing/single.rb index e29ab24..d9c6cec 100644 --- a/lib/table_sync/publishing/single.rb +++ b/lib/table_sync/publishing/single.rb @@ -5,8 +5,8 @@ class TableSync::Publishing::Single attribute :object_class attribute :original_attributes - attribute :event + attribute :event, default: :update attribute :debounce_time, default: 60 def publish_later @@ -14,6 +14,9 @@ def publish_later end def publish_now + # # Update request and object does not exist -> skip publishing + # return if !object && !destroyed? + TableSync::Publishing::Message::Single.new(attributes).publish end @@ -24,7 +27,7 @@ def publish_now # JOB def job - TableSync.single_publishing_job || raise NoJobClassError.new("single") + TableSync.single_publishing_job # || raise NoJobClassError.new("single") end def job_attributes diff --git a/lib/table_sync/setup/active_record.rb b/lib/table_sync/setup/active_record.rb index 55615fb..dfa10d1 100644 --- a/lib/table_sync/setup/active_record.rb +++ b/lib/table_sync/setup/active_record.rb @@ -1,18 +1,16 @@ # frozen-string_literal: true -class TableSync::Setup::ActiveRecord < TableSync::Setup::Base - private +module TableSync::Setup + class ActiveRecord < Base + private - def define_after_commit_on(event) - after_commit(on: event) do - return if not if_condition.call(self) - return if unless_condition.call(self) + def define_after_commit_on(event) + after_commit(on: event) do + return if not if_condition.call(self) + return if unless_condition.call(self) - enqueue_message(self) - end - end - - def adapte - TableSync::ORMAdapter::ActiveRecord - end -end + enqueue_message(self.attributes) + end + end + end +end \ No newline at end of file diff --git a/lib/table_sync/setup/base.rb b/lib/table_sync/setup/base.rb index 8cc48d3..0b84784 100644 --- a/lib/table_sync/setup/base.rb +++ b/lib/table_sync/setup/base.rb @@ -1,67 +1,65 @@ # frozen-string_literal: true -class TableSync::Setup::Base - include Tainbox +module TableSync::Setup + class Base + include Tainbox - EVENTS = %i[create update destroy].freeze - INVALID_EVENTS = Class.new(StandardError) - INVALID_CONDITION = Class.new(StandardError) + EVENTS = %i[create update destroy].freeze + INVALID_EVENTS = Class.new(StandardError) + INVALID_CONDITION = Class.new(StandardError) - attribute :object_class - attribute :debounce_time - attribute :on, default: [] - attribute :if_condition, default: -> { Proc.new {} } - attribute :unless_condition, default: -> { Proc.new {} } + attribute :object_class + attribute :debounce_time + attribute :on, default: [] + attribute :if_condition, default: -> { Proc.new {} } + attribute :unless_condition, default: -> { Proc.new {} } - def initialize(attrs) - super(attrs) + def initialize(attrs) + super(attrs) - self.on = Array.wrap(on).map(:to_sym) + self.on = Array.wrap(on).map(:to_sym) - raise INVALID_EVENTS unless valid_events? - raise INVALID_CONDITIONS unless valid_conditions? - end + raise INVALID_EVENTS unless valid_events? + raise INVALID_CONDITIONS unless valid_conditions? + end - def register_callbacks - applicable_events.each do |event| - object_class.instance_exec(&define_after_commit_on(event)) + def register_callbacks + applicable_events.each do |event| + object_class.instance_exec(&define_after_commit_on(event)) + end end - end - private + private - # VALIDATION + # VALIDATION - def valid_events? - on.all? { |event| event.in?(EVENTS) } - end - - def valid_conditions? - if_condition.is_a?(Proc) && unless_condition.is_a?(Proc) - end + def valid_events? + on.all? { |event| event.in?(EVENTS) } + end - # EVENTS + def valid_conditions? + if_condition.is_a?(Proc) && unless_condition.is_a?(Proc) + end - def applicable_events - on.presence || EVENTS - end + # EVENTS - # CREATING HOOKS + def applicable_events + on.presence || EVENTS + end - def define_after_commit_on(event) - raise NotImplementedError - end + # CREATING HOOKS - def enqueue_message(object) - TableSync::Publishing::Single.new( - object_class: object.class.name, - original_attributes: adapter.attributes(object), - event: event, - debounce_time: debounce_time, - ).publish_later - end + def define_after_commit_on(event) + raise NotImplementedError + end - def adapter - raise NotImplementedError + def enqueue_message(original_attributes) + TableSync::Publishing::Single.new( + object_class: self.class.name, + original_attributes: original_attributes, + event: event, + debounce_time: debounce_time, + ).publish_later + end end end diff --git a/lib/table_sync/setup/sequel.rb b/lib/table_sync/setup/sequel.rb index 7109fe0..fbae7e0 100644 --- a/lib/table_sync/setup/sequel.rb +++ b/lib/table_sync/setup/sequel.rb @@ -1,20 +1,18 @@ # frozen-string_literal: true -class TableSync::Setup::Sequel < TableSync::Setup::Base - private +module TableSync::Setup + class Sequel < Base + private - def define_after_commit_on(event) - define_method("after_#{event}".to_sym) do - return if not if_condition.call(self) - return if unless_condition.call(self) + def define_after_commit_on(event) + define_method("after_#{event}".to_sym) do + return if not if_condition.call(self) + return if unless_condition.call(self) - enqueue_message(self) + enqueue_message(self.values) - super() - end - end - - def adapter - TableSync::ORMAdapter::Sequel - end + super() + end + end + end end From b5ba030a6db7311d378ed5b9b07a0abfc70c67d7 Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Sat, 22 May 2021 09:26:22 -0400 Subject: [PATCH 04/20] current --- lib/table_sync.rb | 6 +- lib/table_sync/errors.rb | 2 +- lib/table_sync/orm_adapter/active_record.rb | 6 + lib/table_sync/orm_adapter/base.rb | 20 ++- lib/table_sync/orm_adapter/sequel.rb | 6 + lib/table_sync/publishing/batch.rb | 6 +- lib/table_sync/publishing/data/objects.rb | 2 +- lib/table_sync/publishing/data/raw.rb | 19 ++- lib/table_sync/publishing/helpers/objects.rb | 20 ++- lib/table_sync/publishing/message/base.rb | 6 +- lib/table_sync/publishing/message/raw.rb | 2 +- lib/table_sync/publishing/message/single.rb | 4 + lib/table_sync/publishing/params/base.rb | 20 +-- lib/table_sync/publishing/params/batch.rb | 10 ++ lib/table_sync/publishing/params/single.rb | 20 ++- lib/table_sync/publishing/raw.rb | 6 +- lib/table_sync/publishing/single.rb | 20 ++- spec/publishing/batch_spec.rb | 25 +++ spec/publishing/helpers/attributes_spec.rb | 0 spec/publishing/helpers/debounce_spec.rb | 0 spec/publishing/helpers/objects_spec.rb | 0 spec/publishing/message/batch_spec.rb | 0 spec/publishing/message/raw_spec.rb | 0 spec/publishing/message/single_spec.rb | 0 spec/publishing/raw_spec.rb | 16 ++ spec/publishing/single_spec.rb | 158 +++++++++++++++++++ spec/support/context/publishing.rb | 60 +++++++ spec/support/table_sync_settings.rb | 2 +- 28 files changed, 391 insertions(+), 45 deletions(-) create mode 100644 spec/publishing/batch_spec.rb create mode 100644 spec/publishing/helpers/attributes_spec.rb create mode 100644 spec/publishing/helpers/debounce_spec.rb create mode 100644 spec/publishing/helpers/objects_spec.rb create mode 100644 spec/publishing/message/batch_spec.rb create mode 100644 spec/publishing/message/raw_spec.rb create mode 100644 spec/publishing/message/single_spec.rb create mode 100644 spec/publishing/raw_spec.rb create mode 100644 spec/publishing/single_spec.rb create mode 100644 spec/support/context/publishing.rb diff --git a/lib/table_sync.rb b/lib/table_sync.rb index 276d580..efa2bf8 100644 --- a/lib/table_sync.rb +++ b/lib/table_sync.rb @@ -30,11 +30,13 @@ module TableSync require_relative "table_sync/setup/sequel" class << self - attr_accessor :publishing_job_class_callable + attr_accessor :raise_on_serialization_failure + attr_accessor :raise_on_empty_message + attr_accessor :single_publishing_job_class_callable attr_accessor :batch_publishing_job_class_callable attr_accessor :routing_key_callable attr_accessor :exchange_name - attr_accessor :routing_metadata_callable + attr_accessor :headers_callable attr_accessor :notifier attr_reader :orm diff --git a/lib/table_sync/errors.rb b/lib/table_sync/errors.rb index 17a88ab..fe3c53a 100644 --- a/lib/table_sync/errors.rb +++ b/lib/table_sync/errors.rb @@ -7,7 +7,7 @@ class NoJobClassError < Error def initialize(type) super(<<~MSG) Can't find job class for publishing! - Please initialize TableSync.#{type}_publishing_job with the required job class! + Please initialize TableSync.#{type}_publishing_job_class_callable with the correct proc! MSG end end diff --git a/lib/table_sync/orm_adapter/active_record.rb b/lib/table_sync/orm_adapter/active_record.rb index 6b760c1..66ad790 100644 --- a/lib/table_sync/orm_adapter/active_record.rb +++ b/lib/table_sync/orm_adapter/active_record.rb @@ -6,6 +6,12 @@ def primary_key object.pk_hash end + def find + @object = object_class.find_by(needle) + + super + end + def attributes object.attributes end diff --git a/lib/table_sync/orm_adapter/base.rb b/lib/table_sync/orm_adapter/base.rb index ce3365c..bc657b6 100644 --- a/lib/table_sync/orm_adapter/base.rb +++ b/lib/table_sync/orm_adapter/base.rb @@ -22,11 +22,19 @@ def validate! # FIND OR INIT OBJECT def init - @object = object_class.new(object_data) + @object = object_class.new(object_data.except(*primary_key_columns)) + + needle.each do |column, value| + @object.send("#{column}=", value) + end + + self end def find - @object = object_class.find(needle) + # @object = object_class.find(needle) + + self end def needle @@ -34,7 +42,7 @@ def needle end def primary_key_columns - Array.wrap(object_class.primary_key) + Array.wrap(object_class.primary_key).map(&:to_sym) # temp! end # ATTRIBUTES @@ -55,6 +63,12 @@ def attributes_for_destroy end end + # MISC + + def empty? + object.nil? + end + # NOT IMPLEMENTED def primary_key diff --git a/lib/table_sync/orm_adapter/sequel.rb b/lib/table_sync/orm_adapter/sequel.rb index a8ce5ae..3c027e8 100644 --- a/lib/table_sync/orm_adapter/sequel.rb +++ b/lib/table_sync/orm_adapter/sequel.rb @@ -9,5 +9,11 @@ def primary_key def attributes object.values end + + def find + @object = object_class.find(needle) + + super + end end end diff --git a/lib/table_sync/publishing/batch.rb b/lib/table_sync/publishing/batch.rb index 8a7cc34..feeeb7e 100644 --- a/lib/table_sync/publishing/batch.rb +++ b/lib/table_sync/publishing/batch.rb @@ -24,7 +24,11 @@ def publish_now # JOB def job - TableSync.batch_publishing_job # || raise NoJobClassError.new("batch") + if TableSync.batch_publishing_job_class_callable + TableSync.batch_publishing_job_class_callable.call + else + raise TableSync::NoJobClassError.new("batch") + end end def job_attributes diff --git a/lib/table_sync/publishing/data/objects.rb b/lib/table_sync/publishing/data/objects.rb index 92e68ee..75b2a46 100644 --- a/lib/table_sync/publishing/data/objects.rb +++ b/lib/table_sync/publishing/data/objects.rb @@ -38,7 +38,7 @@ def metadata end def object_class - objects.first.class + objects.first.object_class end def attributes_for_sync diff --git a/lib/table_sync/publishing/data/raw.rb b/lib/table_sync/publishing/data/raw.rb index a7909de..af676e7 100644 --- a/lib/table_sync/publishing/data/raw.rb +++ b/lib/table_sync/publishing/data/raw.rb @@ -3,9 +3,22 @@ # check if works! module TableSync::Publishing::Data class Raw - include Tainbox + attr_reader :object_class, :attributes_for_sync, :event - attribute :object_class - attribute :attributes_for_sync + def initialize(object_class:, attributes_for_sync:, event:) + @object_class = object_class + @attributes_for_sync = attributes_for_sync + @event = event + end + + def construct + { + model: object_class,# model, + attributes: attributes_for_sync, + version: Time.current.to_f,#version, + event: event, + metadata: {}, #metadata, + } + end end end diff --git a/lib/table_sync/publishing/helpers/objects.rb b/lib/table_sync/publishing/helpers/objects.rb index 2d646a7..2082ca3 100644 --- a/lib/table_sync/publishing/helpers/objects.rb +++ b/lib/table_sync/publishing/helpers/objects.rb @@ -5,25 +5,33 @@ class Objects attr_reader :object_class, :original_attributes, :event def initialize(object_class:, original_attributes:, event:) - self.event = event - self.object_class = object_class.constantize - self.original_attributes = Array.wrap(original_attributes) + @event = event + @object_class = object_class.constantize + @original_attributes = Array.wrap(original_attributes) end def construct_list - destruction? ? init_objects : find_objects + if destruction? + without_empty_objects(init_objects) + else + without_empty_objects(find_objects) + end end private + def without_empty_objects(objects) + objects.reject(&:empty?) + end + def init_objects - original_attributes.each do |attrs| + original_attributes.map do |attrs| TableSync.publishing_adapter.new(object_class, attrs).init end end def find_objects - original_attributes.each do |attrs| + original_attributes.map do |attrs| TableSync.publishing_adapter.new(object_class, attrs).find end end diff --git a/lib/table_sync/publishing/message/base.rb b/lib/table_sync/publishing/message/base.rb index 02ca0fd..b04898c 100644 --- a/lib/table_sync/publishing/message/base.rb +++ b/lib/table_sync/publishing/message/base.rb @@ -17,7 +17,7 @@ def initialize(params) @objects = find_or_init_objects - raise NO_OBJECTS_FOR_SYNC if objects.empty? + raise NO_OBJECTS_FOR_SYNC if objects.empty? && TableSync.raise_on_empty_message end def publish @@ -32,6 +32,10 @@ def notify! # event: event, direction: :publish end + def empty? + objects.empty? + end + private def find_or_init_objects diff --git a/lib/table_sync/publishing/message/raw.rb b/lib/table_sync/publishing/message/raw.rb index 52aa0d3..e8e1ae8 100644 --- a/lib/table_sync/publishing/message/raw.rb +++ b/lib/table_sync/publishing/message/raw.rb @@ -23,7 +23,7 @@ def message_params def data TableSync::Publishing::Data::Raw.new( - attributes_for_sync: original_attributes, event: event, + object_class: object_class, attributes_for_sync: original_attributes, event: event, ).construct end diff --git a/lib/table_sync/publishing/message/single.rb b/lib/table_sync/publishing/message/single.rb index 5dbe7a5..3c48749 100644 --- a/lib/table_sync/publishing/message/single.rb +++ b/lib/table_sync/publishing/message/single.rb @@ -4,6 +4,10 @@ module TableSync::Publishing::Message class Single < Base private + def object + objects.first + end + def params TableSync::Publishing::Params::Single.new(object: object).construct end diff --git a/lib/table_sync/publishing/params/base.rb b/lib/table_sync/publishing/params/base.rb index b745fc8..4ed0e74 100644 --- a/lib/table_sync/publishing/params/base.rb +++ b/lib/table_sync/publishing/params/base.rb @@ -18,8 +18,6 @@ def construct private - # ROUTING KEY - def calculated_routing_key if TableSync.routing_key_callable TableSync.routing_key_callable.call(object_class, attrs_for_routing_key) @@ -28,12 +26,6 @@ def calculated_routing_key end end - def attrs_for_routing_key - {} - end - - # HEADERS - def calculated_headers if TableSync.headers_callable TableSync.headers_callable.call(object_class, attrs_for_headers) @@ -42,10 +34,6 @@ def calculated_headers end end - def attrs_for_headers - {} - end - # NOT IMPLEMENTED # name of the model being synced in the string format @@ -64,5 +52,13 @@ def headers def exchange_name raise NotImplementedError end + + def attrs_for_routing_key + raise NotImplementedError + end + + def attrs_for_headers + raise NotImplementedError + end end end \ No newline at end of file diff --git a/lib/table_sync/publishing/params/batch.rb b/lib/table_sync/publishing/params/batch.rb index e6a9b2b..9466ef2 100644 --- a/lib/table_sync/publishing/params/batch.rb +++ b/lib/table_sync/publishing/params/batch.rb @@ -10,4 +10,14 @@ class Batch < Base attribute :routing_key, default: -> { calculated_routing_key } attribute :headers, default: -> { calculated_headers } end + + private + + def attrs_for_routing_key + {} + end + + def attrs_for_headers + {} + end end diff --git a/lib/table_sync/publishing/params/single.rb b/lib/table_sync/publishing/params/single.rb index c6d51e3..80beb15 100644 --- a/lib/table_sync/publishing/params/single.rb +++ b/lib/table_sync/publishing/params/single.rb @@ -4,7 +4,7 @@ module TableSync::Publishing::Params class Single < Base attr_reader :object, :routing_key, :headers - def initialize(object) + def initialize(object:) @object = object @routing_key = calculated_routing_key @headers = calculated_headers @@ -13,19 +13,23 @@ def initialize(object) private def object_class - object.class.name + object.object_class.name end def attrs_for_routing_key - return object.attrs_for_routing_key if object.respond_to?(:attrs_for_routing_key) - - super + if object.respond_to?(:attrs_for_routing_key) + object.attrs_for_routing_key + else + object.attributes + end end def attrs_for_headers - return object.attrs_for_headers if object.respond_to?(:attrs_for_headers) - - super + if object.respond_to?(:attrs_for_headers) + object.attrs_for_headers + else + object.attributes + end end def exchange_name diff --git a/lib/table_sync/publishing/raw.rb b/lib/table_sync/publishing/raw.rb index ae5e3dd..37e868f 100644 --- a/lib/table_sync/publishing/raw.rb +++ b/lib/table_sync/publishing/raw.rb @@ -12,7 +12,11 @@ class TableSync::Publishing::Raw attribute :event, default: :update def publish_now - TableSync::Publishing::Message::Raw.new(attributes).publish + message.publish + end + + def message + TableSync::Publishing::Message::Raw.new(attributes) end end diff --git a/lib/table_sync/publishing/single.rb b/lib/table_sync/publishing/single.rb index d9c6cec..d263015 100644 --- a/lib/table_sync/publishing/single.rb +++ b/lib/table_sync/publishing/single.rb @@ -2,6 +2,7 @@ class TableSync::Publishing::Single include Tainbox + include Memery attribute :object_class attribute :original_attributes @@ -14,20 +15,31 @@ def publish_later end def publish_now - # # Update request and object does not exist -> skip publishing - # return if !object && !destroyed? + message.publish unless message.empty? && upsert_event? + end - TableSync::Publishing::Message::Single.new(attributes).publish + memoize def message + TableSync::Publishing::Message::Single.new(attributes) end private + def upsert_event? + event.in?(%i[update create]) + end + # DEBOUNCE + + # TO DO # JOB def job - TableSync.single_publishing_job # || raise NoJobClassError.new("single") + if TableSync.single_publishing_job_class_callable + TableSync.single_publishing_job_class_callable&.call + else + raise TableSync::NoJobClassError.new("single") + end end def job_attributes diff --git a/spec/publishing/batch_spec.rb b/spec/publishing/batch_spec.rb new file mode 100644 index 0000000..b0f97fc --- /dev/null +++ b/spec/publishing/batch_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +RSpec.describe TableSync::Publishing::Batch do + let(:original_attributes) { [{ id: 1, time: Time.current }] } + let(:serialized_attributes) { [{ id: 1 }] } + + let(:attributes) do + { + object_class: "User", + original_attributes: original_attributes, + event: :update, + headers: { some_arg: 1 }, + routing_key: "custom_key123", + } + end + + include_examples "publisher#publish_now calls stubbed message with attributes", + TableSync::Publishing::Message::Batch + + context "#publish_later" do + let(:job) { double("BatchJob", perform_later: 1) } + + include_examples "publisher#publish_later behaviour" + end +end diff --git a/spec/publishing/helpers/attributes_spec.rb b/spec/publishing/helpers/attributes_spec.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/publishing/helpers/debounce_spec.rb b/spec/publishing/helpers/debounce_spec.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/publishing/helpers/objects_spec.rb b/spec/publishing/helpers/objects_spec.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/publishing/message/batch_spec.rb b/spec/publishing/message/batch_spec.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/publishing/message/raw_spec.rb b/spec/publishing/message/raw_spec.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/publishing/message/single_spec.rb b/spec/publishing/message/single_spec.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/publishing/raw_spec.rb b/spec/publishing/raw_spec.rb new file mode 100644 index 0000000..1b7946b --- /dev/null +++ b/spec/publishing/raw_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +RSpec.describe TableSync::Publishing::Raw do + let(:attributes) do + { + object_class: "NonExistentClass", + original_attributes: [{ id: 1, name: "purum" }], + routing_key: "custom_routing_key", + headers: { some_key: "123" }, + event: :create, + } + end + + include_examples "publisher#publish_now calls stubbed message with attributes", + TableSync::Publishing::Message::Raw +end diff --git a/spec/publishing/single_spec.rb b/spec/publishing/single_spec.rb new file mode 100644 index 0000000..a4834f9 --- /dev/null +++ b/spec/publishing/single_spec.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +TableSync.orm = :sequel + +# check for active record? + +class User < Sequel::Model; end + +RSpec.describe TableSync::Publishing::Single do + let(:original_attributes) { { id: 1, time: Time.current } } + let(:serialized_attributes) { { id: 1 } } + let(:event) { :update } + + let(:attributes) do + { + object_class: "User", + original_attributes: original_attributes, + event: event, + debounce_time: 30, + } + end + + include_examples "publisher#publish_now calls stubbed message with attributes", + TableSync::Publishing::Message::Single + + context "#publish_later" do + context "empty message" do + let(:original_attributes) { { id: 1 } } + + before do + TableSync.routing_key_callable = -> (klass, attributes) { klass } + TableSync.headers_callable = -> (klass, attributes) { klass } + end + + context "create" do + let(:event) { :create } + + it "doesn't publish" do + expect(Rabbit).not_to receive(:publish) + described_class.new(attributes).publish_now + end + end + + context "update" do + let(:event) { :update } + + it "doesn't publish" do + expect(Rabbit).not_to receive(:publish) + described_class.new(attributes).publish_now + end + end + + context "destroy" do + let(:event) { :destroy } + + it "publishes" do + expect(Rabbit).to receive(:publish) + described_class.new(attributes).publish_now + end + end + end + end + + context "#publish_later" do + let(:job) { double("Job", perform_later: 1) } + + include_examples "publisher#publish_later behaviour" + + context "with debounce" do + it "skips job, returns nil" do + end + end + end +end + + +# class User < Sequel::Model +# end + +# TableSync.orm = :sequel + +# TableSync.routing_key_callable = -> (klass, attributes) { "#{klass}_#{attributes[:ext_id]}" } +# TableSync.headers_callable = -> (klass, attributes) { "#{klass}_#{attributes[:ext_id]}" } +# TableSync.exchange_name = :test + + # context "event" do + # let(:routing_key) { "#{object_class}_#{ext_id}" } + # let(:headers) { "#{object_class}_#{ext_id}" } + + # let(:published_message) do + # { + # confirm_select: true, + # event: :table_sync, + # exchange_name: :test, + # headers: headers, + # realtime: true, + # routing_key: routing_key, + # data: { + # attributes: published_attributes, + # event: event, + # metadata: metadata, + # model: "User", + # version: anything, + # }, + # } + # end + + # shared_examples "publishes rabbit message" do + # specify do + # expect(Rabbit).to receive(:publish).with(published_message) + + # described_class.new(attributes).publish_now + # end + # end + + # shared_examples "raises No Objects Error" do + # specify do + # expect { described_class.new(attributes).publish_now } + # .to raise_error(TableSync::Publishing::Message::Base::NO_OBJECTS_FOR_SYNC) + # end + # end + + # shared_examples "has expected behaviour" do + # context "when published object exists" do + # before { User.insert(user_attributes) } + + # include_examples "publishes rabbit message" + # end + + # context "when published object DOESN'T exist" do + # include_examples "raises No Objects Error" + # end + # end + + # context "create" do + # let(:event) { :create } + # let(:metadata) { { created: true } } + # let(:published_attributes) { [a_hash_including(user_attributes)] } + + # include_examples "has expected behaviour" + # end + + # context "update" do + # let(:event) { :update } + # let(:metadata) { { created: false} } + # let(:published_attributes) { [a_hash_including(user_attributes)] } + + # include_examples "has expected behaviour" + # end + + # context "destroy" do + # let(:event) { :destroy } + # let(:metadata) { { created: false} } + # let(:published_attributes) { [user_attributes.slice(:id)] } + + # include_examples "publishes rabbit message" + # end + # end \ No newline at end of file diff --git a/spec/support/context/publishing.rb b/spec/support/context/publishing.rb new file mode 100644 index 0000000..2a8dad2 --- /dev/null +++ b/spec/support/context/publishing.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# needs let(:attributes) { #attrs } +shared_examples "publisher#publish_now calls stubbed message with attributes" do |message_class| + describe "#publish_now" do + context "with stubbed message" do + let(:event) { :update } + let(:message_double) { double("Message") } + + before do + allow(message_class).to receive(:new).and_return(message_double) + allow(message_double).to receive(:publish) + allow(message_double).to receive(:empty?).and_return(false) + end + + it "initializes message with correct parameters" do + expect(message_class).to receive(:new).with(attributes) + expect(message_double).to receive(:publish) + + described_class.new(attributes).publish_now + end + end + end +end + +# needs let(:job) with perform_later defined +# needs let(:attributes) +# needs let(:original_attributes) with unserializable value +# needs let(:serialized_attributes) -> original_attributes without unserializable value + +shared_examples "publisher#publish_later behaviour" do + describe "#publish_later" do + context "with defined job" do + before do + TableSync.batch_publishing_job_class_callable = -> { job } + TableSync.single_publishing_job_class_callable = -> { job } + end + + it "calls BatchJob with serialized original_attributes" do + expect(job).to receive(:perform_later).with( + attributes.merge(original_attributes: serialized_attributes) + ) + + described_class.new(attributes).publish_later + end + end + + context "without defined job" do + before do + TableSync.batch_publishing_job_class_callable = nil + TableSync.single_publishing_job_class_callable = nil + end + + it "raises no job error" do + expect { described_class.new(attributes).publish_later } + .to raise_error(TableSync::NoJobClassError) + end + end + end +end diff --git a/spec/support/table_sync_settings.rb b/spec/support/table_sync_settings.rb index 0ab5d49..730fccb 100644 --- a/spec/support/table_sync_settings.rb +++ b/spec/support/table_sync_settings.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true TableSync.orm = :active_record -TableSync.publishing_job_class_callable = -> { TestJob } +TableSync.single_publishing_job_class_callable = -> { TestJob } From cb6559c18497caf554b6126acf4f6f919ee1c009 Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Tue, 17 Aug 2021 15:09:32 +0300 Subject: [PATCH 05/20] debounce, specs --- lib/table_sync.rb | 10 +- lib/table_sync/errors.rb | 15 +- lib/table_sync/orm_adapter/active_record.rb | 88 +----- lib/table_sync/orm_adapter/base.rb | 36 ++- lib/table_sync/orm_adapter/sequel.rb | 8 +- lib/table_sync/publishing.rb | 2 +- lib/table_sync/publishing/batch.rb | 2 +- lib/table_sync/publishing/data/objects.rb | 10 +- lib/table_sync/publishing/data/raw.rb | 26 +- .../publishing/helpers/attributes.rb | 18 +- lib/table_sync/publishing/helpers/debounce.rb | 138 +++++++-- lib/table_sync/publishing/helpers/objects.rb | 2 +- lib/table_sync/publishing/message/base.rb | 36 +-- lib/table_sync/publishing/message/batch.rb | 20 +- lib/table_sync/publishing/message/raw.rb | 22 +- lib/table_sync/publishing/message/single.rb | 18 +- lib/table_sync/publishing/params/base.rb | 14 +- lib/table_sync/publishing/params/batch.rb | 26 +- lib/table_sync/publishing/params/raw.rb | 8 +- lib/table_sync/publishing/params/single.rb | 12 +- lib/table_sync/publishing/publisher.rb | 129 -------- lib/table_sync/publishing/raw.rb | 10 +- lib/table_sync/publishing/single.rb | 33 +- lib/table_sync/setup/active_record.rb | 29 +- lib/table_sync/setup/base.rb | 34 +-- lib/table_sync/setup/sequel.rb | 29 +- spec/instrument/publish_spec.rb | 20 +- spec/orm_adapter/active_record_spec.rb | 5 + spec/orm_adapter/sequel_spec.rb | 5 + spec/publishing/batch_publishing_spec.rb | 288 ------------------ spec/publishing/batch_spec.rb | 55 +++- spec/publishing/data/objects_spec.rb | 101 ++++++ spec/publishing/data/raw_spec.rb | 45 +++ spec/publishing/helpers/attributes_spec.rb | 55 ++++ spec/publishing/helpers/debounce_spec.rb | 103 +++++++ spec/publishing/helpers/objects_spec.rb | 44 +++ spec/publishing/message/batch_spec.rb | 76 +++++ spec/publishing/message/raw_spec.rb | 63 ++++ spec/publishing/message/single_spec.rb | 74 +++++ spec/publishing/params/batch_spec.rb | 111 +++++++ spec/publishing/params/raw_spec.rb | 111 +++++++ spec/publishing/params/single_spec.rb | 145 +++++++++ spec/publishing/publishing_spec.rb | 246 --------------- spec/publishing/raw_spec.rb | 29 +- spec/publishing/sequel/adapter_spec.rb | 107 ------- spec/publishing/single_spec.rb | 240 ++++++--------- spec/setup/active_record_spec.rb | 29 ++ spec/setup/sequel_spec.rb | 30 ++ spec/spec_helper.rb | 4 + spec/support/001_setup.rb | 41 --- spec/support/active_job_settings.rb | 3 + spec/support/context/publishing.rb | 60 ---- spec/support/database_settings.rb | 51 ++++ spec/support/shared/adapters.rb | 171 +++++++++++ spec/support/shared/publishers.rb | 98 ++++++ spec/support/shared/publishing.rb | 21 ++ spec/support/shared/setup.rb | 67 ++++ spec/support/table_sync_settings.rb | 15 +- 58 files changed, 1948 insertions(+), 1340 deletions(-) delete mode 100644 lib/table_sync/publishing/publisher.rb create mode 100644 spec/orm_adapter/active_record_spec.rb create mode 100644 spec/orm_adapter/sequel_spec.rb delete mode 100644 spec/publishing/batch_publishing_spec.rb create mode 100644 spec/publishing/data/objects_spec.rb create mode 100644 spec/publishing/data/raw_spec.rb create mode 100644 spec/publishing/params/batch_spec.rb create mode 100644 spec/publishing/params/raw_spec.rb create mode 100644 spec/publishing/params/single_spec.rb delete mode 100644 spec/publishing/publishing_spec.rb delete mode 100644 spec/publishing/sequel/adapter_spec.rb create mode 100644 spec/setup/active_record_spec.rb create mode 100644 spec/setup/sequel_spec.rb delete mode 100644 spec/support/001_setup.rb delete mode 100644 spec/support/context/publishing.rb create mode 100644 spec/support/shared/adapters.rb create mode 100644 spec/support/shared/publishers.rb create mode 100644 spec/support/shared/publishing.rb create mode 100644 spec/support/shared/setup.rb diff --git a/lib/table_sync.rb b/lib/table_sync.rb index efa2bf8..b61efac 100644 --- a/lib/table_sync.rb +++ b/lib/table_sync.rb @@ -44,13 +44,13 @@ class << self attr_reader :receiving_model attr_reader :setup - def sync(object_class, on: nil, if_condition: nil, unless_condition: nil, debounce_time: nil) + def sync(object_class, **options) setup.new( object_class: object_class, - on: on, - if_condition: if_condition, - unless_condition: unless_condition, - debounce_time: debounce_time, + on: options[:on], + if_condition: options[:if], + unless_condition: options[:unless], + debounce_time: options[:debounce_time], ).register_callbacks end diff --git a/lib/table_sync/errors.rb b/lib/table_sync/errors.rb index fe3c53a..6ecfd14 100644 --- a/lib/table_sync/errors.rb +++ b/lib/table_sync/errors.rb @@ -3,11 +3,20 @@ module TableSync Error = Class.new(StandardError) - class NoJobClassError < Error + class NoPrimaryKeyError < Error + def initialize(object_class, object_data, primary_key_columns) + super(<<~MSG) + Can't find or init an object of #{object_class} with #{object_data.inspect}. + Incomplete primary key! object_data must contain: #{primary_key_columns.inspect}. + MSG + end + end + + class NoCallableError < Error def initialize(type) super(<<~MSG) - Can't find job class for publishing! - Please initialize TableSync.#{type}_publishing_job_class_callable with the correct proc! + Can't find callable for #{type}! + Please initialize TableSync.#{type}_callable with the correct proc! MSG end end diff --git a/lib/table_sync/orm_adapter/active_record.rb b/lib/table_sync/orm_adapter/active_record.rb index 66ad790..e9f9f5a 100644 --- a/lib/table_sync/orm_adapter/active_record.rb +++ b/lib/table_sync/orm_adapter/active_record.rb @@ -3,7 +3,7 @@ module TableSync::ORMAdapter class ActiveRecord < Base def primary_key - object.pk_hash + object_class.primary_key end def find @@ -13,87 +13,11 @@ def find end def attributes - object.attributes + object.attributes.symbolize_keys + end + + def self.model_naming(object_class) + TableSync::NamingResolver::ActiveRecord.new(table_name: object_class.table_name) end end end - - -# module TableSync::ORMAdapter -# module ActiveRecord -# module_function - -# def model_naming(object) -# ::TableSync::NamingResolver::ActiveRecord.new(table_name: object.table_name) -# end - -# def find(dataset, conditions) -# dataset.find_by(conditions) -# end - -# def attributes(object) -# object.attributes -# end -# end -# end - - -# frozen_string_literal: true - -# class TableSync::ORMAdapter::Sequel -# attr_reader :object, :object_class, :object_data - -# def initialize(object_class, object_data) -# @object_class = object_class -# @object_data = object_data - -# validate! -# end - -# def init -# @object = object_class.new(object_data) -# end - -# def find -# @object = object_class.find(needle) -# end - -# def needle -# object_data.slice(*primary_key_columns) -# end - -# def validate! -# if (primary_key_columns - object_data.keys).any? -# raise NoPrimaryKeyError.new(object_class, object_data, primary_key_columns) -# end -# end - -# def primary_key_columns -# Array.wrap(object_class.primary_key) -# end - -# def primary_key -# object.pk_hash -# end - -# def attributes -# object.values -# end - -# def attributes_for_update -# if object.respond_to?(:attributes_for_sync) -# object.attributes_for_sync -# else -# attributes -# end -# end - -# def attributes_for_destroy -# if object_class.respond_to?(:table_sync_destroy_attributes) -# object_class.table_sync_destroy_attributes(attributes) -# else -# primary_key -# end -# end -# end -# end diff --git a/lib/table_sync/orm_adapter/base.rb b/lib/table_sync/orm_adapter/base.rb index bc657b6..69898ed 100644 --- a/lib/table_sync/orm_adapter/base.rb +++ b/lib/table_sync/orm_adapter/base.rb @@ -6,16 +6,16 @@ class Base def initialize(object_class, object_data) @object_class = object_class - @object_data = object_data + @object_data = object_data.symbolize_keys validate! end # VALIDATE - + def validate! if (primary_key_columns - object_data.keys).any? - raise NoPrimaryKeyError.new(object_class, object_data, primary_key_columns) + raise TableSync::NoPrimaryKeyError.new(object_class, object_data, primary_key_columns) end end @@ -32,8 +32,6 @@ def init end def find - # @object = object_class.find(needle) - self end @@ -42,7 +40,7 @@ def needle end def primary_key_columns - Array.wrap(object_class.primary_key).map(&:to_sym) # temp! + Array.wrap(object_class.primary_key).map(&:to_sym) end # ATTRIBUTES @@ -56,10 +54,26 @@ def attributes_for_update end def attributes_for_destroy - if object_class.respond_to?(:table_sync_destroy_attributes) - object_class.table_sync_destroy_attributes(attributes) + if object.respond_to?(:attributes_for_destroy) + object.attributes_for_destroy + else + needle + end + end + + def attributes_for_routing_key + if object.respond_to?(:attributes_for_routing_key) + object.attributes_for_routing_key else - primary_key + attributes + end + end + + def attributes_for_headers + if object.respond_to?(:attributes_for_headers) + object.attributes_for_headers + else + attributes end end @@ -78,5 +92,9 @@ def primary_key def attributes raise NotImplementedError end + + def self.model_naming + raise NotImplementedError + end end end diff --git a/lib/table_sync/orm_adapter/sequel.rb b/lib/table_sync/orm_adapter/sequel.rb index 3c027e8..1552694 100644 --- a/lib/table_sync/orm_adapter/sequel.rb +++ b/lib/table_sync/orm_adapter/sequel.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module TableSync::ORMAdapter - class Sequel < Base + class Sequel < Base def primary_key object.pk_hash end @@ -15,5 +15,11 @@ def find super end + + def self.model_naming(object_class) + TableSync::NamingResolver::Sequel.new( + table_name: object_class.table_name, db: object_class.db, + ) + end end end diff --git a/lib/table_sync/publishing.rb b/lib/table_sync/publishing.rb index 49c898a..3c8e0c9 100644 --- a/lib/table_sync/publishing.rb +++ b/lib/table_sync/publishing.rb @@ -2,7 +2,7 @@ module TableSync module Publishing - require_relative "publishing/data/objects" + require_relative "publishing/data/objects" require_relative "publishing/data/raw" require_relative "publishing/helpers/attributes" diff --git a/lib/table_sync/publishing/batch.rb b/lib/table_sync/publishing/batch.rb index feeeb7e..6d018c0 100644 --- a/lib/table_sync/publishing/batch.rb +++ b/lib/table_sync/publishing/batch.rb @@ -27,7 +27,7 @@ def job if TableSync.batch_publishing_job_class_callable TableSync.batch_publishing_job_class_callable.call else - raise TableSync::NoJobClassError.new("batch") + raise TableSync::NoCallableError.new("batch_publishing_job_class") end end diff --git a/lib/table_sync/publishing/data/objects.rb b/lib/table_sync/publishing/data/objects.rb index 75b2a46..9084fac 100644 --- a/lib/table_sync/publishing/data/objects.rb +++ b/lib/table_sync/publishing/data/objects.rb @@ -11,18 +11,18 @@ def initialize(objects:, event:) def construct { - model: model, + model: model, attributes: attributes_for_sync, - version: version, - event: event, - metadata: metadata, + version: version, + event: event, + metadata: metadata, } end private def model - if object_class.method_defined?(:table_sync_model_name) + if object_class.respond_to?(:table_sync_model_name) object_class.table_sync_model_name else object_class.name diff --git a/lib/table_sync/publishing/data/raw.rb b/lib/table_sync/publishing/data/raw.rb index af676e7..094e7ac 100644 --- a/lib/table_sync/publishing/data/raw.rb +++ b/lib/table_sync/publishing/data/raw.rb @@ -2,23 +2,31 @@ # check if works! module TableSync::Publishing::Data - class Raw - attr_reader :object_class, :attributes_for_sync, :event + class Raw + attr_reader :object_class, :attributes_for_sync, :event def initialize(object_class:, attributes_for_sync:, event:) - @object_class = object_class + @object_class = object_class @attributes_for_sync = attributes_for_sync - @event = event + @event = event end def construct { - model: object_class,# model, + model: object_class, attributes: attributes_for_sync, - version: Time.current.to_f,#version, - event: event, - metadata: {}, #metadata, + version: version, + event: event, + metadata: metadata, } end - end + + def metadata + { created: event == :create } # remove? who needs this? + end + + def version + Time.current.to_f + end + end end diff --git a/lib/table_sync/publishing/helpers/attributes.rb b/lib/table_sync/publishing/helpers/attributes.rb index 7415506..7739323 100644 --- a/lib/table_sync/publishing/helpers/attributes.rb +++ b/lib/table_sync/publishing/helpers/attributes.rb @@ -3,14 +3,16 @@ module TableSync::Publishing::Helpers class Attributes BASE_SAFE_JSON_TYPES = [ - NilClass, - String, - TrueClass, - FalseClass, - Numeric, - Symbol, + NilClass, + String, + TrueClass, + FalseClass, + Numeric, + Symbol, ].freeze + # add custom seializables? + NOT_MAPPED = Object.new attr_reader :attributes @@ -20,7 +22,7 @@ def initialize(attributes) end def serialize - filter_safe_for_serialization(attributes) + filter_safe_for_serialization(attributes) end def filter_safe_for_serialization(object) @@ -58,4 +60,4 @@ def object_mapped?(object) object != NOT_MAPPED end end -end \ No newline at end of file +end diff --git a/lib/table_sync/publishing/helpers/debounce.rb b/lib/table_sync/publishing/helpers/debounce.rb index ae5d7c3..f480a80 100644 --- a/lib/table_sync/publishing/helpers/debounce.rb +++ b/lib/table_sync/publishing/helpers/debounce.rb @@ -1,60 +1,134 @@ # frozen_string_literal: true +# CASES FOR DEBOUNCE + +# Cached Sync Time -> CST - time last sync has occured or next sync will occur +# Next Sync Time -> NST - time next sync will occur + +# 0 +# Condition: debounce_time is zero. +# No debounce, sync right now. +# Result: NST -> Time.current + +# 1 +# Condition: CST is empty. +# There was no sync before. Can be synced right now. +# Result: NST -> Time.current + +# 2 +# Condition: CST =< Time.current. +# There was a sync before. + +# 2.1 +# Subcondition: CST + debounce_time <= Time.current +# Enough time passed for debounce condition to be satisfied. +# No need to wait. Can by synced right now. +# Result: NST -> Time.current + +# 2.2 +# Subcondition: CST + debounce_time > Time.current +# Debounce condition is not satisfied. Must wait till debounce period has passed. +# Will be synced after debounce period has passed. +# Result: NST -> CST + debounce_time + +# 3 +# Condition: CST > Time.current +# Sync job has already been enqueued in the future. + +# 3.1 +# Subcondition: event -> update | create +# No need to sync upsert event, since it has already enqueued sync in the future. +# It will sync fresh version anyway. +# NST -> Skip, no sync. + +# 3.2 +# Subcondition: event -> destroy +# In this case the already enqueued job must be upsert. +# Thus destructive sync has to send message after upsert sync. +# NST -> CST + debounce_time + module TableSync::Publishing::Helpers class Debounce - include Memery + include Memery - attr_reader :debounce_time, :object_class, :needle + DEFAULT_TIME = 60 - def initialize(object_class:, needle:, debounce_time: nil) - @debounce_time = debounce_time + attr_reader :debounce_time, :object_class, :needle, :event + + def initialize(object_class:, needle:, event:, debounce_time: nil) + @event = event + @debounce_time = debounce_time || DEFAULT_TIME @object_class = object_class @needle = needle end - def sync_time? - no_last_sync_time? || past_next_sync_time? + def skip? + true if sync_in_the_future? && upsert_event? # case 3.1 end - # No sync before, no need for debounce - def no_last_sync_time? - last_sync_time.nil? + memoize def next_sync_time + return current_time if debounce_time.zero? # case 0 + return current_time if no_sync_before? # case 1 + + return current_time if sync_in_the_past? && debounce_time_passed? # case 2.1 + return debounced_sync_time if sync_in_the_past? && debounce_time_not_passed? # case 2.2 + + return debounced_sync_time if sync_in_the_future? && destroy_event? # case 3.2 end - def past_next_sync_time? + # CASE 1 + def no_sync_before? + cached_sync_time.nil? + end + # CASE 2 + def sync_in_the_past? + cached_sync_time <= current_time end - memoize def last_sync_time - Rails.cache.read(cache_key) + def debounce_time_passed? + cached_sync_time + debounce_time.seconds >= current_time end - memoize def current_time - Time.current + def debounce_time_not_passed? + cached_sync_time + debounce_time.seconds < current_time end - def save_next_sync_time(time) - Rails.cache.write(cache_key, next_sync_time) + # CASE 3 + def sync_in_the_future? + cached_sync_time && (cached_sync_time > current_time) end - def cache_key - "#{object_class}/#{needle.values.join}_table_sync_time".delete(" ") + def destroy_event? + event == :destroy + end + + def upsert_event? + !destroy_event? + end + + # MISC + + def debounced_sync_time + cached_sync_time + debounce_time.seconds + end + + memoize def current_time + Time.current end - end -end - # def publish - # return enqueue_job if destroyed? || debounce_time.zero? + # CACHE - # sync_time = Rails.cache.read(cache_key) || current_time - debounce_time - 1.second - # return if sync_time > current_time + memoize def cached_sync_time + Rails.cache.read(cache_key) + end - # next_sync_time = sync_time + debounce_time - # next_sync_time <= current_time ? enqueue_job : enqueue_job(next_sync_time) - # end + def cache_next_sync_time + Rails.cache.write(cache_key, next_sync_time) + end - # def enqueue_job(perform_at = current_time) - # job = job_class.set(wait_until: perform_at) - # job.perform_later(object_class.name, original_attributes, state: state.to_s, confirm: confirm?) - # Rails.cache.write(cache_key, perform_at) - # end + def cache_key + "#{object_class}/#{needle.values.join}_table_sync_time".delete(" ") + end + end +end diff --git a/lib/table_sync/publishing/helpers/objects.rb b/lib/table_sync/publishing/helpers/objects.rb index 2082ca3..e7b5386 100644 --- a/lib/table_sync/publishing/helpers/objects.rb +++ b/lib/table_sync/publishing/helpers/objects.rb @@ -12,7 +12,7 @@ def initialize(object_class:, original_attributes:, event:) def construct_list if destruction? - without_empty_objects(init_objects) + init_objects else without_empty_objects(find_objects) end diff --git a/lib/table_sync/publishing/message/base.rb b/lib/table_sync/publishing/message/base.rb index b04898c..542573a 100644 --- a/lib/table_sync/publishing/message/base.rb +++ b/lib/table_sync/publishing/message/base.rb @@ -3,7 +3,7 @@ module TableSync::Publishing::Message class Base include Tainbox - + NO_OBJECTS_FOR_SYNC = Class.new(StandardError) attr_reader :objects @@ -26,30 +26,16 @@ def publish notify! end - def notify! - # model_naming = TableSync.publishing_adapter.model_naming(object_class) - # TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema, - # event: event, direction: :publish - end - def empty? objects.empty? end - private - def find_or_init_objects TableSync::Publishing::Helpers::Objects.new( object_class: object_class, original_attributes: original_attributes, event: event, ).construct_list end - def data - TableSync::Publishing::Data::Objects.new( - objects: objects, event: event - ).construct - end - # MESSAGE PARAMS def message_params @@ -58,12 +44,28 @@ def message_params def data TableSync::Publishing::Data::Objects.new( - objects: objects, event: event + objects: objects, event: event, ).construct end def params raise NotImplementedError end + + # NOTIFY + + def notify! + TableSync::Instrument.notify( + table: model_naming.table, + schema: model_naming.schema, + event: event, + direction: :publish, + count: objects.count, + ) + end + + def model_naming + TableSync.publishing_adapter.model_naming(objects.first.object_class) + end end -end \ No newline at end of file +end diff --git a/lib/table_sync/publishing/message/batch.rb b/lib/table_sync/publishing/message/batch.rb index bed746f..f05e0e0 100644 --- a/lib/table_sync/publishing/message/batch.rb +++ b/lib/table_sync/publishing/message/batch.rb @@ -1,16 +1,14 @@ # frozen_string_literal: true module TableSync::Publishing::Message - class Batch < Base - attribute :headers - attribute :routing_key + class Batch < Base + attribute :headers + attribute :routing_key - private - - def params - TableSync::Publishing::Params::Batch.new( - object_class: object_class, headers: headers, routing_key: routing_key - ).construct - end - end + def params + TableSync::Publishing::Params::Batch.new( + object_class: object_class, headers: headers, routing_key: routing_key, + ).construct + end + end end diff --git a/lib/table_sync/publishing/message/raw.rb b/lib/table_sync/publishing/message/raw.rb index e8e1ae8..3cf0a69 100644 --- a/lib/table_sync/publishing/message/raw.rb +++ b/lib/table_sync/publishing/message/raw.rb @@ -13,9 +13,27 @@ class Raw def publish Rabbit.publish(message_params) + + notify! + end + + # NOTIFY + + def notify! + TableSync::Instrument.notify( + table: model_naming.table, + schema: model_naming.schema, + event: event, + count: original_attributes.count, + direction: :publish, + ) + end + + def model_naming + TableSync.publishing_adapter.model_naming(object_class.constantize) end - private + # MESSAGE PARAMS def message_params params.merge(data: data) @@ -33,4 +51,4 @@ def params ).construct end end -end \ No newline at end of file +end diff --git a/lib/table_sync/publishing/message/single.rb b/lib/table_sync/publishing/message/single.rb index 3c48749..3b02d53 100644 --- a/lib/table_sync/publishing/message/single.rb +++ b/lib/table_sync/publishing/message/single.rb @@ -1,15 +1,13 @@ # frozen_string_literal: true module TableSync::Publishing::Message - class Single < Base - private + class Single < Base + def object + objects.first + end - def object - objects.first - end - - def params - TableSync::Publishing::Params::Single.new(object: object).construct - end - end + def params + TableSync::Publishing::Params::Single.new(object: object).construct + end + end end diff --git a/lib/table_sync/publishing/params/base.rb b/lib/table_sync/publishing/params/base.rb index 4ed0e74..bd58d4f 100644 --- a/lib/table_sync/publishing/params/base.rb +++ b/lib/table_sync/publishing/params/base.rb @@ -4,14 +4,14 @@ module TableSync::Publishing::Params class Base DEFAULT_PARAMS = { confirm_select: true, - realtime: true, - event: :table_sync, + realtime: true, + event: :table_sync, }.freeze def construct DEFAULT_PARAMS.merge( - routing_key: routing_key, - headers: headers, + routing_key: routing_key, + headers: headers, exchange_name: exchange_name, ) end @@ -22,7 +22,7 @@ def calculated_routing_key if TableSync.routing_key_callable TableSync.routing_key_callable.call(object_class, attrs_for_routing_key) else - raise "Can't publish, set TableSync.routing_key_callable!" + raise TableSync::NoCallableError.new("routing_key") end end @@ -30,7 +30,7 @@ def calculated_headers if TableSync.headers_callable TableSync.headers_callable.call(object_class, attrs_for_headers) else - raise "Can't publish, set TableSync.headers_callable!" + raise TableSync::NoCallableError.new("headers") end end @@ -61,4 +61,4 @@ def attrs_for_headers raise NotImplementedError end end -end \ No newline at end of file +end diff --git a/lib/table_sync/publishing/params/batch.rb b/lib/table_sync/publishing/params/batch.rb index 9466ef2..1465956 100644 --- a/lib/table_sync/publishing/params/batch.rb +++ b/lib/table_sync/publishing/params/batch.rb @@ -1,23 +1,23 @@ # frozen_string_literal: true module TableSync::Publishing::Params - class Batch < Base - include Tainbox + class Batch < Base + include Tainbox - attribute :object_class + attribute :object_class - attribute :exchange_name, default: -> { TableSync.exchange_name } - attribute :routing_key, default: -> { calculated_routing_key } - attribute :headers, default: -> { calculated_headers } - end + attribute :exchange_name, default: -> { TableSync.exchange_name } + attribute :routing_key, default: -> { calculated_routing_key } + attribute :headers, default: -> { calculated_headers } - private + private - def attrs_for_routing_key - {} - end + def attrs_for_routing_key + {} + end - def attrs_for_headers - {} + def attrs_for_headers + {} + end end end diff --git a/lib/table_sync/publishing/params/raw.rb b/lib/table_sync/publishing/params/raw.rb index 2cfff50..aed2f06 100644 --- a/lib/table_sync/publishing/params/raw.rb +++ b/lib/table_sync/publishing/params/raw.rb @@ -1,7 +1,7 @@ -# frozen_strinf_literal: true +# frozen_string_literal: true module TableSync::Publishing::Params - class Raw < Batch - # FOR NAMING CONSISTENCY - end + class Raw < Batch + # FOR NAMING CONSISTENCY + end end diff --git a/lib/table_sync/publishing/params/single.rb b/lib/table_sync/publishing/params/single.rb index 80beb15..aebdf67 100644 --- a/lib/table_sync/publishing/params/single.rb +++ b/lib/table_sync/publishing/params/single.rb @@ -17,19 +17,11 @@ def object_class end def attrs_for_routing_key - if object.respond_to?(:attrs_for_routing_key) - object.attrs_for_routing_key - else - object.attributes - end + object.attributes_for_routing_key end def attrs_for_headers - if object.respond_to?(:attrs_for_headers) - object.attrs_for_headers - else - object.attributes - end + object.attributes_for_headers end def exchange_name diff --git a/lib/table_sync/publishing/publisher.rb b/lib/table_sync/publishing/publisher.rb deleted file mode 100644 index c9d36d3..0000000 --- a/lib/table_sync/publishing/publisher.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true - -class TableSync::Publishing::Publisher < TableSync::Publishing::BasePublisher - DEBOUNCE_TIME = 1.minute - - # 'original_attributes' are not published, they are used to resolve the routing key - def initialize(object_class, original_attributes, **opts) - @object_class = object_class.constantize - @original_attributes = filter_safe_for_serialization(original_attributes.deep_symbolize_keys) - @confirm = opts.fetch(:confirm, true) - @debounce_time = opts[:debounce_time]&.seconds || DEBOUNCE_TIME - @state = opts.fetch(:state, :updated).to_sym - validate_state - end - - def publish - return enqueue_job if destroyed? || debounce_time.zero? - - sync_time = Rails.cache.read(cache_key) || current_time - debounce_time - 1.second - return if sync_time > current_time - - next_sync_time = sync_time + debounce_time - next_sync_time <= current_time ? enqueue_job : enqueue_job(next_sync_time) - end - - def publish_now - # Update request and object does not exist -> skip publishing - return if !object && !destroyed? - - Rabbit.publish(params) - model_naming = TableSync.publishing_adapter.model_naming(object_class) - TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema, - event: event, direction: :publish - end - - private - - attr_reader :original_attributes - attr_reader :state - attr_reader :debounce_time - - def attrs_for_callables - attributes_for_sync - end - - def attrs_for_routing_key - if object.respond_to?(:attrs_for_routing_key) - object.attrs_for_routing_key - else - attrs_for_callables - end - end - - def attrs_for_metadata - if object.respond_to?(:attrs_for_metadata) - object.attrs_for_metadata - else - attrs_for_callables - end - end - - def job_callable - TableSync.publishing_job_class_callable - end - - def job_callable_error_message - "Can't publish, set TableSync.publishing_job_class_callable" - end - - def enqueue_job(perform_at = current_time) - job = job_class.set(wait_until: perform_at) - job.perform_later(object_class.name, original_attributes, state: state.to_s, confirm: confirm?) - Rails.cache.write(cache_key, perform_at) - end - - def routing_key - resolve_routing_key - end - - def publishing_data - { - **super, - event: event, - metadata: { created: created? }, - } - end - - memoize def attributes_for_sync - if destroyed? - if object_class.respond_to?(:table_sync_destroy_attributes) - object_class.table_sync_destroy_attributes(original_attributes) - else - original_attributes - end - elsif attributes_for_sync_defined? - object.attributes_for_sync - else - TableSync.publishing_adapter.attributes(object) - end - end - - memoize def object - TableSync.publishing_adapter.find(object_class, needle) - end - - def event - destroyed? ? :destroy : :update - end - - def needle - original_attributes.slice(*primary_keys) - end - - def cache_key - "#{object_class}/#{needle}_table_sync_time".delete(" ") - end - - def destroyed? - state == :destroyed - end - - def created? - state == :created - end - - def validate_state - raise "Unknown state: #{state.inspect}" unless %i[created updated destroyed].include?(state) - end -end diff --git a/lib/table_sync/publishing/raw.rb b/lib/table_sync/publishing/raw.rb index 37e868f..be2196d 100644 --- a/lib/table_sync/publishing/raw.rb +++ b/lib/table_sync/publishing/raw.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true class TableSync::Publishing::Raw - include Tainbox + include Tainbox - attribute :object_class + attribute :object_class attribute :original_attributes attribute :routing_key @@ -12,11 +12,11 @@ class TableSync::Publishing::Raw attribute :event, default: :update def publish_now - message.publish + message.publish end def message - TableSync::Publishing::Message::Raw.new(attributes) + TableSync::Publishing::Message::Raw.new(attributes) end end @@ -37,4 +37,4 @@ def message # changes -# add validations? \ No newline at end of file +# add validations? diff --git a/lib/table_sync/publishing/single.rb b/lib/table_sync/publishing/single.rb index d263015..b3acba8 100644 --- a/lib/table_sync/publishing/single.rb +++ b/lib/table_sync/publishing/single.rb @@ -7,44 +7,53 @@ class TableSync::Publishing::Single attribute :object_class attribute :original_attributes - attribute :event, default: :update - attribute :debounce_time, default: 60 + attribute :event, default: :update + attribute :debounce_time + # expect job to have perform_at method + # debounce destroyed event + # because otherwise update event could be sent after destroy def publish_later - job.perform_later(job_attributes) + return if debounce.skip? + + job.perform_at(job_attributes) + + debounce.cache_next_sync_time end def publish_now - message.publish unless message.empty? && upsert_event? + message.publish unless message.empty? end memoize def message TableSync::Publishing::Message::Single.new(attributes) end - private - - def upsert_event? - event.in?(%i[update create]) + memoize def debounce + TableSync::Publishing::Helpers::Debounce.new( + object_class: object_class, + needle: message.object.needle, + debounce_time: debounce_time, + event: event, + ) end - # DEBOUNCE + private - # TO DO - # JOB def job if TableSync.single_publishing_job_class_callable TableSync.single_publishing_job_class_callable&.call else - raise TableSync::NoJobClassError.new("single") + raise TableSync::NoCallableError.new("single_publishing_job_class") end end def job_attributes attributes.merge( original_attributes: serialized_original_attributes, + perform_at: debounce.next_sync_time, ) end diff --git a/lib/table_sync/setup/active_record.rb b/lib/table_sync/setup/active_record.rb index dfa10d1..a5bf3d5 100644 --- a/lib/table_sync/setup/active_record.rb +++ b/lib/table_sync/setup/active_record.rb @@ -1,16 +1,23 @@ # frozen-string_literal: true module TableSync::Setup - class ActiveRecord < Base - private + class ActiveRecord < Base + private - def define_after_commit_on(event) - after_commit(on: event) do - return if not if_condition.call(self) - return if unless_condition.call(self) + def define_after_commit(event) + options = options_exposed_for_block - enqueue_message(self.attributes) - end - end - end -end \ No newline at end of file + object_class.after_commit(on: event) do + next unless options[:if].call(self) + next if options[:unless].call(self) + + TableSync::Publishing::Single.new( + object_class: self.class.name, + original_attributes: attributes, + event: event, + debounce_time: options[:debounce_time], + ).publish_later + end + end + end +end diff --git a/lib/table_sync/setup/base.rb b/lib/table_sync/setup/base.rb index 0b84784..edd8bd4 100644 --- a/lib/table_sync/setup/base.rb +++ b/lib/table_sync/setup/base.rb @@ -5,28 +5,29 @@ class Base include Tainbox EVENTS = %i[create update destroy].freeze - INVALID_EVENTS = Class.new(StandardError) + INVALID_EVENT = Class.new(StandardError) INVALID_CONDITION = Class.new(StandardError) attribute :object_class attribute :debounce_time - attribute :on, default: [] - attribute :if_condition, default: -> { Proc.new {} } - attribute :unless_condition, default: -> { Proc.new {} } + attribute :on + attribute :if_condition + attribute :unless_condition def initialize(attrs) super(attrs) - self.on = Array.wrap(on).map(:to_sym) + self.on = Array.wrap(on).map(&:to_sym) - raise INVALID_EVENTS unless valid_events? - raise INVALID_CONDITIONS unless valid_conditions? + self.if_condition ||= proc { true } + self.unless_condition ||= proc { false } + + raise INVALID_EVENTS unless valid_events? + raise INVALID_CONDITION unless valid_conditions? end def register_callbacks - applicable_events.each do |event| - object_class.instance_exec(&define_after_commit_on(event)) - end + applicable_events.each { |event| define_after_commit(event) } end private @@ -49,17 +50,16 @@ def applicable_events # CREATING HOOKS - def define_after_commit_on(event) + def define_after_commit(event) raise NotImplementedError end - def enqueue_message(original_attributes) - TableSync::Publishing::Single.new( - object_class: self.class.name, - original_attributes: original_attributes, - event: event, + def options_exposed_for_block + { + if: if_condition, + unless: unless_condition, debounce_time: debounce_time, - ).publish_later + } end end end diff --git a/lib/table_sync/setup/sequel.rb b/lib/table_sync/setup/sequel.rb index fbae7e0..7f6bbcf 100644 --- a/lib/table_sync/setup/sequel.rb +++ b/lib/table_sync/setup/sequel.rb @@ -1,18 +1,25 @@ # frozen-string_literal: true module TableSync::Setup - class Sequel < Base - private + class Sequel < Base + private - def define_after_commit_on(event) - define_method("after_#{event}".to_sym) do - return if not if_condition.call(self) - return if unless_condition.call(self) + def define_after_commit(event) + options = options_exposed_for_block - enqueue_message(self.values) + object_class.define_method("after_#{event}".to_sym) do + return unless options[:if].call(self) + return if options[:unless].call(self) - super() - end - end - end + TableSync::Publishing::Single.new( + object_class: self.class.name, + original_attributes: values, + event: event, + debounce_time: options[:debounce_time], + ).publish_later + + super() + end + end + end end diff --git a/spec/instrument/publish_spec.rb b/spec/instrument/publish_spec.rb index e2d69cc..ae9aa87 100644 --- a/spec/instrument/publish_spec.rb +++ b/spec/instrument/publish_spec.rb @@ -40,9 +40,17 @@ def db; end end let(:player) { double("player", values: attributes, attributes: attributes) } - let(:publisher) { publisher_class.new("Player", attributes, state: :updated) } let(:events) { [] } let(:event) { events.first } + let(:attributes) { { "external_id" => 101, "email" => "email@example.com" } } + + let(:publisher) do + publisher_class.new( + object_class: "Player", + original_attributes: original_attributes, + event: :update, + ) + end shared_context "processing" do before do @@ -85,8 +93,8 @@ def db; end end context "when publish with #{publishing_adapter}" do - let(:publisher_class) { TableSync::Publishing::Publisher } - let(:attributes) { { "external_id" => 101, "email" => "email@example.com" } } + let(:publisher_class) { TableSync::Publishing::Single } + let(:original_attributes) { attributes } context "default schema" do include_context "processing" @@ -103,10 +111,8 @@ def db; end end context "when batch publish with #{publishing_adapter}" do - let(:publisher_class) { TableSync::Publishing::BatchPublisher } - let(:attributes) do - Array.new(3) { |e| { "external_id" => e, "email" => "email#{e}@example.com" } } - end + let(:publisher_class) { TableSync::Publishing::Batch } + let(:original_attributes) { [attributes, attributes, attributes] } context "default schema" do include_context "processing" diff --git a/spec/orm_adapter/active_record_spec.rb b/spec/orm_adapter/active_record_spec.rb new file mode 100644 index 0000000..7d4173a --- /dev/null +++ b/spec/orm_adapter/active_record_spec.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +describe TableSync::ORMAdapter::ActiveRecord do + include_examples "adapter behaviour", ARecordUser, CustomARecordUser +end diff --git a/spec/orm_adapter/sequel_spec.rb b/spec/orm_adapter/sequel_spec.rb new file mode 100644 index 0000000..c01570d --- /dev/null +++ b/spec/orm_adapter/sequel_spec.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +describe TableSync::ORMAdapter::Sequel do + include_examples "adapter behaviour", SequelUser, CustomSequelUser +end diff --git a/spec/publishing/batch_publishing_spec.rb b/spec/publishing/batch_publishing_spec.rb deleted file mode 100644 index dcd52fc..0000000 --- a/spec/publishing/batch_publishing_spec.rb +++ /dev/null @@ -1,288 +0,0 @@ -# frozen_string_literal: true - -describe TableSync::Publishing::BatchPublisher do - let(:pk) { "id" } - let(:id) { 1 } - let(:email) { "example@example.org" } - let(:attributes) { [{ "id" => id, "email" => email }] } - let(:options) { {} } - let(:model_name) { "TestUser" } - let(:publisher) { described_class.new(model_name, attributes, **options) } - - let(:push_original_attributes) { false } - - before { Timecop.freeze("2018-01-01 12:00 UTC") } - - before { TableSync.batch_publishing_job_class_callable = -> { TestJob } } - - def assert_last_job - job = ActiveJob::Base.queue_adapter.enqueued_jobs.last - - object_attributes = attributes.map { |attrs| attrs.merge("_aj_symbol_keys" => attrs.keys) } - job_params = { - "confirm" => true, - "_aj_symbol_keys" => %w[confirm push_original_attributes], - "push_original_attributes" => push_original_attributes, - } - - expect(job[:job]).to eq(TestJob) - expect(job[:args]).to eq(["TestUser", object_attributes, job_params]) - expect(job[:at]).to be_nil - end - - def expect_message(attributes, **options) - args = { - routing_key: options[:routing_key] || "TestUser", - event: :table_sync, - confirm_select: true, - realtime: true, - headers: options[:headers], - data: { - event: options[:event] || :update, - model: options[:model_name] || "TestUser", - attributes: attributes, - version: Time.now.to_f, - metadata: {}, - }, - } - - expect_rabbit_message(args) - end - - describe "#publish" do - before { TableSync.routing_key_callable = -> (klass, _) { klass } } - - context "updating" do - it "performs" do - publisher.publish - assert_last_job - end - end - - context "composite keys" do - let(:pk) { %w[id project_id] } - - it "performs" do - publisher.publish - assert_last_job - end - end - - context "with inserting array at attributes" do - let(:attributes) { [{ "example_array" => [1, 2, 3] }] } - - it "doesn't exclude this array from original attributes" do - publisher.publish - assert_last_job - end - end - - context "with not serialized original attributes" do - let(:attributes) do - [ - { good_attribute: { kek: "pek", array_with_nil: [nil] }, - half_bad: { bad_inside: [Time.current, Float::INFINITY], good_inside: 1 }, - Time.current => "wtf?!" }, - ] - end - - it "filters attributes with wrong types" do - publisher.publish - job = ActiveJob::Base.queue_adapter.enqueued_jobs.last - - params = job[:args][1] - expect(params.first.keys.size).to eq(3) - expect(params.first["good_attribute"]).to include("kek" => "pek", "array_with_nil" => [nil]) - expect(params.first["half_bad"]).to include("bad_inside" => [], "good_inside" => 1) - end - end - - # NOTE: for compatibility with active_job 4.2.11 - context "attributes with symbolic values in hashes" do - let(:attributes) do - [ - { half_bad: { bad_inside: { foo: :bar }, good_inside: { foo: "bar" } } }, - ] - end - - it "converts these values to string" do - publisher.publish - job = ActiveJob::Base.queue_adapter.enqueued_jobs.last - params = job[:args][1] - expect(params.first.keys.size).to eq(2) - expect(params.first["half_bad"]).to include( - "bad_inside" => { "_aj_symbol_keys" => ["foo"], "foo" => "bar" }, - "good_inside" => { "_aj_symbol_keys" => ["foo"], "foo" => "bar" }, - ) - end - end - - context "with inserting original attributes" do - let(:push_original_attributes) { true } - let(:options) { { push_original_attributes: true } } - - it "publish job with this option, setted to true" do - publisher.publish - assert_last_job - end - end - end - - describe "#publish_now" do - shared_context "user sync context" do - let(:user) { double(:user) } - - before do - allow(TestUser).to receive(:find_by).with(id: id).and_return(user) - TableSync.routing_key_callable = -> (klass, _) { klass } - end - end - - shared_context "publishing configuration" do - let(:expected_attrs) { [{ "test_attr" => "test_value" }] } - - before do - allow(user).to receive(:attributes).and_return(expected_attrs.first) - end - end - - context "with overriden_routing_key" do - include_context "user sync context" - include_context "publishing configuration" - - let(:options) { { routing_key: "CustomKey" } } - - it "has correct routing key" do - expect_message(expected_attrs, routing_key: options[:routing_key]) - publisher.publish_now - end - end - - context "with custom headers" do - include_context "user sync context" - include_context "publishing configuration" - - let(:options) { { headers: { kek: "purum" } } } - - it "has correct headers" do - expect_message(expected_attrs, headers: options[:headers]) - publisher.publish_now - end - end - - context "with custom event" do - include_context "user sync context" - include_context "publishing configuration" - - let(:options) { { event: :destroy } } - - it "has correct event" do - expect_message(expected_attrs, event: options[:event]) - publisher.publish_now - end - end - - context "custom meta attributes" do - include_context "user sync context" - include_context "publishing configuration" - - let(:custom_metadata) { { "my_data" => "your_data", "test" => true, "advanced" => 123 } } - - before do - expect_message(expected_attrs) - allow_any_instance_of(TableSync::Publishing::BatchPublisher) - .to receive(:attrs_for_metadata) { custom_metadata } - end - - specify "routing_metadata_callable is a callable" do - expect(TableSync).to receive(:routing_metadata_callable) - publisher.publish_now - end - - it "provides custom attributes for routing_metadata_callable" do - allow(TableSync).to receive(:routing_metadata_callable) do - lambda do |_klass, attributes| - expect(attributes).to match(custom_metadata) - end - end - - publisher.publish_now - end - end - - context "updated (alias for not destroyed)" do - include_context "user sync context" - - context "doesn't respond to #attributes_for_sync" do - before do - allow(user).to receive(:attributes).and_return("test_attr" => "test_value") - end - - it "publishes" do - expect_message([{ "test_attr" => "test_value" }]) - publisher.publish_now - end - end - end - - context "responds to #attributes_for_sync" do - include_context "user sync context" - - before do - allow(TestUser).to receive(:method_defined?).with(:attributes_for_sync).and_return(true) - - allow(user).to receive(:attributes_for_sync) - .and_return("the_ultimate_question_of_live_and_everything" => 42) - end - - it "publishes" do - expect_message([{ "the_ultimate_question_of_live_and_everything" => 42 }]) - publisher.publish_now - end - end - - context "doesn't find any object with that pk" do - include_context "user sync context" - - before do - allow(TestUser).to receive(:find_by).with(id: id).and_return(nil) - end - - it "doesn't publish" do - expect_no_rabbit_messages - publisher.publish_now - end - end - - context "responds to #table_sync_model_name" do - include_context "user sync context" - - let(:model_name) { "TestUserWithCustomStuff" } - - before do - allow(user).to receive(:attributes).and_return("test_attr" => "test_value") - TableSync.routing_key_callable = -> (*_) { "TestUser" } - end - - it "publishes" do - expect_message([{ "test_attr" => "test_value" }], model_name: "SomeFancyName") - publisher.publish_now - end - end - - context "inserts original attributes" do - include_context "user sync context" - - before do - allow(user).to receive(:attributes).and_return("kek" => "pek") - end - - let(:options) { { push_original_attributes: true } } - - it "sends original attributes array instead of record attributes" do - expect_message([{ id: 1, email: "example@example.org" }]) - publisher.publish_now - end - end - end -end diff --git a/spec/publishing/batch_spec.rb b/spec/publishing/batch_spec.rb index b0f97fc..9c04c0c 100644 --- a/spec/publishing/batch_spec.rb +++ b/spec/publishing/batch_spec.rb @@ -1,25 +1,50 @@ # frozen_string_literal: true RSpec.describe TableSync::Publishing::Batch do - let(:original_attributes) { [{ id: 1, time: Time.current }] } - let(:serialized_attributes) { [{ id: 1 }] } + include_context "with created users", 1 + + let(:existing_user) { ARecordUser.first } + let(:original_attributes) { [{ id: existing_user.id, time: Time.current }] } + let(:serialized_attributes) { [{ id: existing_user.id }] } + let(:event) { :update } + let(:object_class) { "ARecordUser" } + let(:routing_key) { object_class.tableize } + let(:headers) { { klass: object_class } } let(:attributes) do - { - object_class: "User", - original_attributes: original_attributes, - event: :update, - headers: { some_arg: 1 }, - routing_key: "custom_key123", - } + { + object_class: object_class, + original_attributes: original_attributes, + event: event, + headers: headers, + routing_key: routing_key, + } end - include_examples "publisher#publish_now calls stubbed message with attributes", - TableSync::Publishing::Message::Batch + include_examples "publisher#publish_now with stubbed message", + TableSync::Publishing::Message::Batch + + context "real user" do + context "sequel" do + let(:object_class) { "SequelUser" } - context "#publish_later" do - let(:job) { double("BatchJob", perform_later: 1) } + include_examples "publisher#publish_now with real user, for given orm", + :sequel + end + + context "active_record" do + include_examples "publisher#publish_now with real user, for given orm", + :active_record + end + end - include_examples "publisher#publish_later behaviour" - end + describe "#publish_later" do + let(:job) { double("BatchJob", perform_at: 1) } + + let(:expected_job_attributes) do + attributes.merge(original_attributes: serialized_attributes) + end + + include_examples "publisher#publish_later behaviour", :perform_later + end end diff --git a/spec/publishing/data/objects_spec.rb b/spec/publishing/data/objects_spec.rb new file mode 100644 index 0000000..5600193 --- /dev/null +++ b/spec/publishing/data/objects_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +describe TableSync::Publishing::Data::Objects do + include_context "with created users", 1 + + let(:data) { described_class.new(params) } + let(:object_class) { ARecordUser } + let(:event) { :update } + let(:objects) { [object] } + let(:expected_attributes) { object_class.first.attributes.symbolize_keys } + let(:expected_model) { object_class.to_s } + + let(:params) do + { + objects: objects, + event: event, + } + end + + let(:object) do + TableSync::ORMAdapter::ActiveRecord.new( + object_class, { id: object_class.first.id } + ).find + end + + let(:expected_data) do + { + model: expected_model, + attributes: [expected_attributes], + version: an_instance_of(Float), + event: event, + metadata: metadata, + } + end + + shared_examples "correctly constructs data for message" do + specify do + expect(data.construct).to include(expected_data) + end + end + + describe "#construct" do + let(:metadata) { { created: false } } + + context "event -> create" do + let(:event) { :create } + let(:metadata) { { created: true } } + + include_examples "correctly constructs data for message" + end + + context "event -> update" do + context "without attributes_for_sync" do + include_examples "correctly constructs data for message" + end + + context "attributes_for_sync defined" do + let(:object_class) { CustomARecordUser } + let(:expected_attributes) { { test: "updated" } } + + before do + allow(object.object).to receive(:attributes_for_sync).and_return(expected_attributes) + end + + include_examples "correctly constructs data for message" + end + end + + context "event -> destroy" do + let(:event) { :destroy } + + context "without #attributes_for_destroy" do + let(:expected_attributes) { { id: object.object.id } } + + include_examples "correctly constructs data for message" + end + + context "attributes_for_destroy defined" do + let(:object_class) { CustomARecordUser } + let(:expected_attributes) { { test: "destroyed" } } + + before do + allow(object.object).to receive(:attributes_for_destroy).and_return(expected_attributes) + end + + include_examples "correctly constructs data for message" + end + end + + context "table_sync_model_name defined on object class" do + let(:object_class) { CustomARecordUser } + let(:expected_model) { "CustomARecordModelName111" } + + before do + allow(object_class).to receive(:table_sync_model_name).and_return(expected_model) + end + + include_examples "correctly constructs data for message" + end + end +end diff --git a/spec/publishing/data/raw_spec.rb b/spec/publishing/data/raw_spec.rb new file mode 100644 index 0000000..4dd66ec --- /dev/null +++ b/spec/publishing/data/raw_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +describe TableSync::Publishing::Data::Raw do + let(:data) { described_class.new(params) } + let(:object_class) { "User" } + let(:attributes_for_sync) { [{ id: 1, asd: "asd" }, { id: 22, time: Time.current }] } + let(:event) { :update } + + let(:params) do + { + object_class: object_class, + attributes_for_sync: attributes_for_sync, + event: event, + } + end + + let(:expected_data) do + { + model: object_class, + attributes: attributes_for_sync, + version: an_instance_of(Float), + event: event, + metadata: metadata, + } + end + + shared_examples "correctly constructs data for message" do + specify do + expect(data.construct).to include(expected_data) + end + end + + describe "#construct" do + let(:metadata) { { created: false } } + + include_examples "correctly constructs data for message" + + context "event -> create" do + let(:event) { :create } + let(:metadata) { { created: true } } + + include_examples "correctly constructs data for message" + end + end +end diff --git a/spec/publishing/helpers/attributes_spec.rb b/spec/publishing/helpers/attributes_spec.rb index e69de29..8585ab8 100644 --- a/spec/publishing/helpers/attributes_spec.rb +++ b/spec/publishing/helpers/attributes_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +describe TableSync::Publishing::Helpers::Attributes do + let(:filter) { described_class.new(original_attributes) } + + shared_examples "filters out unsafe keys/values" do + specify do + expect(filter.serialize).to include(expected_attributes) + end + end + + describe "#serialize" do + context "first level" do + let(:original_attributes) do + { :id => 1, :time => Time.current, Object.new => 3, "fd" => 2, true => "kek" } + end + + let(:expected_attributes) { { :id => 1, :fd => 2, true => "kek" } } # string is symbolized + + include_examples "filters out unsafe keys/values" + end + + context "deep array" do + let(:original_attributes) { { id: [1, Time.current, "7"] } } + let(:expected_attributes) { { id: [1, "7"] } } + + include_examples "filters out unsafe keys/values" + end + + context "deep hash" do + let(:original_attributes) { { id: { safe: 1, unsafe: Time.current, Time.current => "7" } } } + let(:expected_attributes) { { id: { safe: 1 } } } + + include_examples "filters out unsafe keys/values" + end + + context "infinity" do + let(:original_attributes) { { id: 1, amount: Float::INFINITY } } + let(:expected_attributes) { { id: 1 } } + + include_examples "filters out unsafe keys/values" + end + + context "with different base safe types" do + let(:original_attributes) { { :id => 1, true => "stuff" } } + let(:expected_attributes) { { true => "stuff" } } + + before do + stub_const("#{described_class}::BASE_SAFE_JSON_TYPES", [TrueClass, String]) + end + + include_examples "filters out unsafe keys/values" + end + end +end diff --git a/spec/publishing/helpers/debounce_spec.rb b/spec/publishing/helpers/debounce_spec.rb index e69de29..75ec37b 100644 --- a/spec/publishing/helpers/debounce_spec.rb +++ b/spec/publishing/helpers/debounce_spec.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +describe TableSync::Publishing::Helpers::Debounce do + let(:params) do + { + object_class: "SequelUser", + needle: { id: 1 }, + debounce_time: debounce_time, + event: event, + } + end + + let(:service) { described_class.new(params) } + let(:debounce_time) { 30 } + let(:event) { :update } + let(:current_time) { Time.current.beginning_of_day } + + before { Timecop.freeze(current_time) } + + shared_examples "returns correct next_sync_time" do + it "next_sync_time returns correct value" do + expect(service.next_sync_time).to eq(expected_time) + end + end + + shared_examples "skip? returns" do |value| + context "skip? is #{value}" do + it value do + expect(service.skip?).to eq(value) + end + end + end + + def set_cached_sync_time(time) + Rails.cache.write(service.cache_key, time) + end + + context "debounce time -> nil" do + let(:debounce_time) { nil } + + it "defaults debounce to 60" do + expect(service.debounce_time).to eq(60) + end + end + + context "case0: debounce time -> zero" do + let(:debounce_time) { 0 } + let(:expected_time) { current_time } + + include_examples "skip? returns", nil + include_examples "returns correct next_sync_time" + end + + context "case 1: cached sync time is empty" do + let(:expected_time) { current_time } + + include_examples "skip? returns", nil + include_examples "returns correct next_sync_time" + end + + context "case 2: cached sync time in the past" do + context "case 2.1: debounce time passed" do + let(:expected_time) { current_time } + let(:cached_time) { current_time - debounce_time.seconds + 10.seconds } + + before { set_cached_sync_time(cached_time) } + + include_examples "skip? returns", nil + include_examples "returns correct next_sync_time" + end + + context "case 2.2: debounce time not passed yet" do + let(:expected_time) { cached_time + debounce_time.seconds } + let(:cached_time) { current_time - debounce_time.seconds - 10.seconds } + + before { set_cached_sync_time(cached_time) } + + include_examples "skip? returns", nil + include_examples "returns correct next_sync_time" + end + end + + context "case 3: cached sync time in the future" do + let(:cached_time) { current_time + 10.seconds } + + before { set_cached_sync_time(cached_time) } + + context "case 3.1: event update" do + let(:expected_time) { nil } + + include_examples "skip? returns", true + include_examples "returns correct next_sync_time" + end + + context "case 3.2: event destroy" do + let(:event) { :destroy } + let(:expected_time) { cached_time + debounce_time.seconds } + + include_examples "skip? returns", nil + include_examples "returns correct next_sync_time" + end + end +end diff --git a/spec/publishing/helpers/objects_spec.rb b/spec/publishing/helpers/objects_spec.rb index e69de29..dd0af61 100644 --- a/spec/publishing/helpers/objects_spec.rb +++ b/spec/publishing/helpers/objects_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +describe TableSync::Publishing::Helpers::Objects do + include_context "with created users", 1 + include_context "with Sequel ORM" + + let(:objects) { described_class.new(params) } + + let(:params) do + { + object_class: "SequelUser", + original_attributes: original_attributes, + event: event, + } + end + + describe "#construct_list" do + context "event -> update" do + let(:event) { :update } + let(:max_id) { SequelUser.max(:id) + 1000 } + let(:user_id) { SequelUser.first.id } + let(:original_attributes) { [ { id: user_id }, { id: max_id }] } + let(:found_ids) { objects.construct_list.map { |i| i.object.id } } + + it "finds existing objects" do + expect(found_ids).to include(user_id) + end + + it "strips the list of missing objects" do + expect(found_ids).not_to include(max_id) + end + end + + context "event -> destroy" do + let(:event) { :destroy } + let(:original_attributes) { [ { id: 100 }, { id: 123 }] } + let(:initialized_ids) { objects.construct_list.map { |i| i.object.id } } + + it "initializes objects with given data" do + expect(initialized_ids).to include(*original_attributes.map(&:values).flatten) + end + end + end +end diff --git a/spec/publishing/message/batch_spec.rb b/spec/publishing/message/batch_spec.rb index e69de29..ee42961 100644 --- a/spec/publishing/message/batch_spec.rb +++ b/spec/publishing/message/batch_spec.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +describe TableSync::Publishing::Message::Batch do + describe "#publish" do + let(:attributes) do + { + object_class: object_class, + original_attributes: [{ id: 1 }], + routing_key: "users", + headers: { kek: 1 }, + event: :destroy, + } + end + + let(:object_class) { "ARecordUser" } + + context "with stubbed data and params" do + let(:data_class) { TableSync::Publishing::Data::Objects } + let(:params_class) { TableSync::Publishing::Params::Batch } + let(:objects_class) { TableSync::Publishing::Helpers::Objects } + + let(:data) { instance_double(data_class) } + let(:params) { instance_double(params_class) } + let(:objects) { instance_double(objects_class) } + let(:collection_of_objects) { double(:collection_of_objects) } + let(:object) { double(:object) } + + let(:data_attributes) do + { + objects: collection_of_objects, + event: attributes[:event], + } + end + + let(:params_attributes) do + { + object_class: attributes[:object_class], + routing_key: attributes[:routing_key], + headers: attributes[:headers], + } + end + + before do + allow(data_class).to receive(:new).and_return(data) + allow(params_class).to receive(:new).and_return(params) + allow(objects_class).to receive(:new).and_return(objects) + + allow(data).to receive(:construct).and_return({}) + allow(params).to receive(:construct).and_return({}) + allow(objects).to receive(:construct_list).and_return(collection_of_objects) + + allow(collection_of_objects).to receive(:empty?).and_return(false) + allow(collection_of_objects).to receive(:first).and_return(object) + allow(collection_of_objects).to receive(:count).and_return(1) + + allow(object).to receive(:object_class).and_return(object_class.constantize) + end + + it "calls data and params with correct attrs" do + expect(data_class).to receive(:new).with(data_attributes) + expect(params_class).to receive(:new).with(params_attributes) + + expect(data).to receive(:construct) + expect(params).to receive(:construct) + + described_class.new(attributes).publish + end + end + + it "calls Rabbit#publish" do + expect(Rabbit).to receive(:publish) + + described_class.new(attributes).publish + end + end +end diff --git a/spec/publishing/message/raw_spec.rb b/spec/publishing/message/raw_spec.rb index e69de29..a04ab61 100644 --- a/spec/publishing/message/raw_spec.rb +++ b/spec/publishing/message/raw_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +describe TableSync::Publishing::Message::Raw do + describe "#publish" do + let(:attributes) do + { + object_class: "SequelUser", + original_attributes: [{ id: 1 }], + routing_key: "users", + headers: { kek: 1 }, + event: :update, + } + end + + context "with stubbed data and params" do + let(:data_class) { TableSync::Publishing::Data::Raw } + let(:params_class) { TableSync::Publishing::Params::Raw } + + let(:data) { instance_double(data_class) } + let(:params) { instance_double(params_class) } + + let(:data_attributes) do + { + object_class: attributes[:object_class], + attributes_for_sync: attributes[:original_attributes], + event: attributes[:event], + } + end + + let(:params_attributes) do + { + object_class: attributes[:object_class], + routing_key: attributes[:routing_key], + headers: attributes[:headers], + } + end + + before do + allow(data_class).to receive(:new).and_return(data) + allow(params_class).to receive(:new).and_return(params) + + allow(data).to receive(:construct).and_return({}) + allow(params).to receive(:construct).and_return({}) + end + + it "calls data and params with correct attrs" do + expect(data_class).to receive(:new).with(data_attributes) + expect(params_class).to receive(:new).with(params_attributes) + + expect(data).to receive(:construct) + expect(params).to receive(:construct) + + described_class.new(attributes).publish + end + end + + it "calls Rabbit#publish" do + expect(Rabbit).to receive(:publish) + + described_class.new(attributes).publish + end + end +end diff --git a/spec/publishing/message/single_spec.rb b/spec/publishing/message/single_spec.rb index e69de29..c39bb15 100644 --- a/spec/publishing/message/single_spec.rb +++ b/spec/publishing/message/single_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# MOVE TO SUPPORT! + +describe TableSync::Publishing::Message::Single do + describe "#publish" do + let(:attributes) do + { + object_class: object_class, + original_attributes: [{ id: 1 }], + routing_key: "users", + headers: { kek: 1 }, + event: :destroy, + } + end + + let(:object_class) { "ARecordUser" } + + context "with stubbed data and params" do + let(:data_class) { TableSync::Publishing::Data::Objects } + let(:params_class) { TableSync::Publishing::Params::Single } + let(:objects_class) { TableSync::Publishing::Helpers::Objects } + + let(:data) { instance_double(data_class) } + let(:params) { instance_double(params_class) } + let(:objects) { instance_double(objects_class) } + let(:collection_of_objects) { double(:collection_of_objects) } + let(:object) { double(:object, object_class: object_class.constantize) } + + let(:data_attributes) do + { + objects: collection_of_objects, + event: attributes[:event], + } + end + + let(:params_attributes) do + { + object: object, + } + end + + before do + allow(data_class).to receive(:new).and_return(data) + allow(params_class).to receive(:new).and_return(params) + allow(objects_class).to receive(:new).and_return(objects) + + allow(data).to receive(:construct).and_return({}) + allow(params).to receive(:construct).and_return({}) + allow(objects).to receive(:construct_list).and_return(collection_of_objects) + + allow(collection_of_objects).to receive(:empty?).and_return(false) + allow(collection_of_objects).to receive(:first).and_return(object) + allow(collection_of_objects).to receive(:count).and_return(1) + end + + it "calls data and params with correct attrs" do + expect(data_class).to receive(:new).with(data_attributes) + expect(params_class).to receive(:new).with(params_attributes) + + expect(data).to receive(:construct) + expect(params).to receive(:construct) + + described_class.new(attributes).publish + end + end + + it "calls Rabbit#publish" do + expect(Rabbit).to receive(:publish) + + described_class.new(attributes).publish + end + end +end diff --git a/spec/publishing/params/batch_spec.rb b/spec/publishing/params/batch_spec.rb new file mode 100644 index 0000000..661a212 --- /dev/null +++ b/spec/publishing/params/batch_spec.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +describe TableSync::Publishing::Params::Batch do + let(:object_class) { "User" } + let(:attributes) { default_attributes } + let(:default_attributes) { { object_class: object_class } } + let(:service) { described_class.new(attributes) } + + let(:default_expected_values) do + { + confirm_select: true, + realtime: true, + event: :table_sync, + } + end + + shared_examples "constructs with expected values" do + specify do + expect(service.construct).to include(expected_values) + end + end + + shared_examples "raises callable error" do |error| + specify do + expect { service.construct }.to raise_error(error) + end + end + + describe "#construct" do + context "default params" do + let(:expected_values) { default_expected_values } + + include_examples "constructs with expected values" + end + + context "headers" do + context "from attributes" do + let(:headers) { { custom: "kek" } } + let(:attributes) { default_attributes.merge(headers: headers) } + let(:expected_values) { default_expected_values.merge(headers: headers) } + + include_examples "constructs with expected values" + end + + context "calculated" do + let(:expected_values) do + default_expected_values.merge(headers: { object_class: object_class }) + end + + before do + TableSync.headers_callable = -> (object_class, _atrs) { { object_class: object_class } } + end + + include_examples "constructs with expected values" + end + + context "without headers callable" do + before { TableSync.headers_callable = nil } + + include_examples "raises callable error", TableSync::NoCallableError + end + end + + context "routing_key" do + context "from attributes" do + let(:routing_key) { "custom_routing_key" } + let(:attributes) { default_attributes.merge(routing_key: routing_key) } + let(:expected_values) { default_expected_values.merge(routing_key: routing_key) } + + include_examples "constructs with expected values" + end + + context "calculated" do + let(:expected_values) do + default_expected_values.merge(routing_key: object_class) + end + + before do + TableSync.routing_key_callable = -> (object_class, _atrs) { object_class } + end + + include_examples "constructs with expected values" + end + + context "without routing_key callable" do + before { TableSync.routing_key_callable = nil } + + include_examples "raises callable error", TableSync::NoCallableError + end + end + + context "exchange_name" do + context "from attributes" do + let(:exchange_name) { "custom_exchange_name" } + let(:attributes) { default_attributes.merge(exchange_name: exchange_name) } + let(:expected_values) { default_expected_values.merge(exchange_name: exchange_name) } + + include_examples "constructs with expected values" + end + + context "by default" do + let(:exchange_name) { "some.project.table_sync" } + let(:expected_values) { default_expected_values.merge(exchange_name: exchange_name) } + + before { TableSync.exchange_name = exchange_name } + + include_examples "constructs with expected values" + end + end + end +end diff --git a/spec/publishing/params/raw_spec.rb b/spec/publishing/params/raw_spec.rb new file mode 100644 index 0000000..db5e5d0 --- /dev/null +++ b/spec/publishing/params/raw_spec.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +describe TableSync::Publishing::Params::Raw do + let(:object_class) { "User" } + let(:attributes) { default_attributes } + let(:default_attributes) { { object_class: object_class } } + let(:service) { described_class.new(attributes) } + + let(:default_expected_values) do + { + confirm_select: true, + realtime: true, + event: :table_sync, + } + end + + shared_examples "constructs with expected values" do + specify do + expect(service.construct).to include(expected_values) + end + end + + shared_examples "raises callable error" do |error| + specify do + expect { service.construct }.to raise_error(error) + end + end + + describe "#construct" do + context "default params" do + let(:expected_values) { default_expected_values } + + include_examples "constructs with expected values" + end + + context "headers" do + context "from attributes" do + let(:headers) { { custom: "kek" } } + let(:attributes) { default_attributes.merge(headers: headers) } + let(:expected_values) { default_expected_values.merge(headers: headers) } + + include_examples "constructs with expected values" + end + + context "calculated" do + let(:expected_values) do + default_expected_values.merge(headers: { object_class: object_class }) + end + + before do + TableSync.headers_callable = -> (object_class, _atrs) { { object_class: object_class } } + end + + include_examples "constructs with expected values" + end + + context "without headers callable" do + before { TableSync.headers_callable = nil } + + include_examples "raises callable error", TableSync::NoCallableError + end + end + + context "routing_key" do + context "from attributes" do + let(:routing_key) { "custom_routing_key" } + let(:attributes) { default_attributes.merge(routing_key: routing_key) } + let(:expected_values) { default_expected_values.merge(routing_key: routing_key) } + + include_examples "constructs with expected values" + end + + context "calculated" do + let(:expected_values) do + default_expected_values.merge(routing_key: object_class) + end + + before do + TableSync.routing_key_callable = -> (object_class, _atrs) { object_class } + end + + include_examples "constructs with expected values" + end + + context "without routing_key callable" do + before { TableSync.routing_key_callable = nil } + + include_examples "raises callable error", TableSync::NoCallableError + end + end + + context "exchange_name" do + context "from attributes" do + let(:exchange_name) { "custom_exchange_name" } + let(:attributes) { default_attributes.merge(exchange_name: exchange_name) } + let(:expected_values) { default_expected_values.merge(exchange_name: exchange_name) } + + include_examples "constructs with expected values" + end + + context "by default" do + let(:exchange_name) { "some.project.table_sync" } + let(:expected_values) { default_expected_values.merge(exchange_name: exchange_name) } + + before { TableSync.exchange_name = exchange_name } + + include_examples "constructs with expected values" + end + end + end +end diff --git a/spec/publishing/params/single_spec.rb b/spec/publishing/params/single_spec.rb new file mode 100644 index 0000000..47e58fd --- /dev/null +++ b/spec/publishing/params/single_spec.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +describe TableSync::Publishing::Params::Single do + let(:object_class) { "ARecordUser" } + let(:attributes) { default_attributes } + let(:default_attributes) { { object: object } } + let(:service) { described_class.new(attributes) } + + let(:object) do + TableSync::Publishing::Helpers::Objects.new( + object_class: object_class, original_attributes: { id: 1 }, event: :update, + ).construct_list.first + end + + let(:default_expected_values) do + { + confirm_select: true, + realtime: true, + event: :table_sync, + } + end + + shared_examples "constructs with expected values" do + specify do + expect(service.construct).to include(expected_values) + end + end + + shared_examples "raises callable error" do |error| + specify do + expect { service.construct }.to raise_error(error) + end + end + + before do + DB[:users].insert( + id: 1, name: "user", email: "user@mail.com", ext_id: 123, ext_project_id: 1, + ) + end + + describe "#construct" do + context "default params" do + let(:expected_values) { default_expected_values } + + include_examples "constructs with expected values" + end + + context "headers" do + context "calculated" do + let(:expected_values) do + default_expected_values.merge(headers: { object_class: object_class }) + end + + before do + TableSync.headers_callable = -> (object_class, _atrs) { { object_class: object_class } } + end + + include_examples "constructs with expected values" + end + + context "without headers callable" do + before { TableSync.headers_callable = nil } + + include_examples "raises callable error", TableSync::NoCallableError + end + + it "calls callable with attributes" do + expect(TableSync.headers_callable).to receive(:call).with(object_class, object.attributes) + service.construct + end + + context "with attrs for headers" do + let(:object_class) { "CustomARecordUser" } + + before do + allow(object.object).to receive( + :attributes_for_headers, + ).and_return(:attributes_for_headers) + end + + it "calls callable with attributes" do + expect(TableSync.headers_callable) + .to receive(:call).with(object_class, :attributes_for_headers) + + service.construct + end + end + end + + context "routing_key" do + context "calculated" do + let(:expected_values) do + default_expected_values.merge(routing_key: object_class) + end + + before do + TableSync.routing_key_callable = -> (object_class, _atrs) { object_class } + end + + include_examples "constructs with expected values" + end + + context "without routing_key callable" do + before { TableSync.routing_key_callable = nil } + + include_examples "raises callable error", TableSync::NoCallableError + end + + it "calls callable with attributes" do + expect(TableSync.routing_key_callable) + .to receive(:call).with(object_class, object.attributes) + + service.construct + end + + context "with attributes for routing key" do + let(:object_class) { "CustomARecordUser" } + + before do + allow(object.object).to receive( + :attributes_for_routing_key, + ).and_return(:attributes_for_routing_key) + end + + it "calls callable with attributes" do + expect(TableSync.routing_key_callable) + .to receive(:call).with(object_class, :attributes_for_routing_key) + + service.construct + end + end + end + + context "exchange_name" do + context "by default" do + let(:exchange_name) { "some.project.table_sync" } + let(:expected_values) { default_expected_values.merge(exchange_name: exchange_name) } + + before { TableSync.exchange_name = exchange_name } + + include_examples "constructs with expected values" + end + end + end +end diff --git a/spec/publishing/publishing_spec.rb b/spec/publishing/publishing_spec.rb deleted file mode 100644 index b5e3462..0000000 --- a/spec/publishing/publishing_spec.rb +++ /dev/null @@ -1,246 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe TableSync::Publishing::Publisher do - let(:id) { 1 } - let(:email) { "example@example.org" } - let(:attributes) { { "id" => id, "email" => email } } - let(:user_attributes) { attributes.symbolize_keys } - let!(:pk) { "id" } - let(:debounce_time) { nil } - - before { Timecop.freeze("2010-01-01 12:00 UTC") } - - def aj_keys - RUBY_VERSION >= "2.7" && Rails.version >= "6.0.3" ? "_aj_ruby2_keywords" : "_aj_symbol_keys" - end - - describe "#publish" do - def publish - described_class.new( - "TestUser", attributes, state: state, debounce_time: debounce_time - ).publish - end - - def assert_last_job(time) - job = ActiveJob::Base.queue_adapter.enqueued_jobs.last - - object_attributes = attributes.merge("_aj_symbol_keys" => attributes.keys) - job_params = { - "state" => state.to_s, - "confirm" => true, - aj_keys => %w[state confirm], - } - - expect(job[:job]).to eq(TestJob) - expect(job[:args]).to eq(["TestUser", object_attributes, job_params]) - expect(job[:at]).to eq(time.to_i) - end - - context "destroying" do - let(:state) { :destroyed } - - it "enqueues job immediately" do - expect { publish }.to change { ActiveJob::Base.queue_adapter.enqueued_jobs.count }.by(1) - expect { publish }.to change { ActiveJob::Base.queue_adapter.enqueued_jobs.count }.by(1) - assert_last_job(Time.now) - end - end - - context "updating" do - let(:state) { :updated } - - it "debounces" do - publish - assert_last_job(Time.now) - - Timecop.travel(40.seconds) - expect { publish }.to change { ActiveJob::Base.queue_adapter.enqueued_jobs.count }.by(1) - assert_last_job(20.seconds.from_now) - - expect { publish }.not_to change { ActiveJob::Base.queue_adapter.enqueued_jobs.count } - Timecop.travel(30.seconds) - - expect { publish }.to change { ActiveJob::Base.queue_adapter.enqueued_jobs.count }.by(1) - assert_last_job(50.seconds.from_now) - end - - context "when skip_debounce is set" do - let(:debounce_time) { 25 } - - specify do - publish - assert_last_job(Time.now) - - Timecop.travel(10.seconds) - expect { publish }.to change { ActiveJob::Base.queue_adapter.enqueued_jobs.count }.by(1) - assert_last_job(15.seconds.from_now) - - expect { publish }.not_to change { ActiveJob::Base.queue_adapter.enqueued_jobs.count } - Timecop.travel(25.seconds) - - expect { publish }.to change { ActiveJob::Base.queue_adapter.enqueued_jobs.count }.by(1) - assert_last_job(15.seconds.from_now) - end - - context "enqueues job immediately" do - let(:debounce_time) { 0 } - - specify do - expect { publish }.to change { ActiveJob::Base.queue_adapter.enqueued_jobs.count }.by(1) - expect { publish }.to change { ActiveJob::Base.queue_adapter.enqueued_jobs.count }.by(1) - assert_last_job(Time.now) - end - end - end - end - - context "composite keys" do - let(:state) { :updated } - let(:pk) { %w[id project_id] } - - it "works" do - publish - assert_last_job(Time.now) - end - end - end - - describe "#publish_now" do - def publish - described_class.new(class_name, attributes, state: state).publish_now - end - - def expect_message(event, message_attributes, created:) - args = { - routing_key: routing_key, - event: :table_sync, - confirm_select: true, - realtime: true, - headers: headers, - data: { - event: event, - model: expected_model_name, - attributes: message_attributes, - version: Time.now.to_f, - metadata: { created: created }, - }, - } - - expect_rabbit_message(args) - end - - let(:user) { double(:user) } - let(:class_name) { "TestUser" } - let(:expected_model_name) { "TestUser" } - let(:routing_key) { "#{class_name}-#{email}" } - let(:headers) { { class_name => email } } - - let(:routing_key_callable) { -> (klass, attrs) { "#{klass}-#{attrs[:email]}" } } - let(:metadata_callable) { -> (klass, attrs) { { klass => attrs[:email] } } } - - before do - TableSync.routing_key_callable = routing_key_callable - TableSync.routing_metadata_callable = metadata_callable - end - - context "destroyed" do - let(:state) { :destroyed } - - it "publishes" do - expect_message(:destroy, user_attributes, created: false) - publish - end - - context "class with custom destroy attributes" do - let(:class_name) { "TestUserWithCustomStuff" } - let(:expected_model_name) { "SomeFancyName" } - - let(:routing_key_callable) { -> (klass, attrs) { "#{klass}-#{attrs[:mail_address]}" } } - let(:metadata_callable) { -> (klass, attrs) { { klass => attrs[:mail_address] } } } - - it "uses that attributes" do - expect_message(:destroy, { id: id, mail_address: "example@example.org" }, created: false) - publish - end - end - end - - context "unknown state" do - let(:state) { :unknown } - - specify { expect { publish }.to raise_error("Unknown state: :unknown") } - end - - context "not destroyed" do - let(:state) { :created } - - before do - allow(TestUser).to receive(:find_by).with(id: id).and_return(user) - end - - context "does not respond #attributes_for_sync" do - before { allow(user).to receive(:attributes).and_return(user_attributes) } - - it "publishes" do - expect_message(:update, user_attributes, created: true) - publish - end - end - - context "override methods" do - let(:expected_attributes) { user_attributes } - - let(:override_methods) do - %i[attributes_for_sync attrs_for_metadata attrs_for_routing_key] - end - - before do - override_methods.each do |m| - allow(TestUser).to receive(:method_defined?).with(m).and_return(false) - end - - allow(user).to receive(:attributes).and_return(user_attributes) - end - - shared_examples "responds_to" do |override_method| - before do - allow(TestUser).to receive(:method_defined?).with(override_method).and_return(true) - - allow(user).to receive(override_method).and_return(override_data) - end - - it "overrides default" do - expect_message(:update, expected_attributes, created: true) - publish - end - end - - context "attributes_for_sync" do - let(:override_data) { user_attributes.merge(custom_attribute: "custom_value") } - let(:expected_attributes) { override_data } - - include_examples "responds_to", :attributes_for_sync - end - - context "attrs_for_metadata" do - let(:override_data) { { "custom_header" => "some_header" } } - let(:headers) { override_data } - let(:metadata_callable) { -> (_klass, attrs) { attrs } } - - include_examples "responds_to", :attrs_for_metadata - end - - context "attrs_for_routing_key" do - let(:override_data) { { "custom_key" => "custom_attr" } } - let(:routing_key) { "TestUser-custom_attr" } - - let(:routing_key_callable) do - -> (klass, attrs) { "#{klass}-#{attrs["custom_key"]}" } - end - - include_examples "responds_to", :attrs_for_routing_key - end - end - end - end -end diff --git a/spec/publishing/raw_spec.rb b/spec/publishing/raw_spec.rb index 1b7946b..b8f016f 100644 --- a/spec/publishing/raw_spec.rb +++ b/spec/publishing/raw_spec.rb @@ -1,16 +1,27 @@ # frozen_string_literal: true RSpec.describe TableSync::Publishing::Raw do + let(:object_class) { "SequelUser" } + let(:event) { :update } + let(:routing_key) { "custom_routing_key" } + let(:headers) { { some_key: "123" } } + let(:original_attributes) { [{ id: 1, name: "purum" }] } + let(:attributes) do - { - object_class: "NonExistentClass", - original_attributes: [{ id: 1, name: "purum" }], - routing_key: "custom_routing_key", - headers: { some_key: "123" }, - event: :create, - } + { + object_class: object_class, + original_attributes: original_attributes, + routing_key: routing_key, + headers: headers, + event: event, + } end - include_examples "publisher#publish_now calls stubbed message with attributes", - TableSync::Publishing::Message::Raw + let(:expected_object_data) { original_attributes } + + include_examples "publisher#publish_now with stubbed message", + TableSync::Publishing::Message::Raw + + include_examples "publisher#publish_now without stubbed message", + TableSync::Publishing::Message::Raw end diff --git a/spec/publishing/sequel/adapter_spec.rb b/spec/publishing/sequel/adapter_spec.rb deleted file mode 100644 index 19cad91..0000000 --- a/spec/publishing/sequel/adapter_spec.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -TableSync.orm = :sequel - -class ItemWithoutPredicate < Sequel::Model(:items) - TableSync.sync(self) -end - -class ItemWithPositivePredicate < Sequel::Model(:items) - TableSync.sync(self, if: -> (*) { true }) -end - -class ItemWithNegativePredicate < Sequel::Model(:items) - TableSync.sync(self, if: -> (*) { false }) -end - -class ItemWithBothPredicates < Sequel::Model(:items) - TableSync.sync(self, if: -> (*) { if_expr }, unless: -> (*) { unless_expr }) - - attr_accessor :if_expr - attr_accessor :unless_expr -end - -TableSync.orm = :active_record - -RSpec.describe TableSync::ORMAdapter::Sequel, :sequel do - let(:item_class) { ItemWithoutPredicate } - let(:attrs) { { name: "An item", price: 10 } } - let(:queue_adapter) { ActiveJob::Base.queue_adapter } - let(:opts) { {} } - let!(:item) { item_class.create(attrs) } - - def create - item_class.create(attrs) - end - - def update - item.update(price: 20) - end - - def destroy - item.destroy - end - - def enqueues(&block) - expect(block).to change(queue_adapter.enqueued_jobs, :count).by(1) - end - - def ignores(&block) - expect(block).not_to change(queue_adapter.enqueued_jobs, :count) - end - - context "without predicate" do - it { enqueues { create } } - it { enqueues { update } } - it { enqueues { destroy } } - end - - context "with positive predicate" do - let(:item_class) { ItemWithPositivePredicate } - - it { enqueues { create } } - it { enqueues { update } } - it { enqueues { destroy } } - end - - context "with negative predicate" do - let(:item_class) { ItemWithNegativePredicate } - - it { ignores { create } } - it { ignores { update } } - it { ignores { destroy } } - end - - context "with both predicates" do - let(:item_class) { ItemWithBothPredicates } - - let(:if_expr) { true } - let(:unless_expr) { true } - - before do - item.if_expr = if_expr - item.unless_expr = unless_expr - end - - context "positive" do - let(:if_expr) { true } - let(:unless_expr) { false } - - it { enqueues { update } } - end - - context "conflicts" do - let(:if_expr) { true } - let(:unless_expr) { true } - - it { ignores { update } } - end - - context "negative" do - let(:if_expr) { false } - let(:unless_expr) { true } - - it { ignores { update } } - end - end -end diff --git a/spec/publishing/single_spec.rb b/spec/publishing/single_spec.rb index a4834f9..5ee7bb1 100644 --- a/spec/publishing/single_spec.rb +++ b/spec/publishing/single_spec.rb @@ -1,158 +1,100 @@ # frozen_string_literal: true -TableSync.orm = :sequel - -# check for active record? - -class User < Sequel::Model; end - RSpec.describe TableSync::Publishing::Single do - let(:original_attributes) { { id: 1, time: Time.current } } - let(:serialized_attributes) { { id: 1 } } - let(:event) { :update } + include_context "with created users", 1 + + let(:existing_user) { ARecordUser.first } + let(:original_attributes) { { id: existing_user.id } } + let(:event) { :update } + let(:object_class) { "ARecordUser" } + let(:routing_key) { object_class.tableize } + let(:headers) { { klass: object_class } } + let(:debounce_time) { 30 } let(:attributes) do - { - object_class: "User", - original_attributes: original_attributes, - event: event, - debounce_time: 30, - } + { + object_class: object_class, + original_attributes: original_attributes, + event: event, + debounce_time: debounce_time, + } end - include_examples "publisher#publish_now calls stubbed message with attributes", - TableSync::Publishing::Message::Single - - context "#publish_later" do - context "empty message" do - let(:original_attributes) { { id: 1 } } - - before do - TableSync.routing_key_callable = -> (klass, attributes) { klass } - TableSync.headers_callable = -> (klass, attributes) { klass } - end - - context "create" do - let(:event) { :create } - - it "doesn't publish" do - expect(Rabbit).not_to receive(:publish) - described_class.new(attributes).publish_now - end - end - - context "update" do - let(:event) { :update } - - it "doesn't publish" do - expect(Rabbit).not_to receive(:publish) - described_class.new(attributes).publish_now - end - end - - context "destroy" do - let(:event) { :destroy } - - it "publishes" do - expect(Rabbit).to receive(:publish) - described_class.new(attributes).publish_now - end - end - end - end - - context "#publish_later" do - let(:job) { double("Job", perform_later: 1) } - - include_examples "publisher#publish_later behaviour" - - context "with debounce" do - it "skips job, returns nil" do - end - end - end -end + describe "#publish_now" do + include_examples "publisher#publish_now with stubbed message", + TableSync::Publishing::Message::Single + + context "real user" do + context "sequel" do + let(:object_class) { "SequelUser" } + include_examples "publisher#publish_now with real user, for given orm", + :sequel + end -# class User < Sequel::Model -# end - -# TableSync.orm = :sequel - -# TableSync.routing_key_callable = -> (klass, attributes) { "#{klass}_#{attributes[:ext_id]}" } -# TableSync.headers_callable = -> (klass, attributes) { "#{klass}_#{attributes[:ext_id]}" } -# TableSync.exchange_name = :test - - # context "event" do - # let(:routing_key) { "#{object_class}_#{ext_id}" } - # let(:headers) { "#{object_class}_#{ext_id}" } - - # let(:published_message) do - # { - # confirm_select: true, - # event: :table_sync, - # exchange_name: :test, - # headers: headers, - # realtime: true, - # routing_key: routing_key, - # data: { - # attributes: published_attributes, - # event: event, - # metadata: metadata, - # model: "User", - # version: anything, - # }, - # } - # end - - # shared_examples "publishes rabbit message" do - # specify do - # expect(Rabbit).to receive(:publish).with(published_message) - - # described_class.new(attributes).publish_now - # end - # end - - # shared_examples "raises No Objects Error" do - # specify do - # expect { described_class.new(attributes).publish_now } - # .to raise_error(TableSync::Publishing::Message::Base::NO_OBJECTS_FOR_SYNC) - # end - # end - - # shared_examples "has expected behaviour" do - # context "when published object exists" do - # before { User.insert(user_attributes) } - - # include_examples "publishes rabbit message" - # end - - # context "when published object DOESN'T exist" do - # include_examples "raises No Objects Error" - # end - # end - - # context "create" do - # let(:event) { :create } - # let(:metadata) { { created: true } } - # let(:published_attributes) { [a_hash_including(user_attributes)] } - - # include_examples "has expected behaviour" - # end - - # context "update" do - # let(:event) { :update } - # let(:metadata) { { created: false} } - # let(:published_attributes) { [a_hash_including(user_attributes)] } - - # include_examples "has expected behaviour" - # end - - # context "destroy" do - # let(:event) { :destroy } - # let(:metadata) { { created: false} } - # let(:published_attributes) { [user_attributes.slice(:id)] } - - # include_examples "publishes rabbit message" - # end - # end \ No newline at end of file + context "active_record" do + include_examples "publisher#publish_now with real user, for given orm", + :active_record + end + end + + describe "#empty message" do + let(:original_attributes) { { id: existing_user.id + 100 } } + + it "skips publish" do + expect(Rabbit).not_to receive(:publish) + described_class.new(attributes).publish_now + end + end + end + + describe "#publish_later" do + let(:original_attributes) { { id: 1, time: Time.current } } + let(:serialized_attributes) { { id: 1 } } + let(:job) { double("Job", perform_later: 1) } + + let(:expected_job_attributes) do + attributes.merge(original_attributes: serialized_attributes, perform_at: anything) + end + + include_examples "publisher#publish_later behaviour", :perform_at + + context "debounce" do + before do + allow_any_instance_of(described_class).to receive(:job).and_return(job) + allow(job).to receive(:perform_at) + + allow(TableSync::Publishing::Helpers::Debounce).to receive(:new).and_call_original + end + + it "calls debounce" do + expect(TableSync::Publishing::Helpers::Debounce).to receive(:new).with( + object_class: object_class, + needle: { id: 1 }, + debounce_time: debounce_time, + event: event, + ) + + described_class.new(attributes).publish_later + end + + context "within debounce limit" do + context "upsert event" do + let(:event) { :update } + + xit "skips publishing" do + expect(job).not_to receive(:perform_at).with(any_args) + described_class.new(attributes).publish_later + end + end + + context "destroy event" do + xit "publishes message" do + expect(job).to receive(:perform_at).with(any_args) + described_class.new(attributes).publish_later + end + end + end + end + end +end diff --git a/spec/setup/active_record_spec.rb b/spec/setup/active_record_spec.rb new file mode 100644 index 0000000..7c974f8 --- /dev/null +++ b/spec/setup/active_record_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +describe TableSync::Setup::ActiveRecord do + include_context "with created users", 1 + + let(:job) { double("Job", perform_at: 1) } + + before do + TableSync.single_publishing_job_class_callable = -> { job } + + stub_const("TestARUser", Class.new(ARecordUser)) + end + + def setup_sync(options = {}) + TestARUser.instance_exec { TableSync.sync(self, **options) } + end + + include_examples "setup: enqueue job behaviour", "TestARUser" + + context "setup" do + it "sends after_commit for all events" do + expect(TestARUser).to receive(:after_commit).with(on: :create) + expect(TestARUser).to receive(:after_commit).with(on: :update) + expect(TestARUser).to receive(:after_commit).with(on: :destroy) + + setup_sync + end + end +end diff --git a/spec/setup/sequel_spec.rb b/spec/setup/sequel_spec.rb new file mode 100644 index 0000000..4f917ce --- /dev/null +++ b/spec/setup/sequel_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +describe TableSync::Setup::Sequel do + include_context "with Sequel ORM" + include_context "with created users", 1 + + let(:job) { double("Job", perform_at: 1) } + + before do + TableSync.single_publishing_job_class_callable = -> { job } + + stub_const("TestSequelUser", Class.new(SequelUser)) + end + + def setup_sync(options = {}) + TestSequelUser.instance_exec { TableSync.sync(self, **options) } + end + + include_examples "setup: enqueue job behaviour", "TestSequelUser" + + context "setup" do + it "sends define_method for all events" do + expect(TestSequelUser).to receive(:define_method).with(:after_create) + expect(TestSequelUser).to receive(:define_method).with(:after_update) + expect(TestSequelUser).to receive(:define_method).with(:after_destroy) + + setup_sync + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 19e51e7..098806c 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -26,6 +26,8 @@ Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].sort.each { |file| require file } +TableSync::TestEnv.setup! + RSpec.configure do |config| config.include Rabbit::TestHelpers @@ -43,4 +45,6 @@ expectations.syntax = :expect expectations.include_chain_clauses_in_custom_matcher_descriptions = true end + + config.after { TableSync::TestEnv.setup! } end diff --git a/spec/support/001_setup.rb b/spec/support/001_setup.rb deleted file mode 100644 index 2e6d036..0000000 --- a/spec/support/001_setup.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -class TestUser - class << self - def find_by(*) - # Stub - end - - def lock(*) - # Stub - self - end - - def primary_key - "id" - end - - def table_name - :test_users - end - end -end - -class TestUserWithCustomStuff < TestUser - def self.table_sync_model_name - "SomeFancyName" - end - - def self.table_sync_destroy_attributes(attrs) - { - id: attrs[:id], - mail_address: attrs[:email], - } - end - - def attrs_for_metadata; end - - def attrs_for_routing_key; end -end - -TestJob = Class.new(ActiveJob::Base) diff --git a/spec/support/active_job_settings.rb b/spec/support/active_job_settings.rb index 39ee8fe..36336c4 100644 --- a/spec/support/active_job_settings.rb +++ b/spec/support/active_job_settings.rb @@ -2,3 +2,6 @@ ActiveJob::Base.queue_adapter = :test ActiveJob::Base.logger = Logger.new("/dev/null") + +SingleTestJob = Class.new(ActiveJob::Base) +BatchTestJob = Class.new(ActiveJob::Base) diff --git a/spec/support/context/publishing.rb b/spec/support/context/publishing.rb deleted file mode 100644 index 2a8dad2..0000000 --- a/spec/support/context/publishing.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -# needs let(:attributes) { #attrs } -shared_examples "publisher#publish_now calls stubbed message with attributes" do |message_class| - describe "#publish_now" do - context "with stubbed message" do - let(:event) { :update } - let(:message_double) { double("Message") } - - before do - allow(message_class).to receive(:new).and_return(message_double) - allow(message_double).to receive(:publish) - allow(message_double).to receive(:empty?).and_return(false) - end - - it "initializes message with correct parameters" do - expect(message_class).to receive(:new).with(attributes) - expect(message_double).to receive(:publish) - - described_class.new(attributes).publish_now - end - end - end -end - -# needs let(:job) with perform_later defined -# needs let(:attributes) -# needs let(:original_attributes) with unserializable value -# needs let(:serialized_attributes) -> original_attributes without unserializable value - -shared_examples "publisher#publish_later behaviour" do - describe "#publish_later" do - context "with defined job" do - before do - TableSync.batch_publishing_job_class_callable = -> { job } - TableSync.single_publishing_job_class_callable = -> { job } - end - - it "calls BatchJob with serialized original_attributes" do - expect(job).to receive(:perform_later).with( - attributes.merge(original_attributes: serialized_attributes) - ) - - described_class.new(attributes).publish_later - end - end - - context "without defined job" do - before do - TableSync.batch_publishing_job_class_callable = nil - TableSync.single_publishing_job_class_callable = nil - end - - it "raises no job error" do - expect { described_class.new(attributes).publish_later } - .to raise_error(TableSync::NoJobClassError) - end - end - end -end diff --git a/spec/support/database_settings.rb b/spec/support/database_settings.rb index 0299b7a..233eed5 100644 --- a/spec/support/database_settings.rb +++ b/spec/support/database_settings.rb @@ -96,3 +96,54 @@ DB.run("TRUNCATE #{tables.join(', ')}") end end + +class SequelUser < Sequel::Model(:users) +end + +class ARecordUser < ActiveRecord::Base + self.table_name = "users" +end + +class CustomARecordUser < ARecordUser + def self.table_sync_model_name + name + end + + def attributes_for_destroy + attributes.symbolize_keys + end + + def attributes_for_sync + attributes.symbolize_keys + end + + def attributes_for_headers + attributes.symbolize_keys + end + + def attributes_for_routing_key + attributes.symbolize_keys + end +end + +class CustomSequelUser < SequelUser + def self.table_sync_model_name + name + end + + def attributes_for_destroy + attributes.symbolize_keys + end + + def attributes_for_sync + attributes.symbolize_keys + end + + def attributes_for_headers + attributes.symbolize_keys + end + + def attributes_for_routing_key + attributes.symbolize_keys + end +end diff --git a/spec/support/shared/adapters.rb b/spec/support/shared/adapters.rb new file mode 100644 index 0000000..4cce558 --- /dev/null +++ b/spec/support/shared/adapters.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true + +shared_examples "adapter behaviour" do |klass, custom_klass| + include_context "with created users", 1 + + let(:adapter) { described_class.new(object_class, object_data).find } + let(:object_class) { klass } + let(:existing_object) { object_class.first } + let(:object_data) { { id: existing_object.id, email: "email" } } + + shared_examples "returns expected_attributes for method" do |meth| + it meth do + expect(adapter.send(meth)).to include(expected_attributes) + end + end + + describe "#initialize" do + context "object data without complete primary key" do + let(:object_data) { { email: "email" } } + + it "raises error" do + expect { adapter }.to raise_error(TableSync::NoPrimaryKeyError) + end + end + end + + describe "#init" do + let(:object_data) { { id: existing_object.id + 100 } } + + it "initializes @object with new instance and returns self" do + expect(adapter.object).to eq(nil) + + adapter.init + + expect(adapter.object.id).to eq(existing_object.id + 100) + end + end + + describe "#find" do + context "if object exists" do + it "initializes @object with found object and returns self" do + expect(adapter.object.id).to eq(existing_object.id) + expect(adapter.object.class).to eq(object_class) + end + end + + context "if object doesn't exist" do + let(:object_data) { { id: existing_object.id + 100 } } + + it "initializes @object with nil and returns self" do + expect(adapter.object).to eq(nil) + end + end + end + + describe "#needle" do + it "returns pk attributes from object_data" do + expect(adapter.needle).to eq({ id: existing_object.id }) + end + end + + describe "#primary_key_columns" do + it "returns array of symbolized pk column names" do + expect(adapter.primary_key_columns).to eq([:id]) + end + end + + describe "#attributes_for_update" do + context "object responds to method" do + let(:expected_attributes) { { custom: "data" } } + let(:object_class) { custom_klass } + + before do + allow(adapter.object).to receive(:attributes_for_sync).and_return(expected_attributes) + end + + include_examples "returns expected_attributes for method", :attributes_for_update + end + + context "object DOESN'T respond to method" do + let(:expected_attributes) { adapter.attributes } + + include_examples "returns expected_attributes for method", :attributes_for_update + end + end + + describe "#attributes_for_destroy" do + context "object responds to method" do + let(:expected_attributes) { { custom: "data" } } + let(:object_class) { custom_klass } + + before do + allow(adapter.object).to receive(:attributes_for_destroy).and_return(expected_attributes) + end + + include_examples "returns expected_attributes for method", :attributes_for_destroy + end + + context "object DOESN'T respond to method" do + let(:expected_attributes) { adapter.needle } + + include_examples "returns expected_attributes for method", :attributes_for_destroy + end + end + + describe "#attributes_for_routing_key" do + context "object responds to method" do + let(:expected_attributes) { { custom: "data" } } + let(:object_class) { custom_klass } + + before do + allow(adapter.object).to receive( + :attributes_for_routing_key, + ).and_return(expected_attributes) + end + + include_examples "returns expected_attributes for method", :attributes_for_routing_key + end + + context "object DOESN'T respond to method" do + let(:expected_attributes) { adapter.attributes } + + include_examples "returns expected_attributes for method", :attributes_for_routing_key + end + end + + describe "#attributes_for_headers" do + context "object responds to method" do + let(:expected_attributes) { { custom: "data" } } + let(:object_class) { custom_klass } + + before do + allow(adapter.object).to receive(:attributes_for_headers).and_return(expected_attributes) + end + + include_examples "returns expected_attributes for method", :attributes_for_headers + end + + context "object DOESN'T respond to method" do + let(:expected_attributes) { adapter.attributes } + + include_examples "returns expected_attributes for method", :attributes_for_headers + end + end + + describe "#empty?" do + context "without object" do + let(:object_data) { { id: existing_object.id + 100 } } + + it "returns true" do + expect(adapter.empty?).to eq(true) + end + end + + context "with object" do + it "returns false" do + expect(adapter.empty?).to eq(false) + end + end + end + + describe "#attributes" do + let(:expected_values) do + DB[:users].where(id: existing_object.id).naked.all.first + end + + it "returns symbolized attributes of an object" do + expect(adapter.attributes).to include(expected_values) + end + end +end diff --git a/spec/support/shared/publishers.rb b/spec/support/shared/publishers.rb new file mode 100644 index 0000000..504dbd3 --- /dev/null +++ b/spec/support/shared/publishers.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +# needs let(:attributes) +shared_examples "publisher#publish_now with stubbed message" do |message_class| + describe "#publish_now" do + context "with stubbed message" do + let(:event) { :update } + let(:message_double) { double("Message") } + + before do + allow(message_class).to receive(:new).and_return(message_double) + allow(message_double).to receive(:publish) + allow(message_double).to receive(:empty?).and_return(false) + end + + it "initializes message with correct parameters" do + expect(message_class).to receive(:new).with(attributes) + expect(message_double).to receive(:publish) + + described_class.new(attributes).publish_now + end + end + end +end + +# needs let(:attributes) +# needs let(:object_class) - String +# needs let(:expected_object_data) +# needs let(:headers) +# needs let(:routing_key) +shared_examples "publisher#publish_now without stubbed message" do + describe "#publish_now" do + context "without stubbed message" do + let(:rabbit_params) do + a_hash_including( + data: a_hash_including( + attributes: expected_object_data, + event: event, + metadata: { created: false }, + model: object_class, + version: an_instance_of(Float), + ), + routing_key: routing_key, # defined by callable by default + headers: headers, # defined by callable by default + ) + end + + it "calls Rabbit#publish with attributes" do + expect(Rabbit).to receive(:publish).with(rabbit_params) + + described_class.new(attributes).publish_now + end + end + end +end + +# needs let(:existing_user) +shared_examples "publisher#publish_now with real user, for given orm" do |orm| + let(:user) { DB[:users].where(id: existing_user.id).first } + let(:expected_object_data) { [a_hash_including(user)] } + + before { TableSync.orm = orm } + + include_examples "publisher#publish_now without stubbed message" +end + +# needs let(:job) with perform_at defined +# needs let(:attributes) +# needs let(expected_job_attributes) + +shared_examples "publisher#publish_later behaviour" do |expected_method| + describe "#publish_later" do + context "with defined job" do + before do + TableSync.batch_publishing_job_class_callable = -> { job } + TableSync.single_publishing_job_class_callable = -> { job } + end + + it "calls job with serialized original_attributes" do + expect(job).to receive(expected_method).with(expected_job_attributes) + + described_class.new(attributes).publish_later + end + end + + context "without defined job" do + before do + TableSync.batch_publishing_job_class_callable = nil + TableSync.single_publishing_job_class_callable = nil + end + + it "raises no job error" do + expect { described_class.new(attributes).publish_later } + .to raise_error(TableSync::NoCallableError) + end + end + end +end diff --git a/spec/support/shared/publishing.rb b/spec/support/shared/publishing.rb new file mode 100644 index 0000000..fe51757 --- /dev/null +++ b/spec/support/shared/publishing.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +shared_context "with created users" do |quantity| + before do + (1..quantity).each do |id| + DB[:users].insert({ + id: id, + name: "test#{id}", + email: "mail#{id}", + ext_id: id + 100, + ext_project_id: 12, + version: 123, + rest: nil, + }) + end + end +end + +shared_context "with Sequel ORM" do + before { TableSync.orm = :sequel } +end diff --git a/spec/support/shared/setup.rb b/spec/support/shared/setup.rb new file mode 100644 index 0000000..992f8cb --- /dev/null +++ b/spec/support/shared/setup.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +shared_examples "setup: enqueue job behaviour" do |test_class_name| + let(:test_class) { test_class_name.constantize } + + shared_examples "enqueues job" do + specify do + expect(job).to receive(:perform_at) + test_class.first.update(name: "new_name") + end + end + + shared_examples "doesn't enqueue job" do + specify do + expect(job).not_to receive(:perform_at) + test_class.first.update(name: "new_name") + end + end + + context "without options" do + before { setup_sync } + + include_examples "enqueues job" + end + + context "if option" do + context "true" do + before { setup_sync(if: -> (_) { true }) } + + include_examples "enqueues job" + end + + context "false" do + before { setup_sync(if: -> (_) { false }) } + + include_examples "doesn't enqueue job" + end + end + + context "unless option" do + context "false" do + before { setup_sync(unless: -> (_) { false }) } + + include_examples "enqueues job" + end + + context "true" do + before { setup_sync(unless: -> (_) { true }) } + + include_examples "doesn't enqueue job" + end + end + + context "both options" do + context "skips by if (false)" do + before { setup_sync(if: -> (_) { false }, unless: -> (_) { false }) } + + include_examples "doesn't enqueue job" + end + + context "skips by unless (true)" do + before { setup_sync(if: -> (_) { true }, unless: -> (_) { true }) } + + include_examples "doesn't enqueue job" + end + end +end diff --git a/spec/support/table_sync_settings.rb b/spec/support/table_sync_settings.rb index 730fccb..f3fa003 100644 --- a/spec/support/table_sync_settings.rb +++ b/spec/support/table_sync_settings.rb @@ -1,4 +1,15 @@ # frozen_string_literal: true -TableSync.orm = :active_record -TableSync.single_publishing_job_class_callable = -> { TestJob } +module TableSync::TestEnv + module_function + + def setup! + TableSync.orm = :active_record + TableSync.raise_on_empty_message = nil + + TableSync.single_publishing_job_class_callable = -> { SingleTestJob } + TableSync.batch_publishing_job_class_callable = -> { BatchTestJob } + TableSync.routing_key_callable = -> (klass, _attributes) { klass.tableize } + TableSync.headers_callable = -> (klass, _attributes) { { klass: klass } } + end +end From e928729dd58f0c161da5fdc2dd43870c86cf871b Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Mon, 23 Aug 2021 14:39:09 +0300 Subject: [PATCH 06/20] event to sym --- lib/table_sync/publishing/single.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/table_sync/publishing/single.rb b/lib/table_sync/publishing/single.rb index b3acba8..8c4dd87 100644 --- a/lib/table_sync/publishing/single.rb +++ b/lib/table_sync/publishing/single.rb @@ -6,10 +6,10 @@ class TableSync::Publishing::Single attribute :object_class attribute :original_attributes - - attribute :event, default: :update attribute :debounce_time + attribute :event, Symbol, default: :update + # expect job to have perform_at method # debounce destroyed event # because otherwise update event could be sent after destroy From 78cf6f3e1bb60776c2a074dfce821894cc935558 Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Tue, 24 Aug 2021 16:23:43 +0300 Subject: [PATCH 07/20] fix --- lib/table_sync/setup/active_record.rb | 17 ++++++++--------- lib/table_sync/setup/sequel.rb | 20 +++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/lib/table_sync/setup/active_record.rb b/lib/table_sync/setup/active_record.rb index a5bf3d5..4101357 100644 --- a/lib/table_sync/setup/active_record.rb +++ b/lib/table_sync/setup/active_record.rb @@ -8,15 +8,14 @@ def define_after_commit(event) options = options_exposed_for_block object_class.after_commit(on: event) do - next unless options[:if].call(self) - next if options[:unless].call(self) - - TableSync::Publishing::Single.new( - object_class: self.class.name, - original_attributes: attributes, - event: event, - debounce_time: options[:debounce_time], - ).publish_later + if instance_eval(&options[:if]) && !instance_eval(&options[:unless]) + TableSync::Publishing::Single.new( + object_class: self.class.name, + original_attributes: attributes, + event: event, + debounce_time: options[:debounce_time], + ).publish_later + end end end end diff --git a/lib/table_sync/setup/sequel.rb b/lib/table_sync/setup/sequel.rb index 7f6bbcf..0b851b1 100644 --- a/lib/table_sync/setup/sequel.rb +++ b/lib/table_sync/setup/sequel.rb @@ -8,15 +8,17 @@ def define_after_commit(event) options = options_exposed_for_block object_class.define_method("after_#{event}".to_sym) do - return unless options[:if].call(self) - return if options[:unless].call(self) - - TableSync::Publishing::Single.new( - object_class: self.class.name, - original_attributes: values, - event: event, - debounce_time: options[:debounce_time], - ).publish_later + # if options[:if].call(self) && !options[:unless].call(self) + if instance_eval(&options[:if]) && !instance_eval(&options[:unless]) + db.after_commit do + TableSync::Publishing::Single.new( + object_class: self.class.name, + original_attributes: values, + event: event, + debounce_time: options[:debounce_time], + ).publish_later + end + end super() end From c227939917c62672feea2cb0f38bd2a7ff998a60 Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Tue, 7 Sep 2021 11:23:42 +0300 Subject: [PATCH 08/20] return on empty original attrs --- lib/table_sync/publishing/message/base.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/table_sync/publishing/message/base.rb b/lib/table_sync/publishing/message/base.rb index 542573a..07b9535 100644 --- a/lib/table_sync/publishing/message/base.rb +++ b/lib/table_sync/publishing/message/base.rb @@ -21,6 +21,8 @@ def initialize(params) end def publish + return if original_attributes.blank? + Rabbit.publish(message_params) notify! From a6f3a1bd358a75845598070e6ea01bede8f340cf Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Tue, 7 Sep 2021 11:38:00 +0300 Subject: [PATCH 09/20] uddate for audit --- Gemfile.lock | 134 +++++++++++++++++++++++++-------------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c87baa1..6d14c66 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,60 +10,60 @@ PATH GEM remote: https://rubygems.org/ specs: - actioncable (6.1.3.1) - actionpack (= 6.1.3.1) - activesupport (= 6.1.3.1) + actioncable (6.1.4.1) + actionpack (= 6.1.4.1) + activesupport (= 6.1.4.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.3.1) - actionpack (= 6.1.3.1) - activejob (= 6.1.3.1) - activerecord (= 6.1.3.1) - activestorage (= 6.1.3.1) - activesupport (= 6.1.3.1) + actionmailbox (6.1.4.1) + actionpack (= 6.1.4.1) + activejob (= 6.1.4.1) + activerecord (= 6.1.4.1) + activestorage (= 6.1.4.1) + activesupport (= 6.1.4.1) mail (>= 2.7.1) - actionmailer (6.1.3.1) - actionpack (= 6.1.3.1) - actionview (= 6.1.3.1) - activejob (= 6.1.3.1) - activesupport (= 6.1.3.1) + actionmailer (6.1.4.1) + actionpack (= 6.1.4.1) + actionview (= 6.1.4.1) + activejob (= 6.1.4.1) + activesupport (= 6.1.4.1) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.3.1) - actionview (= 6.1.3.1) - activesupport (= 6.1.3.1) + actionpack (6.1.4.1) + actionview (= 6.1.4.1) + activesupport (= 6.1.4.1) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.3.1) - actionpack (= 6.1.3.1) - activerecord (= 6.1.3.1) - activestorage (= 6.1.3.1) - activesupport (= 6.1.3.1) + actiontext (6.1.4.1) + actionpack (= 6.1.4.1) + activerecord (= 6.1.4.1) + activestorage (= 6.1.4.1) + activesupport (= 6.1.4.1) nokogiri (>= 1.8.5) - actionview (6.1.3.1) - activesupport (= 6.1.3.1) + actionview (6.1.4.1) + activesupport (= 6.1.4.1) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.1.3.1) - activesupport (= 6.1.3.1) + activejob (6.1.4.1) + activesupport (= 6.1.4.1) globalid (>= 0.3.6) - activemodel (6.1.3.1) - activesupport (= 6.1.3.1) - activerecord (6.1.3.1) - activemodel (= 6.1.3.1) - activesupport (= 6.1.3.1) - activestorage (6.1.3.1) - actionpack (= 6.1.3.1) - activejob (= 6.1.3.1) - activerecord (= 6.1.3.1) - activesupport (= 6.1.3.1) + activemodel (6.1.4.1) + activesupport (= 6.1.4.1) + activerecord (6.1.4.1) + activemodel (= 6.1.4.1) + activesupport (= 6.1.4.1) + activestorage (6.1.4.1) + actionpack (= 6.1.4.1) + activejob (= 6.1.4.1) + activerecord (= 6.1.4.1) + activesupport (= 6.1.4.1) marcel (~> 1.0.0) - mini_mime (~> 1.0.2) - activesupport (6.1.3.1) + mini_mime (>= 1.1.0) + activesupport (6.1.4.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -78,7 +78,7 @@ GEM bunny (2.17.0) amq-protocol (~> 2.3, >= 2.3.1) coderay (1.1.3) - concurrent-ruby (1.1.8) + concurrent-ruby (1.1.9) crass (1.0.6) diff-lcs (1.4.4) docile (1.3.5) @@ -86,13 +86,13 @@ GEM exception_notification (4.4.3) actionmailer (>= 4.0, < 7) activesupport (>= 4.0, < 7) - globalid (0.4.2) - activesupport (>= 4.2.0) + globalid (0.5.2) + activesupport (>= 5.0) i18n (1.8.10) concurrent-ruby (~> 1.0) lamian (1.2.0) rails (>= 4.2) - loofah (2.9.0) + loofah (2.12.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -101,12 +101,12 @@ GEM memery (1.4.0) ruby2_keywords (~> 0.0.2) method_source (1.0.0) - mini_mime (1.0.3) - mini_portile2 (2.5.0) + mini_mime (1.1.1) + mini_portile2 (2.6.1) minitest (5.14.4) - nio4r (2.5.7) - nokogiri (1.11.2) - mini_portile2 (~> 2.5.0) + nio4r (2.5.8) + nokogiri (1.12.4) + mini_portile2 (~> 2.6.1) racc (~> 1.4) parallel (1.20.1) parser (3.0.0.0) @@ -126,34 +126,34 @@ GEM rack (2.2.3) rack-test (1.1.0) rack (>= 1.0, < 3) - rails (6.1.3.1) - actioncable (= 6.1.3.1) - actionmailbox (= 6.1.3.1) - actionmailer (= 6.1.3.1) - actionpack (= 6.1.3.1) - actiontext (= 6.1.3.1) - actionview (= 6.1.3.1) - activejob (= 6.1.3.1) - activemodel (= 6.1.3.1) - activerecord (= 6.1.3.1) - activestorage (= 6.1.3.1) - activesupport (= 6.1.3.1) + rails (6.1.4.1) + actioncable (= 6.1.4.1) + actionmailbox (= 6.1.4.1) + actionmailer (= 6.1.4.1) + actionpack (= 6.1.4.1) + actiontext (= 6.1.4.1) + actionview (= 6.1.4.1) + activejob (= 6.1.4.1) + activemodel (= 6.1.4.1) + activerecord (= 6.1.4.1) + activestorage (= 6.1.4.1) + activesupport (= 6.1.4.1) bundler (>= 1.15.0) - railties (= 6.1.3.1) + railties (= 6.1.4.1) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.3.0) + rails-html-sanitizer (1.4.2) loofah (~> 2.3) - railties (6.1.3.1) - actionpack (= 6.1.3.1) - activesupport (= 6.1.3.1) + railties (6.1.4.1) + actionpack (= 6.1.4.1) + activesupport (= 6.1.4.1) method_source - rake (>= 0.8.7) + rake (>= 0.13) thor (~> 1.0) rainbow (3.0.0) - rake (13.0.3) + rake (13.0.6) regexp_parser (2.1.1) rexml (3.2.5) rspec (3.10.0) @@ -235,7 +235,7 @@ GEM tzinfo (2.0.4) concurrent-ruby (~> 1.0) unicode-display_width (2.0.0) - websocket-driver (0.7.3) + websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) zeitwerk (2.4.2) From 625ef714e5cd16df6348d2879747b91471858a05 Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Tue, 7 Sep 2021 15:35:04 +0300 Subject: [PATCH 10/20] fixes for ruby 3 --- spec/publishing/data/objects_spec.rb | 2 +- spec/publishing/data/raw_spec.rb | 2 +- spec/publishing/helpers/debounce_spec.rb | 2 +- spec/publishing/helpers/objects_spec.rb | 2 +- spec/publishing/params/single_spec.rb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/publishing/data/objects_spec.rb b/spec/publishing/data/objects_spec.rb index 5600193..52945b1 100644 --- a/spec/publishing/data/objects_spec.rb +++ b/spec/publishing/data/objects_spec.rb @@ -3,7 +3,7 @@ describe TableSync::Publishing::Data::Objects do include_context "with created users", 1 - let(:data) { described_class.new(params) } + let(:data) { described_class.new(**params) } let(:object_class) { ARecordUser } let(:event) { :update } let(:objects) { [object] } diff --git a/spec/publishing/data/raw_spec.rb b/spec/publishing/data/raw_spec.rb index 4dd66ec..3d04669 100644 --- a/spec/publishing/data/raw_spec.rb +++ b/spec/publishing/data/raw_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true describe TableSync::Publishing::Data::Raw do - let(:data) { described_class.new(params) } + let(:data) { described_class.new(**params) } let(:object_class) { "User" } let(:attributes_for_sync) { [{ id: 1, asd: "asd" }, { id: 22, time: Time.current }] } let(:event) { :update } diff --git a/spec/publishing/helpers/debounce_spec.rb b/spec/publishing/helpers/debounce_spec.rb index 75ec37b..c064377 100644 --- a/spec/publishing/helpers/debounce_spec.rb +++ b/spec/publishing/helpers/debounce_spec.rb @@ -10,7 +10,7 @@ } end - let(:service) { described_class.new(params) } + let(:service) { described_class.new(**params) } let(:debounce_time) { 30 } let(:event) { :update } let(:current_time) { Time.current.beginning_of_day } diff --git a/spec/publishing/helpers/objects_spec.rb b/spec/publishing/helpers/objects_spec.rb index dd0af61..362a36f 100644 --- a/spec/publishing/helpers/objects_spec.rb +++ b/spec/publishing/helpers/objects_spec.rb @@ -4,7 +4,7 @@ include_context "with created users", 1 include_context "with Sequel ORM" - let(:objects) { described_class.new(params) } + let(:objects) { described_class.new(**params) } let(:params) do { diff --git a/spec/publishing/params/single_spec.rb b/spec/publishing/params/single_spec.rb index 47e58fd..8f4387e 100644 --- a/spec/publishing/params/single_spec.rb +++ b/spec/publishing/params/single_spec.rb @@ -4,7 +4,7 @@ let(:object_class) { "ARecordUser" } let(:attributes) { default_attributes } let(:default_attributes) { { object: object } } - let(:service) { described_class.new(attributes) } + let(:service) { described_class.new(**attributes) } let(:object) do TableSync::Publishing::Helpers::Objects.new( From 8297ac286c074103ca979f9d307b02119fbdb75a Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Mon, 27 Sep 2021 13:43:25 +0300 Subject: [PATCH 11/20] fixes, changelog --- CHANGELOG.md | 32 +++++++++++++ lib/table_sync.rb | 2 +- lib/table_sync/errors.rb | 12 +++++ lib/table_sync/event.rb | 35 +++++++++++++++ lib/table_sync/publishing/data/objects.rb | 16 ++----- lib/table_sync/publishing/data/raw.rb | 11 ++--- lib/table_sync/publishing/helpers/objects.rb | 8 +--- lib/table_sync/publishing/message/base.rb | 4 +- lib/table_sync/publishing/raw.rb | 19 -------- lib/table_sync/receiving.rb | 2 - lib/table_sync/receiving/config.rb | 10 +++-- lib/table_sync/receiving/handler.rb | 16 ++++--- spec/event_spec.rb | 47 ++++++++++++++++++++ spec/publishing/data/objects_spec.rb | 6 ++- spec/publishing/data/raw_spec.rb | 3 +- spec/publishing/message/batch_spec.rb | 23 +++++++++- spec/publishing/message/single_spec.rb | 23 +++++++++- 17 files changed, 203 insertions(+), 66 deletions(-) create mode 100644 lib/table_sync/event.rb create mode 100644 spec/event_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b12c1b..220277a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,38 @@ # Changelog All notable changes to this project will be documented in this file. +## [6.0.0] - 2021-09-07 +### Changed +- Heavy refactoring of Publisher and BatchPublisher. +All code is separated in different modules and classes. + +1. Job callables are now called: +- single_publishing_job_class_callable +- batch_publishing_job_class_callable + +2. Now there are three main classes for messaging: +- TableSync::Publishing::Single - sends one row with initialization +- TableSync::Publishing::Batch - sends batch of rows with initialization +- TableSync::Publishing::Raw - sends raw data without checks + +Separate classes for publishing, object data, Rabbit params, debounce, serialization. + +3. Jobs are not constrained by being ActiveJob anymore. Just need to have #perform_at method + +4. Changed some method names towards consistency: +- attrs_for_routing_key -> attributes_for_routing_key +- attrs_for_metadata -> attributes_for_headers + +5. Moved TableSync setup into separate classes. + +6. Changed ORMAdapters. + +7. Destroyed objects are initialized. +Now custom attributes for destruction will be called on instances. +- Obj.table_sync_destroy_attributes() -> Obj#attributes_for_destroy + +8. Event constants are now kept in one place. + ## [5.0.1] - 2021-04-06 ### Fixed - documentation diff --git a/lib/table_sync.rb b/lib/table_sync.rb index b61efac..545a650 100644 --- a/lib/table_sync.rb +++ b/lib/table_sync.rb @@ -8,6 +8,7 @@ require "active_support/core_ext/numeric/time" module TableSync + require_relative "table_sync/event" require_relative "table_sync/utils" require_relative "table_sync/version" require_relative "table_sync/errors" @@ -30,7 +31,6 @@ module TableSync require_relative "table_sync/setup/sequel" class << self - attr_accessor :raise_on_serialization_failure attr_accessor :raise_on_empty_message attr_accessor :single_publishing_job_class_callable attr_accessor :batch_publishing_job_class_callable diff --git a/lib/table_sync/errors.rb b/lib/table_sync/errors.rb index 6ecfd14..c96a231 100644 --- a/lib/table_sync/errors.rb +++ b/lib/table_sync/errors.rb @@ -3,6 +3,18 @@ module TableSync Error = Class.new(StandardError) + NoObjectsForSyncError = Class.new(Error) + + class EventError < Error + def initialize(event) + super(<<~MSG) + Invalid event! + Given event: #{event}. + Valid events: #{TableSync::Event::VALID_RAW_EVENTS}. + MSG + end + end + class NoPrimaryKeyError < Error def initialize(object_class, object_data, primary_key_columns) super(<<~MSG) diff --git a/lib/table_sync/event.rb b/lib/table_sync/event.rb new file mode 100644 index 0000000..142eae0 --- /dev/null +++ b/lib/table_sync/event.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class TableSync::Event + attr_reader :event + + UPSERT_EVENTS = %i[create update].freeze + VALID_RESOLVED_EVENTS = %i[update destroy].freeze + VALID_RAW_EVENTS = %i[create update destroy].freeze + + def initialize(event) + @event = event + + validate! + end + + def validate! + raise TableSync::EventError.new(event) unless event.in?(VALID_RAW_EVENTS) + end + + def resolve + destroy? ? :destroy : :update + end + + def metadata + { created: event == :create } + end + + def destroy? + event == :destroy + end + + def upsert? + event.in?(UPSERT_EVENTS) + end +end diff --git a/lib/table_sync/publishing/data/objects.rb b/lib/table_sync/publishing/data/objects.rb index 9084fac..f1ab5e3 100644 --- a/lib/table_sync/publishing/data/objects.rb +++ b/lib/table_sync/publishing/data/objects.rb @@ -6,7 +6,7 @@ class Objects def initialize(objects:, event:) @objects = objects - @event = event + @event = TableSync::Event.new(event) end def construct @@ -14,8 +14,8 @@ def construct model: model, attributes: attributes_for_sync, version: version, - event: event, - metadata: metadata, + event: event.resolve, + metadata: event.metadata, } end @@ -33,26 +33,18 @@ def version Time.current.to_f end - def metadata - { created: event == :create } # remove? who needs this? - end - def object_class objects.first.object_class end def attributes_for_sync objects.map do |object| - if destruction? + if event.destroy? object.attributes_for_destroy else object.attributes_for_update end end end - - def destruction? - event == :destroy - end end end diff --git a/lib/table_sync/publishing/data/raw.rb b/lib/table_sync/publishing/data/raw.rb index 094e7ac..e52de81 100644 --- a/lib/table_sync/publishing/data/raw.rb +++ b/lib/table_sync/publishing/data/raw.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -# check if works! module TableSync::Publishing::Data class Raw attr_reader :object_class, :attributes_for_sync, :event @@ -8,7 +7,7 @@ class Raw def initialize(object_class:, attributes_for_sync:, event:) @object_class = object_class @attributes_for_sync = attributes_for_sync - @event = event + @event = TableSync::Event.new(event) end def construct @@ -16,15 +15,11 @@ def construct model: object_class, attributes: attributes_for_sync, version: version, - event: event, - metadata: metadata, + event: event.resolve, + metadata: event.metadata, } end - def metadata - { created: event == :create } # remove? who needs this? - end - def version Time.current.to_f end diff --git a/lib/table_sync/publishing/helpers/objects.rb b/lib/table_sync/publishing/helpers/objects.rb index e7b5386..1809090 100644 --- a/lib/table_sync/publishing/helpers/objects.rb +++ b/lib/table_sync/publishing/helpers/objects.rb @@ -5,13 +5,13 @@ class Objects attr_reader :object_class, :original_attributes, :event def initialize(object_class:, original_attributes:, event:) - @event = event + @event = TableSync::Event.new(event) @object_class = object_class.constantize @original_attributes = Array.wrap(original_attributes) end def construct_list - if destruction? + if event.destroy? init_objects else without_empty_objects(find_objects) @@ -35,9 +35,5 @@ def find_objects TableSync.publishing_adapter.new(object_class, attrs).find end end - - def destruction? - event == :destroy - end end end diff --git a/lib/table_sync/publishing/message/base.rb b/lib/table_sync/publishing/message/base.rb index 07b9535..0cc63fc 100644 --- a/lib/table_sync/publishing/message/base.rb +++ b/lib/table_sync/publishing/message/base.rb @@ -4,8 +4,6 @@ module TableSync::Publishing::Message class Base include Tainbox - NO_OBJECTS_FOR_SYNC = Class.new(StandardError) - attr_reader :objects attribute :object_class @@ -17,7 +15,7 @@ def initialize(params) @objects = find_or_init_objects - raise NO_OBJECTS_FOR_SYNC if objects.empty? && TableSync.raise_on_empty_message + raise TableSync::NoObjectsForSyncError if objects.empty? && TableSync.raise_on_empty_message end def publish diff --git a/lib/table_sync/publishing/raw.rb b/lib/table_sync/publishing/raw.rb index be2196d..6e1fa0f 100644 --- a/lib/table_sync/publishing/raw.rb +++ b/lib/table_sync/publishing/raw.rb @@ -19,22 +19,3 @@ def message TableSync::Publishing::Message::Raw.new(attributes) end end - -# event - -# debounce -# serialization -# def jobs -# enqueue - -# publishers - -# specs - -# docs - -# cases - -# changes - -# add validations? diff --git a/lib/table_sync/receiving.rb b/lib/table_sync/receiving.rb index 1e83923..dda279c 100644 --- a/lib/table_sync/receiving.rb +++ b/lib/table_sync/receiving.rb @@ -2,8 +2,6 @@ module TableSync module Receiving - AVAILABLE_EVENTS = [:update, :destroy].freeze - require_relative "receiving/config" require_relative "receiving/config_decorator" require_relative "receiving/dsl" diff --git a/lib/table_sync/receiving/config.rb b/lib/table_sync/receiving/config.rb index 774229a..a7d739b 100644 --- a/lib/table_sync/receiving/config.rb +++ b/lib/table_sync/receiving/config.rb @@ -4,20 +4,22 @@ module TableSync::Receiving class Config attr_reader :model, :events - def initialize(model:, events: AVAILABLE_EVENTS) + def initialize(model:, events: TableSync::Event::VALID_RESOLVED_EVENTS) @model = model @events = [events].flatten.map(&:to_sym) - unless @events.all? { |event| AVAILABLE_EVENTS.include?(event) } - raise TableSync::UndefinedEvent.new(events) - end + raise TableSync::UndefinedEvent.new(events) if any_invalid_events? self.class.default_values_for_options.each do |ivar, default_value_generator| instance_variable_set(ivar, default_value_generator.call(self)) end end + def any_invalid_events? + (events - TableSync::Event::VALID_RESOLVED_EVENTS).any? + end + class << self attr_reader :default_values_for_options diff --git a/lib/table_sync/receiving/handler.rb b/lib/table_sync/receiving/handler.rb index 13b6806..f0e3ee8 100644 --- a/lib/table_sync/receiving/handler.rb +++ b/lib/table_sync/receiving/handler.rb @@ -44,14 +44,18 @@ def data=(data) super(Array.wrap(data[:attributes])) end - def event=(name) - name = name.to_sym - raise TableSync::UndefinedEvent.new(event) unless %i[update destroy].include?(name) - super(name) + def event=(event_name) + event_name = event_name.to_sym + + if event_name.in?(TableSync::Event::VALID_RESOLVED_EVENTS) + super(event_name) + else + raise TableSync::UndefinedEvent.new(event) + end end - def model=(name) - super(name.to_s) + def model=(model_name) + super(model_name.to_s) end def configs diff --git a/spec/event_spec.rb b/spec/event_spec.rb new file mode 100644 index 0000000..f166a18 --- /dev/null +++ b/spec/event_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +describe TableSync::Event do + let(:service) { TableSync::Event.new(event) } + + context "invalid event" do + let(:event) { :invalid_event } + + it "raises error" do + expect { service }.to raise_error(TableSync::EventError) + end + end + + shared_examples "method returns correct value" do |meth, raw_event, value| + context "#{meth} - #{raw_event}" do + let(:event) { raw_event } + + it "returns #{value}" do + expect(service.public_send(meth)).to eq(value) + end + end + end + + describe "#resolve" do + include_examples "method returns correct value", :resolve, :create, :update + include_examples "method returns correct value", :resolve, :update, :update + include_examples "method returns correct value", :resolve, :destroy, :destroy + end + + describe "#metadata" do + include_examples "method returns correct value", :metadata, :create, { created: true } + include_examples "method returns correct value", :metadata, :update, { created: false } + include_examples "method returns correct value", :metadata, :destroy, { created: false } + end + + describe "#destroy?" do + include_examples "method returns correct value", :destroy?, :create, false + include_examples "method returns correct value", :destroy?, :update, false + include_examples "method returns correct value", :destroy?, :destroy, true + end + + describe "#upsert?" do + include_examples "method returns correct value", :upsert?, :create, true + include_examples "method returns correct value", :upsert?, :update, true + include_examples "method returns correct value", :upsert?, :destroy, false + end +end diff --git a/spec/publishing/data/objects_spec.rb b/spec/publishing/data/objects_spec.rb index 52945b1..31f0cff 100644 --- a/spec/publishing/data/objects_spec.rb +++ b/spec/publishing/data/objects_spec.rb @@ -6,6 +6,7 @@ let(:data) { described_class.new(**params) } let(:object_class) { ARecordUser } let(:event) { :update } + let(:resolved_event) { :update } let(:objects) { [object] } let(:expected_attributes) { object_class.first.attributes.symbolize_keys } let(:expected_model) { object_class.to_s } @@ -28,7 +29,7 @@ model: expected_model, attributes: [expected_attributes], version: an_instance_of(Float), - event: event, + event: resolved_event, metadata: metadata, } end @@ -67,7 +68,8 @@ end context "event -> destroy" do - let(:event) { :destroy } + let(:event) { :destroy } + let(:resolved_event) { :destroy } context "without #attributes_for_destroy" do let(:expected_attributes) { { id: object.object.id } } diff --git a/spec/publishing/data/raw_spec.rb b/spec/publishing/data/raw_spec.rb index 3d04669..7cbfbb0 100644 --- a/spec/publishing/data/raw_spec.rb +++ b/spec/publishing/data/raw_spec.rb @@ -5,6 +5,7 @@ let(:object_class) { "User" } let(:attributes_for_sync) { [{ id: 1, asd: "asd" }, { id: 22, time: Time.current }] } let(:event) { :update } + let(:resolved_event) { :update } let(:params) do { @@ -19,7 +20,7 @@ model: object_class, attributes: attributes_for_sync, version: an_instance_of(Float), - event: event, + event: resolved_event, metadata: metadata, } end diff --git a/spec/publishing/message/batch_spec.rb b/spec/publishing/message/batch_spec.rb index ee42961..bdc6afb 100644 --- a/spec/publishing/message/batch_spec.rb +++ b/spec/publishing/message/batch_spec.rb @@ -2,13 +2,15 @@ describe TableSync::Publishing::Message::Batch do describe "#publish" do + let(:event) { :destroy } + let(:attributes) do { object_class: object_class, original_attributes: [{ id: 1 }], routing_key: "users", headers: { kek: 1 }, - event: :destroy, + event: event, } end @@ -72,5 +74,24 @@ described_class.new(attributes).publish end + + context "with no objects found" do + let(:event) { :update } + + around do |example| + before_value = TableSync.raise_on_empty_message + + TableSync.raise_on_empty_message = true + + example.run + + TableSync.raise_on_empty_message = before_value + end + + it "raises error" do + expect { described_class.new(attributes).publish } + .to raise_error(TableSync::NoObjectsForSyncError) + end + end end end diff --git a/spec/publishing/message/single_spec.rb b/spec/publishing/message/single_spec.rb index c39bb15..ae87d1e 100644 --- a/spec/publishing/message/single_spec.rb +++ b/spec/publishing/message/single_spec.rb @@ -4,13 +4,15 @@ describe TableSync::Publishing::Message::Single do describe "#publish" do + let(:event) { :destroy } + let(:attributes) do { object_class: object_class, original_attributes: [{ id: 1 }], routing_key: "users", headers: { kek: 1 }, - event: :destroy, + event: event, } end @@ -70,5 +72,24 @@ described_class.new(attributes).publish end + + context "with no objects found" do + let(:event) { :update } + + around do |example| + before_value = TableSync.raise_on_empty_message + + TableSync.raise_on_empty_message = true + + example.run + + TableSync.raise_on_empty_message = before_value + end + + it "raises error" do + expect { described_class.new(attributes).publish } + .to raise_error(TableSync::NoObjectsForSyncError) + end + end end end From bc4055cfa095cef4d4bdc5170cec7aab9cdf5842 Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Mon, 27 Sep 2021 13:49:47 +0300 Subject: [PATCH 12/20] fix Gemfile.lock --- Gemfile.lock | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b7154b5..06b89cd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -98,7 +98,7 @@ GEM nokogiri (>= 1.5.9) mail (2.7.1) mini_mime (>= 0.1.1) - marcel (1.0.1) + marcel (1.0.2) memery (1.4.1) ruby2_keywords (~> 0.0.2) method_source (1.0.0) @@ -155,10 +155,7 @@ GEM thor (~> 1.0) rainbow (3.0.0) rake (13.0.6) -<<<<<<< HEAD -======= rbtree (0.4.4) ->>>>>>> master regexp_parser (2.1.1) rexml (3.2.5) rspec (3.10.0) From 836bc1d306c2f9f1a3bae823ce15468da77b17ef Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Wed, 29 Sep 2021 17:00:13 +0300 Subject: [PATCH 13/20] docs --- CHANGELOG.md | 4 + README.md | 3 + docs/publishing.md | 136 +-- docs/publishing/configuration.md | 143 +++ docs/publishing/manual.html | 1138 ++++++++++++++++++++ docs/publishing/manual.md | 155 +++ docs/publishing/publishers.md | 162 +++ lib/table_sync/publishing/batch.rb | 6 +- lib/table_sync/publishing/params/base.rb | 8 +- lib/table_sync/publishing/params/batch.rb | 4 +- lib/table_sync/publishing/params/single.rb | 4 +- lib/table_sync/setup/sequel.rb | 1 - 12 files changed, 1644 insertions(+), 120 deletions(-) create mode 100644 docs/publishing/configuration.md create mode 100644 docs/publishing/manual.html create mode 100644 docs/publishing/manual.md create mode 100644 docs/publishing/publishers.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c59919..e5971cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. ## [6.0.0] - 2021-10-01 +### Added + +A lot of specs for all the refactoring. + ### Changed - Heavy refactoring of Publisher and BatchPublisher. All code is separated in different modules and classes. diff --git a/README.md b/README.md index b6b3574..0723339 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,9 @@ require 'table_sync' - [Message protocol](docs/message_protocol.md) - [Publishing](docs/publishing.md) + - [Publishers](docs/publishing/publishers.md) + - [Configuration](docs/publishing/configuration.md) + - [Manual Sync (examples)](docs/publishing/manual.md) - [Receiving](docs/receiving.md) - [Notifications](docs/notifications.md) diff --git a/docs/publishing.md b/docs/publishing.md index 941a076..23868f4 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -1,133 +1,49 @@ -# Publishing changes +# Publishing -Include `TableSync.sync(self)` into a Sequel or ActiveRecord model. `:if` and `:unless` are supported for Sequel and ActiveRecord +TableSync can be used to send the data using RabbitMQ. -Functioning `Rails.cache` is required +You can send the data in two ways. Automatic and manual. +Each one has its own pros and cons. -Example: - -```ruby -class SomeModel < Sequel::Model - TableSync.sync(self, { if: -> (*) { some_code } }) -end -``` - -#### #attributes_for_sync - -Models can implement `#attributes_for_sync` to override which attributes are published. If not present, all attributes are published - -#### #attrs_for_routing_key - -Models can implement `#attrs_for_routing_key` to override which attributes are given to `routing_key_callable`. If not present, published attributes are given +Automatic is used to publish changes in realtime, as soon as the tracked entity changes. +Usually syncs one entity at a time. -#### #attrs_for_metadata +Manual allows to sync a lot of entities per message. +But demands greater amount of work and data preparation. -Models can implement `#attrs_for_metadata` to override which attributes are given to `metadata_callable`. If not present, published attributes are given +## Automatic -#### .table_sync_model_name +Include `TableSync.sync(self)` into a Sequel or ActiveRecord model. `:if` and `:unless` are supported for Sequel and ActiveRecord. -Models can implement `.table_sync_model_name` class method to override the model name used for publishing events. Default is model class name +Functioning `Rails.cache` is required. -#### .table_sync_destroy_attributes(original_attributes) - -Models can implement `.table_sync_destroy_attributes` class method to override the attributes used for publishing destroy events. Default is object's original attributes - -## Configuration - -- `TableSync.publishing_job_class_callable` is a callable which should resolve to a ActiveJob subclass that calls TableSync back to actually publish changes (required) +After some change happens, TableSync enqueues a job which then publishes a message. Example: ```ruby -class TableSync::Job < ActiveJob::Base - def perform(*args) - TableSync::Publishing::Publisher.new(*args).publish_now - end +class SomeModel < Sequel::Model + TableSync.sync(self, { if: -> (*) { some_code } }) end ``` -- `TableSync.batch_publishing_job_class_callable` is a callable which should resolve to a ActiveJob subclass that calls TableSync batch publisher back to actually publish changes (required for batch publisher) - -- `TableSync.routing_key_callable` is a callable which resolves which routing key to use when publishing changes. It receives object class and published attributes (required) - -Example: - -```ruby -TableSync.routing_key_callable = -> (klass, attributes) { klass.gsub('::', '_').tableize } -``` - -- `TableSync.routing_metadata_callable` is a callable that adds RabbitMQ headers which can be used in routing (optional). It receives object class and published attributes. One possible way of using it is defining a headers exchange and routing rules based on key-value pairs (which correspond to sent headers) - -Example: - -```ruby -TableSync.routing_metadata_callable = -> (klass, attributes) { attributes.slice("project_id") } -``` - -- `TableSync.exchange_name` defines the exchange name used for publishing (optional, falls back to default Rabbit gem configuration). - -- `TableSync.notifier` is a module that provides publish and recieve notifications. - -# Manual publishing - -`TableSync::Publishing::Publisher.new(object_class, original_attributes, confirm: true, state: :updated, debounce_time: 45)` -where state is one of `:created / :updated / :destroyed` and `confirm` is Rabbit's confirm delivery flag and optional param `debounce_time` determines debounce time in seconds, 1 minute by default. - -# Manual publishing with batches +## Manual -You can use `TableSync::Publishing::BatchPublisher` to publish changes in batches (array of hashes in `attributes`). - -When using `TableSync::Publishing::BatchPublisher`,` TableSync.routing_key_callable` is called as follows: `TableSync.routing_key_callable.call(klass, {})`, i.e. empty hash is passed instead of attributes. And `TableSync.routing_metadata_callable` is not called at all: metadata is set to empty hash. - -`TableSync::Publishing::BatchPublisher.new(object_class, original_attributes_array, **options)`, where `original_attributes_array` is an array with hash of attributes of published objects and `options` is a hash of options. - -`options` consists of: -- `confirm`, which is a flag for RabbitMQ, `true` by default -- `routing_key`, which is a custom key used (if given) to override one from `TableSync.routing_key_callable`, `nil` by default -- `push_original_attributes` (default value is `false`), if this option is set to `true`, -original_attributes_array will be pushed to Rabbit instead of fetching records from database and sending their mapped attributes. -- `headers`, which is an option for custom headers (can be used for headers exchanges routes), `nil` by default -- `event`, which is an option for event specification (`:destroy` or `:update`), `:update` by default +Directly call one of the publishers. It's the best if you need to sync a lot of data. +This way you don't even need for the changes to occur. Example: ```ruby -TableSync::Publishing::BatchPublisher.new( - "SomeClass", - [{ id: 1 }, { id: 2 }], - confirm: false, - routing_key: "custom_routing_key", - push_original_attributes: true, - headers: { key: :value }, - event: :destroy, -) + TableSync::Publishing::Batch.new( + object_class: "User", + original_attributes: [{ id: 1 }, { id: 2 }], + event: :update, + ).publish_now ``` -# Manual publishing with batches (Russian) - -С помощью класса `TableSync::Publishing::BatchPublisher` вы можете опубликовать изменения батчами (массивом в `attributes`). - -При использовании `TableSync::Publishing::BatchPublisher`, `TableSync.routing_key_callable` вызывается следующим образом: `TableSync.routing_key_callable.call(klass, {})`, то есть вместо аттрибутов передается пустой хэш. А `TableSync.routing_metadata_callable` не вызывается вовсе: в метадате устанавливается пустой хэш. - -`TableSync::Publishing::BatchPublisher.new(object_class, original_attributes_array, **options)`, где `original_attributes_array` - массив с аттрибутами публикуемых объектов и `options`- это хэш с дополнительными опциями. +## Read More -`options` состоит из: -- `confirm`, флаг для RabbitMQ, по умолчанию - `true` -- `routing_key`, ключ, который (если указан) замещает ключ, получаемый из `TableSync.routing_key_callable`, по умолчанию - `nil` -- `push_original_attributes` (значение по умолчанию `false`), если для этой опции задано значение true, в Rabbit будут отправлены original_attributes_array, вместо получения значений записей из базы непосредственно перед отправкой. -- `headers`, опция для задания headers (можно использовать для задания маршрутов в headers exchange'ах), `nil` по умолчанию -- `event`, опция для указания типа события (`:destroy` или `:update`), `:update` по умолчанию - -Example: - -```ruby -TableSync::Publishing::BatchPublisher.new( - "SomeClass", - [{ id: 1 }, { id: 2 }], - confirm: false, - routing_key: "custom_routing_key", - push_original_attributes: true, - headers: { key: :value }, - event: :destroy, -) -``` +- [Publishers](docs/publishing/publishers.md) +- [Configuration](docs/publishing/configuration.md) +- [Manual Sync (examples)](docs/publishing/manual.md) \ No newline at end of file diff --git a/docs/publishing/configuration.md b/docs/publishing/configuration.md new file mode 100644 index 0000000..e2352ab --- /dev/null +++ b/docs/publishing/configuration.md @@ -0,0 +1,143 @@ +# Configuration + +Customization, configuration and other options. + +## Model Customization + +There are methods you can define on a synched model to customize published messages for it. + +### `#attributes_for_sync` + +Models can implement `#attributes_for_sync` to override which attributes are published for `update` and `create` events. If not present, all attributes are published. + +### `#attributes_for_destroy` + +Models can implement `#attributes_for_destroy` to override which attributes are published for `destroy` events. If not present, `needle` (primary key) is published. + +### `#attributes_for_routing_key` + +Models can implement `#attributes_for_routing_key` to override which attributes are given to the `routing_key_callable`. If not present, published attributes are given. + +### `#attributes_for_headers` + +Models can implement `#attributes_for_headers` to override which attributes are given to the `headers_callable`. If not present, published attributes are given. + +### `.table_sync_model_name` + +Models can implement `.table_sync_model_name` class method to override the model name used for publishing events. Default is a model class name. + +## Callables + +Callables are defined once. TableSync will use them to dynamically resolve things like jobs, routing_key and headers. + +### Single publishing job (required for automatic and delayed publishing) + +- `TableSync.single_publishing_job_class_callable` is a callable which should resolve to a class that calls TableSync back to actually publish changes (required). + +It is expected to have `.perform_at(hash_with_options)` and it will be passed a hash with the following keys: + +- `original_attributes` - serialized `original_attributes` +- `object_class` - model name +- `debounce_time` - pause between pblishing messages +- `event` - type of event that happened to synched entity +- `perform_at` - time to perform the job at (depends on debounce) + +Example: + +```ruby +TableSync.single_publishing_job_class_callable = -> { TableSync::Job } + +class TableSync::Job < ActiveJob::Base + def perform(jsoned_attributes) + TableSync::Publishing::Single.new( + JSON.parse(jsoned_attributes), + ).publish_now + end + + def self.perform_at(attributes) + set(wait_until: attributes.delete(:perform_at)) + .perform_later(attributes.to_json) + end +end + +# will call the job described above + +TableSync::Publishing::Single.new( + object_class: "User", + original_attributes: { id: 1, name: "Mark" }, # will be serialized! + debounce_time: 60, + event: :update, +).publish_later +``` + +### Batch publishing job (required only for `Batch#publish_later`) + +- `TableSync.batch_publishing_job_class_callable` is a callable which should resolve to a class that calls TableSync back to actually publish changes. + +It is expected to have `.perform_later(hash_with_options)` and it will be passed a hash with the following keys: + +- `original_attributes` - array of serialized `original_attributes` +- `object_class` - model name +- `event` - type of event that happened to synched entity +- `routing_key` - custom routing_key (optional) +- `headers` - custom headers (optional) + +More often than not this job is not very useful, since it makes more sense to use `#publish_now` from an already existing job that does a lot of things (not just publishing messages). + +### Example + +```ruby +TableSync.batch_publishing_job_class_callable = -> { TableSync::BatchJob } + +class TableSync::BatchJob < ActiveJob::Base + def perform(jsoned_attributes) + TableSync::Publishing::Batch.new( + JSON.parse(jsoned_attributes), + ).publish_now + end + + def self.perform_later(attributes) + super(attributes.map(&:to_json)) + end +end + +TableSync::Publishing::Batch.new( + object_class: "User", + original_attributes: [{ id: 1, name: "Mark" }, { id: 2, name: "Bob" }], + event: :create, + routing_key: :custom_key, # optional + headers: { type: "admin" }, # optional +).publish_later +``` + +### Routing key callable (required) + +- `TableSync.routing_key_callable` is a callable which resolves which routing key to use when publishing changes. It receives object class and published attributes or `#attributes_for_routing_key` (if defined). + +Example: + +```ruby +TableSync.routing_key_callable = -> (klass, attributes) { klass.gsub('::', '_').tableize } +``` + +### Headers callable (required) + +- `TableSync.headers_callable` is a callable that adds RabbitMQ headers which can be used in routing. It receives object class and published attributes or `#attributes_for_headers` (if defined). + +One possible way of using it is defining a headers exchange and routing rules based on key-value pairs (which correspond to sent headers). + +Example: + +```ruby +TableSync.routing_metadata_callable = -> (klass, attributes) { attributes.slice("project_id") } +``` + +## Other + +- `TableSync.exchange_name` defines the exchange name used for publishing (optional, falls back to default Rabbit gem configuration). + +- `TableSync.notifier` is a module that provides publish and recieve notifications. + +- `TableSync.raise_on_empty_message` - raises error on empty message if set to true. + +- `TableSync.orm` - set ORM (ActiveRecord or Sequel) used to process given entities. Required! \ No newline at end of file diff --git a/docs/publishing/manual.html b/docs/publishing/manual.html new file mode 100644 index 0000000..8927b9b --- /dev/null +++ b/docs/publishing/manual.html @@ -0,0 +1,1138 @@ +manual

Manual Sync

+

There are two ways you can manually publish large amounts of data.

+

TableSync::Publishing:Batch

+

Easier to use, but does a lot of DB queries. May filter out invalid PK values.

+

Pros:

+
    +
  • requires less work
    • +
  • it will automatically use methods for data customization (#attributes_for_sync, #attributes_for_destroy)
    • +
  • you can be sure that data you publish is valid (more or less)
    • +
  • serializes values for (#publish_later)
    • +
+

Cons:

+
    +
  • it queries database for each entity in batch (for create and update)
    • +
  • it may also do a lot of queries if #attributes_for_sync contains additional data from other connected entities
    • +
  • serializes values for (#publish_later); if your PK contains invalid values (ex. Date) they will be filtered out
    • +
+

TableSync::Publishing:Raw

+

More complex to use, but requires very few DB queries. +You are responsible for the data you send!

+

Pros:

+
    +
  • very customizable; only requirement - object_class must exist
    • +
  • you can send whatever data you want
    • +
+

Cons:

+
    +
  • you have to manually prepare data for publishing
    • +
  • you have to be really sure you are sending valid data
    • +
+

Tips

+
    +
  • Don’t make data batches too large!
    • +
+

It may result in failure to process them. Good rule is to send approx. 5000 rows in one batch.

+
    +
  • Make pauses between publishing data batches!
    • +
+

Publishing without pause may overwhelm the receiving side. Either their background job processor (ex. Sidekiq) may clog with jobs, or their consumers may not be able to get messages from Rabbit server fast enough.

+

1 or 2 seconds is a good wait period.

+
    +
  • Do not use TableSync::Publishing:Single to send millions or even thousands of rows of data.
    • +
+

Or just calling update on rows with automatic sync. +It WILL overwhelm the receiving side. Especially if they have some additional receiving logic.

+
    +
  • On the receiving side don’t create job with custom logic (if it exists) for every row in a batch.
    • +
+

Better to process it whole. Otherwise three batches of 5000 will result in 15000 new jobs.

+
    +
  • Send one test batch before publishing the rest of the data.
    • +
+

Make sure it was received properly. This way you won’t send a lot invalid messages.

+
    +
  • +

    Check the other arguments.

    +
      +
    • Ensure the routing_key is correct if you are using a custom one. Remember, that batch publishing ignores #attributes_for_routing_key on a model.
      • +
    • Ensure that object_class is correct. And it belongs to entities you want to send.
      • +
    • Ensure that you chose the correct event.
      • +
    • If you have some logic depending on headers, ensure they are also correct.
      • +
    +
    • +
  • +

    You can check what you send before publishing with:

    +
    • +
+
TableSync::Publishing:Batch.new(...).message.message_params
+
+TableSync::Publishing:Raw.new(...).message.message_params
+
+ +

Examples

\ No newline at end of file diff --git a/docs/publishing/manual.md b/docs/publishing/manual.md new file mode 100644 index 0000000..8bf2c35 --- /dev/null +++ b/docs/publishing/manual.md @@ -0,0 +1,155 @@ +# Manual Sync + +There are two ways you can manually publish large amounts of data. + +### `TableSync::Publishing:Batch` + +Easier to use, but does a lot of DB queries. May filter out invalid PK values. + +#### Pros: + +- requires less work +- it will automatically use methods for data customization (`#attributes_for_sync`, `#attributes_for_destroy`) +- you can be sure that data you publish is valid (more or less) +- serializes values for (`#publish_later`) + +#### Cons: + +- it queries database for each entity in batch (for create and update) +- it may also do a lot of queries if `#attributes_for_sync` contains additional data from other connected entities +- serializes values for (`#publish_later`); if your PK contains invalid values (ex. Date) they will be filtered out + +### `TableSync::Publishing:Raw` + +More complex to use, but requires very few DB queries. +You are responsible for the data you send! + +#### Pros: + +- very customizable; only requirement - `object_class` must exist +- you can send whatever data you want + +#### Cons: + +- you have to manually prepare data for publishing +- you have to be really sure you are sending valid data + +## Tips + +- **Don't make data batches too large!** + +It may result in failure to process them. Good rule is to send approx. 5000 rows in one batch. + +- **Make pauses between publishing data batches!** + +Publishing without pause may overwhelm the receiving side. Either their background job processor (ex. Sidekiq) may clog with jobs, or their consumers may not be able to get messages from Rabbit server fast enough. + +1 or 2 seconds is a good wait period. + +- **Do not use `TableSync::Publishing:Single` to send millions or even thousands of rows of data.** + +Or just calling update on rows with automatic sync. +It WILL overwhelm the receiving side. Especially if they have some additional receiving logic. + +- **On the receiving side don't create job with custom logic (if it exists) for every row in a batch.** + +Better to process it whole. Otherwise three batches of 5000 will result in 15000 new jobs. + +- **Send one test batch before publishing the rest of the data.** + +Make sure it was received properly. This way you won't send a lot invalid messages. + +- **Check the other arguments.** + + - Ensure the routing_key is correct if you are using a custom one. Remember, that batch publishing ignores `#attributes_for_routing_key` on a model. + - Ensure that `object_class` is correct. And it belongs to entities you want to send. + - Ensure that you chose the correct event. + - If you have some logic depending on headers, ensure they are also correct. Remember, that batch publishing ignores `#attributes_for_headers` on a model. + +- **You can check what you send before publishing with:** + +```ruby +TableSync::Publishing:Batch.new(...).message.message_params + +TableSync::Publishing:Raw.new(...).message.message_params +``` + +## Examples + +### `TableSync::Publishing:Raw` + +```ruby + # For Sequel + # gem 'sequel-batches' or equivalent that will allow you to chunk data somehow + # or #each_page from Sequel + + # this is a simple query + # they can be much more complex, with joins and other things + # just make sure that it results in a set of data you expect + data = User.in_batches(of: 5000).naked.select(:id, :name, :email) + + data.each_with_index do |batch, i| + TableSync::Publishing::Raw.new( + object_class: "User", + original_attributes: batch, + event: :create, + routing_key: :custom_key, # optional + headers: { type: "admin" }, # optional + ).publish_now + + # make a puse between batches + sleep 1 + + # for when you are sending from terminal + # allows you to keep an eye on progress + # you can create more complex output + puts "Batch #{i} sent!" + end + +``` + +#### Another way to gather data + +If you don't want to create data query (maybe it's too complex) but there are a lot of quereing in `#attributes_for_sync` and you are willing to trade a little bit of perfomance, you can try the following. + +```ruby +class User < Sequel + one_to_many :user_info + + # For example our receiving side wants to know the ips user logged in under + # But doesn't want to sync the user_info + def attributes_for_sync + attributes.merge( + ips: user_info.ips + ) + end +end + +# to prevent the need to query for every peice of additional data we can user eager load +# and constract published data by calling #attributes_for_sync +# don't forget to chunk it into more managable sizes before trying to send +data = User.eager(:statuses).map { |user| user.attributes_for_sync } +``` +This way it will not make unnecessary queries. + +### `TableSync::Publishing::Batch` + +Remember, it will query or initialize each row. + +```ruby + # You can just send ids. + data = User.in_batches(of: 5000).naked.select(:id) + + data.each_with_index do |data, i| + TableSync::Publishing::Batch.new( + object_class: "User", + original_attributes: data, + event: :create, + routing_key: :custom_key, # optional + headers: { type: "admin" }, # optional + ).publish_now + + sleep 1 + puts "Batch #{i} sent!" + end +``` diff --git a/docs/publishing/publishers.md b/docs/publishing/publishers.md new file mode 100644 index 0000000..55d0bae --- /dev/null +++ b/docs/publishing/publishers.md @@ -0,0 +1,162 @@ +# Publishers + +There are three publishers you can use to send data. + +- `TableSync::Publishing::Single` - sends one row with initialization. +- `TableSync::Publishing::Batch` - sends batch of rows with initialization. +- `TableSync::Publishing::Raw` - sends raw data without checks. + +## Single + +`TableSync::Publishing::Single` - sends one row with initialization. + +This is a publisher called by `TableSync.sync(self)`. + +### Expected parameters: + +- `object_class` - class (model) used to initialize published object +- `original_attributes` - attributes used to initialize `object_class` with +- `debounce_time` - minimum allowed time between delayed publishings +- `event` - type of event that happened to the published object (create, update, destroy); `update` by default + +### What it does (when uses `#publish_now`): +- takes in the `original_attributes`, `object_class`, `event` +- constantizes `object_class` +- extracts the primary key (`needle`) of the `object_class` from the `original_attributes` +- queries the database for the object with the `needle` (for `update` and `create`) or initializes the `object_class` with it (for `destroy`) +- constructs routing_key using `routing_key_callable` and `#attributes_for_routing_key` (if defined) +- constructs headers using `headers_callable` and `#attributes_for_headers` (if defined) +- publishes Rabbit message (uses attributes from queried/initialized object as data) +- sends notification (if set up) + +### What it does (when uses `#publish_later`): +- takes in the `original_attributes`, `object_class`, `debounce_time`, `event` +- serializes the `original_attributes`, silently filters out unserializable keys/values +- enqueues (or skips) the job with the `serialized_original_attributes` to be performed in time according to debounce +- job (if enqueued) calls `TableSync::Publishing::Single#publish_now` with `serialized_original_attributes` and the same `object_class`, `debounce_time`, `event` + +### Serialization + +Currently allowed key/values are: + `NilClass`, `String`, `TrueClass`, `FalseClass`, `Numeric`, `Symbol`. + +### Job + +Job is defined in `TableSync.single_publishing_job_class_callable` as a proc. Read more in [Configuration](docs/publishing/configuration.md). + +### Example #1 (send right now) + +```ruby + TableSync::Publishing::Single.new( + object_class: "User", + original_attributes: { id: 1. name: "Mark" }, + debounce_time: 60, # useless for #publish _now, can be skipped + event: :create, + ).publish_now +``` + +### Example #2 (enqueue job) + +```ruby + TableSync::Publishing::Single.new( + object_class: "User", + original_attributes: { id: 1, name: "Mark" }, # will be serialized! + debounce_time: 60, + event: :update, + ).publish_later +``` + +## Batch + +- `TableSync::Publishing::Batch` - sends batch of rows with initialization. + +### Expected parameters: + +- `object_class` - class (model) used to initialize published objects +- `original_attributes` - array of attributes used to initialize `object_class` with +- `event` - type of event that happened to the published objects (create, update, destroy); `update` by default +- `routing_key` - custom routing_key +- `headers` - custom headers + +### What it does (when uses `#publish_now`): +- takes in the `original_attributes`, `object_class`, `event`, `routing_key`, `headers` +- constantizes `object_class` +- extracts primary keys (`needles`) of the `object_class` from the array of `original_attributes` +- queries the database for the objects with `needles` (for `update` and `create`) or initializes the `object_class` with it (for `destroy`) +- constructs routing_key using `routing_key_callable` (ignores `#attributes_for_routing_key`) or uses `routing_key` if given +- constructs headers using `headers_callable` (ignores `#attributes_for_headers`) or uses `headers` if given +- publishes Rabbit message (uses attributes from queried/initialized objects as data) +- sends notification (if set up) + +### What it does (when uses `#publish_later`): +- takes in the `original_attributes`, `object_class`, `event`, `routing_key`, `headers` +- serializes the array of `original_attributes`, silently filters out unserializable keys/values +- enqueues the job with the `serialized_original_attributes` +- job calls `TableSync::Publishing::Batch#publish_now` with `serialized_original_attributes` and the same `object_class`, `event`, `routing_key`, `headers` + +### Serialization + +Currently allowed key/values are: + `NilClass`, `String`, `TrueClass`, `FalseClass`, `Numeric`, `Symbol`. + +### Job + +Job is defined in `TableSync.batch_publishing_job_class_callable` as a proc. Read more in [Configuration](docs/publishing/configuration.md). + +### Example #1 (send right now) + +```ruby + TableSync::Publishing::Batch.new( + object_class: "User", + original_attributes: [{ id: 1, name: "Mark" }, { id: 2, name: "Bob" }], + event: :create, + routing_key: :custom_key, # optional + headers: { type: "admin" }, # optional + ).publish_now +``` + +### Example #2 (enqueue job) + +```ruby + TableSync::Publishing::Batch.new( + object_class: "User", + original_attributes: [{ id: 1, name: "Mark" }, { id: 2, name: "Bob" }], + event: :create, + routing_key: :custom_key, # optional + headers: { type: "admin" }, # optional + ).publish_later +``` + +## Raw +- `TableSync::Publishing::Raw` - sends raw data without checks. + +Be carefull with this publisher. There are no checks for the data sent. +You can send anything. + +### Expected parameters: + +- `object_class` - model +- `original_attributes` - raw data that will be sent +- `event` - type of event that happened to the published objects (create, update, destroy); `update` by default +- `routing_key` - custom routing_key +- `headers` - custom headers + +### What it does (when uses `#publish_now`): +- takes in the `original_attributes`, `object_class`, `event`, `routing_key`, `headers` +- constantizes `object_class` +- constructs routing_key using `routing_key_callable` (ignores `#attributes_for_routing_key`) or uses `routing_key` if given +- constructs headers using `headers_callable` (ignores `#attributes_for_headers`) or uses `headers` if given +- publishes Rabbit message (uses `original_attributes` as is) +- sends notification (if set up) + +### Example + +```ruby + TableSync::Publishing::Raw.new( + object_class: "User", + original_attributes: [{ id: 1, name: "Mark" }, { id: 2, name: "Bob" }], + event: :create, + routing_key: :custom_key, # optional + headers: { type: "admin" }, # optional + ).publish_now +``` diff --git a/lib/table_sync/publishing/batch.rb b/lib/table_sync/publishing/batch.rb index 6d018c0..3e6efaf 100644 --- a/lib/table_sync/publishing/batch.rb +++ b/lib/table_sync/publishing/batch.rb @@ -16,7 +16,11 @@ def publish_later end def publish_now - TableSync::Publishing::Message::Batch.new(attributes).publish + message.publish + end + + def message + TableSync::Publishing::Message::Batch.new(attributes) end private diff --git a/lib/table_sync/publishing/params/base.rb b/lib/table_sync/publishing/params/base.rb index bd58d4f..519128b 100644 --- a/lib/table_sync/publishing/params/base.rb +++ b/lib/table_sync/publishing/params/base.rb @@ -20,7 +20,7 @@ def construct def calculated_routing_key if TableSync.routing_key_callable - TableSync.routing_key_callable.call(object_class, attrs_for_routing_key) + TableSync.routing_key_callable.call(object_class, attributes_for_routing_key) else raise TableSync::NoCallableError.new("routing_key") end @@ -28,7 +28,7 @@ def calculated_routing_key def calculated_headers if TableSync.headers_callable - TableSync.headers_callable.call(object_class, attrs_for_headers) + TableSync.headers_callable.call(object_class, attributes_for_headers) else raise TableSync::NoCallableError.new("headers") end @@ -53,11 +53,11 @@ def exchange_name raise NotImplementedError end - def attrs_for_routing_key + def attributes_for_routing_key raise NotImplementedError end - def attrs_for_headers + def attributes_for_headers raise NotImplementedError end end diff --git a/lib/table_sync/publishing/params/batch.rb b/lib/table_sync/publishing/params/batch.rb index 1465956..bec9147 100644 --- a/lib/table_sync/publishing/params/batch.rb +++ b/lib/table_sync/publishing/params/batch.rb @@ -12,11 +12,11 @@ class Batch < Base private - def attrs_for_routing_key + def attributes_for_routing_key {} end - def attrs_for_headers + def attributes_for_headers {} end end diff --git a/lib/table_sync/publishing/params/single.rb b/lib/table_sync/publishing/params/single.rb index aebdf67..3c5e733 100644 --- a/lib/table_sync/publishing/params/single.rb +++ b/lib/table_sync/publishing/params/single.rb @@ -16,11 +16,11 @@ def object_class object.object_class.name end - def attrs_for_routing_key + def attributes_for_routing_key object.attributes_for_routing_key end - def attrs_for_headers + def attributes_for_headers object.attributes_for_headers end diff --git a/lib/table_sync/setup/sequel.rb b/lib/table_sync/setup/sequel.rb index 0b851b1..d5eebfa 100644 --- a/lib/table_sync/setup/sequel.rb +++ b/lib/table_sync/setup/sequel.rb @@ -8,7 +8,6 @@ def define_after_commit(event) options = options_exposed_for_block object_class.define_method("after_#{event}".to_sym) do - # if options[:if].call(self) && !options[:unless].call(self) if instance_eval(&options[:if]) && !instance_eval(&options[:unless]) db.after_commit do TableSync::Publishing::Single.new( From c3cee182c40587f06bde850935bbc2aa60394481 Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Wed, 29 Sep 2021 17:02:50 +0300 Subject: [PATCH 14/20] docs fix --- docs/publishing.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/publishing.md b/docs/publishing.md index 23868f4..9ec2415 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -44,6 +44,6 @@ Example: ## Read More -- [Publishers](docs/publishing/publishers.md) -- [Configuration](docs/publishing/configuration.md) -- [Manual Sync (examples)](docs/publishing/manual.md) \ No newline at end of file +- [Publishers](publishing/publishers.md) +- [Configuration](publishing/configuration.md) +- [Manual Sync (examples)](publishing/manual.md) \ No newline at end of file From 83edd7a5e44c018a96a83073e171eba082041fcd Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Wed, 29 Sep 2021 17:03:51 +0300 Subject: [PATCH 15/20] update nokogiri --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 06b89cd..1bb4d12 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,7 +106,7 @@ GEM mini_portile2 (2.6.1) minitest (5.14.4) nio4r (2.5.8) - nokogiri (1.12.4) + nokogiri (1.12.5) mini_portile2 (~> 2.6.1) racc (~> 1.4) parallel (1.20.1) From 64c5fd48e1ce42eb155369e88a5e9fdd6cc17b5b Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Thu, 30 Sep 2021 15:21:46 +0300 Subject: [PATCH 16/20] 100 coverage --- CHANGELOG.md | 8 +- docs/publishing.md | 4 +- docs/publishing/configuration.md | 18 +- docs/publishing/manual.html | 1138 ----------------- docs/publishing/manual.md | 10 +- docs/publishing/publishers.md | 14 +- lib/table_sync/errors.rb | 24 +- lib/table_sync/orm_adapter/active_record.rb | 4 - lib/table_sync/orm_adapter/base.rb | 14 +- lib/table_sync/orm_adapter/sequel.rb | 4 - .../publishing/helpers/attributes.rb | 2 +- lib/table_sync/publishing/message/base.rb | 2 + lib/table_sync/publishing/params/base.rb | 2 + lib/table_sync/setup/base.rb | 2 + spec/publishing/helpers/attributes_spec.rb | 6 +- spec/receiving/dsl_spec.rb | 64 +- spec/receiving/handler_spec.rb | 17 + spec/table_sync_spec.rb | 9 + 18 files changed, 133 insertions(+), 1209 deletions(-) delete mode 100644 docs/publishing/manual.html create mode 100644 spec/table_sync_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index e5971cb..e034928 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,9 @@ All notable changes to this project will be documented in this file. ## [6.0.0] - 2021-10-01 ### Added -A lot of specs for all the refactoring. +- A lot of specs for all the refactoring. +- Docs +- 100% coverage ### Changed - Heavy refactoring of Publisher and BatchPublisher. @@ -37,6 +39,10 @@ Now custom attributes for destruction will be called on instances. 8. Event constants are now kept in one place. +### Removed + +- Plugin Errors + ## [5.1.0] - 2021-09-09 ### Changed diff --git a/docs/publishing.md b/docs/publishing.md index 9ec2415..7707be8 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -1,8 +1,8 @@ # Publishing -TableSync can be used to send the data using RabbitMQ. +TableSync can be used to send data using RabbitMQ. -You can send the data in two ways. Automatic and manual. +You can do in two ways. Automatic and manual. Each one has its own pros and cons. Automatic is used to publish changes in realtime, as soon as the tracked entity changes. diff --git a/docs/publishing/configuration.md b/docs/publishing/configuration.md index e2352ab..f1b8a66 100644 --- a/docs/publishing/configuration.md +++ b/docs/publishing/configuration.md @@ -32,7 +32,7 @@ Callables are defined once. TableSync will use them to dynamically resolve thing ### Single publishing job (required for automatic and delayed publishing) -- `TableSync.single_publishing_job_class_callable` is a callable which should resolve to a class that calls TableSync back to actually publish changes (required). +- `TableSync.single_publishing_job_class_callable` is a callable which should resolve to a class that calls TableSync back to actually publish changes. It is expected to have `.perform_at(hash_with_options)` and it will be passed a hash with the following keys: @@ -60,7 +60,7 @@ class TableSync::Job < ActiveJob::Base end end -# will call the job described above +# will enqueue the job described above TableSync::Publishing::Single.new( object_class: "User", @@ -70,7 +70,7 @@ TableSync::Publishing::Single.new( ).publish_later ``` -### Batch publishing job (required only for `Batch#publish_later`) +### Batch publishing job (required only for `TableSync::Publishing::Batch#publish_later`) - `TableSync.batch_publishing_job_class_callable` is a callable which should resolve to a class that calls TableSync back to actually publish changes. @@ -97,13 +97,13 @@ class TableSync::BatchJob < ActiveJob::Base end def self.perform_later(attributes) - super(attributes.map(&:to_json)) + super(attributes.to_json) end end TableSync::Publishing::Batch.new( object_class: "User", - original_attributes: [{ id: 1, name: "Mark" }, { id: 2, name: "Bob" }], + original_attributes: [{ id: 1, name: "Mark" }, { id: 2, name: "Bob" }], # will be serialized! event: :create, routing_key: :custom_key, # optional headers: { type: "admin" }, # optional @@ -112,7 +112,7 @@ TableSync::Publishing::Batch.new( ### Routing key callable (required) -- `TableSync.routing_key_callable` is a callable which resolves which routing key to use when publishing changes. It receives object class and published attributes or `#attributes_for_routing_key` (if defined). +- `TableSync.routing_key_callable` is a callable that resolves which routing key to use when publishing changes. It receives object class and published attributes or `#attributes_for_routing_key` (if defined). Example: @@ -124,7 +124,7 @@ TableSync.routing_key_callable = -> (klass, attributes) { klass.gsub('::', '_'). - `TableSync.headers_callable` is a callable that adds RabbitMQ headers which can be used in routing. It receives object class and published attributes or `#attributes_for_headers` (if defined). -One possible way of using it is defining a headers exchange and routing rules based on key-value pairs (which correspond to sent headers). +One possible way of using it is defininga headers exchange and routing rules based on key-value pairs (which correspond to sent headers). Example: @@ -138,6 +138,6 @@ TableSync.routing_metadata_callable = -> (klass, attributes) { attributes.slice( - `TableSync.notifier` is a module that provides publish and recieve notifications. -- `TableSync.raise_on_empty_message` - raises error on empty message if set to true. +- `TableSync.raise_on_empty_message` - raises an error on empty message if set to true. -- `TableSync.orm` - set ORM (ActiveRecord or Sequel) used to process given entities. Required! \ No newline at end of file +- `TableSync.orm` - set ORM (ActiveRecord or Sequel) used to process given entities. Required! diff --git a/docs/publishing/manual.html b/docs/publishing/manual.html deleted file mode 100644 index 8927b9b..0000000 --- a/docs/publishing/manual.html +++ /dev/null @@ -1,1138 +0,0 @@ -manual

Manual Sync

-

There are two ways you can manually publish large amounts of data.

-

TableSync::Publishing:Batch

-

Easier to use, but does a lot of DB queries. May filter out invalid PK values.

-

Pros:

-
    -
  • requires less work
    • -
  • it will automatically use methods for data customization (#attributes_for_sync, #attributes_for_destroy)
    • -
  • you can be sure that data you publish is valid (more or less)
    • -
  • serializes values for (#publish_later)
    • -
-

Cons:

-
    -
  • it queries database for each entity in batch (for create and update)
    • -
  • it may also do a lot of queries if #attributes_for_sync contains additional data from other connected entities
    • -
  • serializes values for (#publish_later); if your PK contains invalid values (ex. Date) they will be filtered out
    • -
-

TableSync::Publishing:Raw

-

More complex to use, but requires very few DB queries. -You are responsible for the data you send!

-

Pros:

-
    -
  • very customizable; only requirement - object_class must exist
    • -
  • you can send whatever data you want
    • -
-

Cons:

-
    -
  • you have to manually prepare data for publishing
    • -
  • you have to be really sure you are sending valid data
    • -
-

Tips

-
    -
  • Don’t make data batches too large!
    • -
-

It may result in failure to process them. Good rule is to send approx. 5000 rows in one batch.

-
    -
  • Make pauses between publishing data batches!
    • -
-

Publishing without pause may overwhelm the receiving side. Either their background job processor (ex. Sidekiq) may clog with jobs, or their consumers may not be able to get messages from Rabbit server fast enough.

-

1 or 2 seconds is a good wait period.

-
    -
  • Do not use TableSync::Publishing:Single to send millions or even thousands of rows of data.
    • -
-

Or just calling update on rows with automatic sync. -It WILL overwhelm the receiving side. Especially if they have some additional receiving logic.

-
    -
  • On the receiving side don’t create job with custom logic (if it exists) for every row in a batch.
    • -
-

Better to process it whole. Otherwise three batches of 5000 will result in 15000 new jobs.

-
    -
  • Send one test batch before publishing the rest of the data.
    • -
-

Make sure it was received properly. This way you won’t send a lot invalid messages.

-
    -
  • -

    Check the other arguments.

    -
      -
    • Ensure the routing_key is correct if you are using a custom one. Remember, that batch publishing ignores #attributes_for_routing_key on a model.
      • -
    • Ensure that object_class is correct. And it belongs to entities you want to send.
      • -
    • Ensure that you chose the correct event.
      • -
    • If you have some logic depending on headers, ensure they are also correct.
      • -
    -
    • -
  • -

    You can check what you send before publishing with:

    -
    • -
-
TableSync::Publishing:Batch.new(...).message.message_params
-
-TableSync::Publishing:Raw.new(...).message.message_params
-
- -

Examples

\ No newline at end of file diff --git a/docs/publishing/manual.md b/docs/publishing/manual.md index 8bf2c35..598ebb1 100644 --- a/docs/publishing/manual.md +++ b/docs/publishing/manual.md @@ -15,7 +15,7 @@ Easier to use, but does a lot of DB queries. May filter out invalid PK values. #### Cons: -- it queries database for each entity in batch (for create and update) +- it queries database for each entity in batch (for `create` and `update`) - it may also do a lot of queries if `#attributes_for_sync` contains additional data from other connected entities - serializes values for (`#publish_later`); if your PK contains invalid values (ex. Date) they will be filtered out @@ -97,7 +97,7 @@ TableSync::Publishing:Raw.new(...).message.message_params headers: { type: "admin" }, # optional ).publish_now - # make a puse between batches + # make a pause between batches sleep 1 # for when you are sending from terminal @@ -110,7 +110,7 @@ TableSync::Publishing:Raw.new(...).message.message_params #### Another way to gather data -If you don't want to create data query (maybe it's too complex) but there are a lot of quereing in `#attributes_for_sync` and you are willing to trade a little bit of perfomance, you can try the following. +If you don't want to create a data query (maybe it's too complex) but there is a lot of quereing in `#attributes_for_sync` and you are willing to trade a little bit of perfomance, you can try the following. ```ruby class User < Sequel @@ -125,8 +125,8 @@ class User < Sequel end end -# to prevent the need to query for every peice of additional data we can user eager load -# and constract published data by calling #attributes_for_sync +# to prevent the need to query for every piece of additional data we can user eager load +# and construct published data by calling #attributes_for_sync # don't forget to chunk it into more managable sizes before trying to send data = User.eager(:statuses).map { |user| user.attributes_for_sync } ``` diff --git a/docs/publishing/publishers.md b/docs/publishing/publishers.md index 55d0bae..864a6a7 100644 --- a/docs/publishing/publishers.md +++ b/docs/publishing/publishers.md @@ -3,7 +3,7 @@ There are three publishers you can use to send data. - `TableSync::Publishing::Single` - sends one row with initialization. -- `TableSync::Publishing::Batch` - sends batch of rows with initialization. +- `TableSync::Publishing::Batch` - sends a batch of rows with initialization. - `TableSync::Publishing::Raw` - sends raw data without checks. ## Single @@ -17,13 +17,13 @@ This is a publisher called by `TableSync.sync(self)`. - `object_class` - class (model) used to initialize published object - `original_attributes` - attributes used to initialize `object_class` with - `debounce_time` - minimum allowed time between delayed publishings -- `event` - type of event that happened to the published object (create, update, destroy); `update` by default +- `event` - type of event that happened to the published object (`create`, `update`, `destroy`); `update` by default ### What it does (when uses `#publish_now`): - takes in the `original_attributes`, `object_class`, `event` - constantizes `object_class` - extracts the primary key (`needle`) of the `object_class` from the `original_attributes` -- queries the database for the object with the `needle` (for `update` and `create`) or initializes the `object_class` with it (for `destroy`) +- queries the database for the object with the `needle` (for `update` and `create`) or initializes the `object_class` with `original_attributes` (for `destroy`) - constructs routing_key using `routing_key_callable` and `#attributes_for_routing_key` (if defined) - constructs headers using `headers_callable` and `#attributes_for_headers` (if defined) - publishes Rabbit message (uses attributes from queried/initialized object as data) @@ -68,13 +68,13 @@ Job is defined in `TableSync.single_publishing_job_class_callable` as a proc. Re ## Batch -- `TableSync::Publishing::Batch` - sends batch of rows with initialization. +- `TableSync::Publishing::Batch` - sends a batch of rows with initialization. ### Expected parameters: - `object_class` - class (model) used to initialize published objects - `original_attributes` - array of attributes used to initialize `object_class` with -- `event` - type of event that happened to the published objects (create, update, destroy); `update` by default +- `event` - type of event that happened to the published objects (`create`, `update`, `destroy`); `update` by default - `routing_key` - custom routing_key - `headers` - custom headers @@ -82,7 +82,7 @@ Job is defined in `TableSync.single_publishing_job_class_callable` as a proc. Re - takes in the `original_attributes`, `object_class`, `event`, `routing_key`, `headers` - constantizes `object_class` - extracts primary keys (`needles`) of the `object_class` from the array of `original_attributes` -- queries the database for the objects with `needles` (for `update` and `create`) or initializes the `object_class` with it (for `destroy`) +- queries the database for the objects with `needles` (for `update` and `create`) or initializes the `object_class` with `original_attributes` (for `destroy`) - constructs routing_key using `routing_key_callable` (ignores `#attributes_for_routing_key`) or uses `routing_key` if given - constructs headers using `headers_callable` (ignores `#attributes_for_headers`) or uses `headers` if given - publishes Rabbit message (uses attributes from queried/initialized objects as data) @@ -137,7 +137,7 @@ You can send anything. - `object_class` - model - `original_attributes` - raw data that will be sent -- `event` - type of event that happened to the published objects (create, update, destroy); `update` by default +- `event` - type of event that happened to the published objects (`create`, `update`, `destroy`); `update` by default - `routing_key` - custom routing_key - `headers` - custom headers diff --git a/lib/table_sync/errors.rb b/lib/table_sync/errors.rb index c96a231..d9bfeb9 100644 --- a/lib/table_sync/errors.rb +++ b/lib/table_sync/errors.rb @@ -55,28 +55,6 @@ def initialize(data, target_keys, description) end end - # @api public - # @since 2.2.0 - PluginError = Class.new(Error) - - # @api public - # @since 2.2.0 - class UnregisteredPluginError < PluginError - # @param plugin_name [Any] - def initialize(plugin_name) - super("#{plugin_name} plugin is not registered") - end - end - - # @api public - # @since 2.2.0 - class AlreadyRegisteredPluginError < PluginError - # @param plugin_name [Any] - def initialize(plugin_name) - super("#{plugin_name} plugin already exists") - end - end - class InterfaceError < Error def initialize(object, method_name, parameters, description) parameters = parameters.map do |parameter| @@ -84,7 +62,9 @@ def initialize(object, method_name, parameters, description) case type when :req + #:nocov: name.to_s + #:nocov: when :keyreq "#{name}:" when :block diff --git a/lib/table_sync/orm_adapter/active_record.rb b/lib/table_sync/orm_adapter/active_record.rb index e9f9f5a..942dec6 100644 --- a/lib/table_sync/orm_adapter/active_record.rb +++ b/lib/table_sync/orm_adapter/active_record.rb @@ -2,10 +2,6 @@ module TableSync::ORMAdapter class ActiveRecord < Base - def primary_key - object_class.primary_key - end - def find @object = object_class.find_by(needle) diff --git a/lib/table_sync/orm_adapter/base.rb b/lib/table_sync/orm_adapter/base.rb index 69898ed..be949d5 100644 --- a/lib/table_sync/orm_adapter/base.rb +++ b/lib/table_sync/orm_adapter/base.rb @@ -39,10 +39,6 @@ def needle object_data.slice(*primary_key_columns) end - def primary_key_columns - Array.wrap(object_class.primary_key).map(&:to_sym) - end - # ATTRIBUTES def attributes_for_update @@ -77,6 +73,10 @@ def attributes_for_headers end end + def primary_key_columns + Array.wrap(object_class.primary_key).map(&:to_sym) + end + # MISC def empty? @@ -85,10 +85,7 @@ def empty? # NOT IMPLEMENTED - def primary_key - raise NotImplementedError - end - + # :nocov: def attributes raise NotImplementedError end @@ -96,5 +93,6 @@ def attributes def self.model_naming raise NotImplementedError end + # :nocov: end end diff --git a/lib/table_sync/orm_adapter/sequel.rb b/lib/table_sync/orm_adapter/sequel.rb index 1552694..3097234 100644 --- a/lib/table_sync/orm_adapter/sequel.rb +++ b/lib/table_sync/orm_adapter/sequel.rb @@ -2,10 +2,6 @@ module TableSync::ORMAdapter class Sequel < Base - def primary_key - object.pk_hash - end - def attributes object.values end diff --git a/lib/table_sync/publishing/helpers/attributes.rb b/lib/table_sync/publishing/helpers/attributes.rb index 7739323..baa1756 100644 --- a/lib/table_sync/publishing/helpers/attributes.rb +++ b/lib/table_sync/publishing/helpers/attributes.rb @@ -50,7 +50,7 @@ def filter_safe_for_serialization(object) def filter_safe_hash_values(value) case value when Symbol - value.to_s + value.to_s # why? else filter_safe_for_serialization(value) end diff --git a/lib/table_sync/publishing/message/base.rb b/lib/table_sync/publishing/message/base.rb index 0cc63fc..a2af8bc 100644 --- a/lib/table_sync/publishing/message/base.rb +++ b/lib/table_sync/publishing/message/base.rb @@ -48,9 +48,11 @@ def data ).construct end + # :nocov: def params raise NotImplementedError end + # :nocov: # NOTIFY diff --git a/lib/table_sync/publishing/params/base.rb b/lib/table_sync/publishing/params/base.rb index 519128b..178642f 100644 --- a/lib/table_sync/publishing/params/base.rb +++ b/lib/table_sync/publishing/params/base.rb @@ -37,6 +37,7 @@ def calculated_headers # NOT IMPLEMENTED # name of the model being synced in the string format + # :nocov: def object_class raise NotImplementedError end @@ -60,5 +61,6 @@ def attributes_for_routing_key def attributes_for_headers raise NotImplementedError end + # :nocov: end end diff --git a/lib/table_sync/setup/base.rb b/lib/table_sync/setup/base.rb index edd8bd4..a28fa6d 100644 --- a/lib/table_sync/setup/base.rb +++ b/lib/table_sync/setup/base.rb @@ -50,9 +50,11 @@ def applicable_events # CREATING HOOKS + # :nocov: def define_after_commit(event) raise NotImplementedError end + # :nocov: def options_exposed_for_block { diff --git a/spec/publishing/helpers/attributes_spec.rb b/spec/publishing/helpers/attributes_spec.rb index 8585ab8..2b87043 100644 --- a/spec/publishing/helpers/attributes_spec.rb +++ b/spec/publishing/helpers/attributes_spec.rb @@ -28,8 +28,10 @@ end context "deep hash" do - let(:original_attributes) { { id: { safe: 1, unsafe: Time.current, Time.current => "7" } } } - let(:expected_attributes) { { id: { safe: 1 } } } + let(:original_attributes) do + { id: { safe: :yes, unsafe: Time.current, Time.current => "7" } } + end + let(:expected_attributes) { { id: { safe: "yes" } } } include_examples "filters out unsafe keys/values" end diff --git a/spec/receiving/dsl_spec.rb b/spec/receiving/dsl_spec.rb index 4f1e0c7..02f16d9 100644 --- a/spec/receiving/dsl_spec.rb +++ b/spec/receiving/dsl_spec.rb @@ -50,12 +50,6 @@ config.wrap_receiving.call(data: "data", &test_block) end - it "raises Interface error" do - model = Object.new - - expect { handler1.receive("User", to_model: model) }.to raise_error(TableSync::InterfaceError) - end - describe "inherited handler" do before { handler1.receive("User", to_table: :clients) } @@ -69,5 +63,63 @@ expect(handler3.configs["User"].size).to eq(2) end end + + context "to_model" do + let(:valid_model) do + Class.new do + class << self + def upsert(data:, target_keys:, version_key:, default_values:); end + + def destroy(data:, target_keys:); end + + def transaction(&block); end + + def after_commit(&block); end + + def columns; end + + def primary_keys; end + + def table; end + + def schema; end + end + end + end + + it "doesn't raise error" do + expect { handler1.receive("User", to_model: valid_model) }.not_to raise_error + end + + context "invalid model" do + TableSync::Utils::InterfaceChecker::INTERFACES[:receiving_model].each do |meth| + context "without #{meth.first}" do + let(:invalid_model) { valid_model } + + before { invalid_model.singleton_class.undef_method(meth.first) } + + it "raises Interface Error" do + expect do + handler1.receive("User", to_model: invalid_model) + end.to raise_error(TableSync::InterfaceError) + end + end + end + + context "method present, but incorrect" do + let(:invalid_model) { valid_model } + + before do + invalid_model.singleton_class.define_method(:upsert) { "kek" } + end + + it "raises Interface Error" do + expect do + handler1.receive("User", to_model: invalid_model) + end.to raise_error(TableSync::InterfaceError) + end + end + end + end end end diff --git a/spec/receiving/handler_spec.rb b/spec/receiving/handler_spec.rb index 6743b2a..6399b6a 100644 --- a/spec/receiving/handler_spec.rb +++ b/spec/receiving/handler_spec.rb @@ -526,6 +526,23 @@ def destroy(data:, target_keys:, version_key:) ) end + describe "error with invalid event" do + let(:create_event) do + OpenStruct.new( + data: { + event: "create", + model: "User", + attributes: { id: user_id }, + }, + project_id: "pid", + ) + end + + it "raises TableSync::UndefinedEvent" do + expect { described_class.new(create_event).call }.to raise_error(TableSync::UndefinedEvent) + end + end + describe "error with target keys" do let(:handler) do Class.new(described_class).receive("User", to_table: :users) do diff --git a/spec/table_sync_spec.rb b/spec/table_sync_spec.rb new file mode 100644 index 0000000..57b1c06 --- /dev/null +++ b/spec/table_sync_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +describe TableSync do + context "invalid ORM" do + it "raises error" do + expect { TableSync.orm = :incorrect_orm }.to raise_error(TableSync::ORMNotSupported) + end + end +end From 71551cd2ea380781ce029979b29c01291f49bf5b Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Fri, 8 Oct 2021 05:44:29 -0400 Subject: [PATCH 17/20] review fixes --- docs/publishing/publishers.md | 2 +- lib/table_sync/errors.rb | 11 +++++------ lib/table_sync/orm_adapter/active_record.rb | 6 ++++++ lib/table_sync/orm_adapter/base.rb | 6 ------ lib/table_sync/orm_adapter/sequel.rb | 8 ++++++++ lib/table_sync/publishing/single.rb | 2 +- lib/table_sync/receiving/config.rb | 6 +++--- 7 files changed, 24 insertions(+), 17 deletions(-) diff --git a/docs/publishing/publishers.md b/docs/publishing/publishers.md index 864a6a7..3d16b08 100644 --- a/docs/publishing/publishers.md +++ b/docs/publishing/publishers.md @@ -49,7 +49,7 @@ Job is defined in `TableSync.single_publishing_job_class_callable` as a proc. Re ```ruby TableSync::Publishing::Single.new( object_class: "User", - original_attributes: { id: 1. name: "Mark" }, + original_attributes: { id: 1, name: "Mark" }, debounce_time: 60, # useless for #publish _now, can be skipped event: :create, ).publish_now diff --git a/lib/table_sync/errors.rb b/lib/table_sync/errors.rb index d9bfeb9..16e3375 100644 --- a/lib/table_sync/errors.rb +++ b/lib/table_sync/errors.rb @@ -7,17 +7,16 @@ module TableSync class EventError < Error def initialize(event) - super(<<~MSG) - Invalid event! - Given event: #{event}. - Valid events: #{TableSync::Event::VALID_RAW_EVENTS}. + super(<<~MSG.squish) + Event #{event.inspect} is invalid.#{' '} + Expected: #{TableSync::Event::VALID_RAW_EVENTS.inspect}. MSG end end class NoPrimaryKeyError < Error def initialize(object_class, object_data, primary_key_columns) - super(<<~MSG) + super(<<~MSG.squish) Can't find or init an object of #{object_class} with #{object_data.inspect}. Incomplete primary key! object_data must contain: #{primary_key_columns.inspect}. MSG @@ -26,7 +25,7 @@ def initialize(object_class, object_data, primary_key_columns) class NoCallableError < Error def initialize(type) - super(<<~MSG) + super(<<~MSG.squish) Can't find callable for #{type}! Please initialize TableSync.#{type}_callable with the correct proc! MSG diff --git a/lib/table_sync/orm_adapter/active_record.rb b/lib/table_sync/orm_adapter/active_record.rb index 942dec6..03f4b13 100644 --- a/lib/table_sync/orm_adapter/active_record.rb +++ b/lib/table_sync/orm_adapter/active_record.rb @@ -8,6 +8,12 @@ def find super end + def init + @object = object_class.new(object_data) + + super + end + def attributes object.attributes.symbolize_keys end diff --git a/lib/table_sync/orm_adapter/base.rb b/lib/table_sync/orm_adapter/base.rb index be949d5..039addd 100644 --- a/lib/table_sync/orm_adapter/base.rb +++ b/lib/table_sync/orm_adapter/base.rb @@ -22,12 +22,6 @@ def validate! # FIND OR INIT OBJECT def init - @object = object_class.new(object_data.except(*primary_key_columns)) - - needle.each do |column, value| - @object.send("#{column}=", value) - end - self end diff --git a/lib/table_sync/orm_adapter/sequel.rb b/lib/table_sync/orm_adapter/sequel.rb index 3097234..8097a25 100644 --- a/lib/table_sync/orm_adapter/sequel.rb +++ b/lib/table_sync/orm_adapter/sequel.rb @@ -6,6 +6,14 @@ def attributes object.values end + def init + @object = object_class.new(object_data.except(*primary_key_columns)) + + @object.set_fields(needle, needle.keys) + + super + end + def find @object = object_class.find(needle) diff --git a/lib/table_sync/publishing/single.rb b/lib/table_sync/publishing/single.rb index 8c4dd87..063b6fa 100644 --- a/lib/table_sync/publishing/single.rb +++ b/lib/table_sync/publishing/single.rb @@ -44,7 +44,7 @@ def publish_now def job if TableSync.single_publishing_job_class_callable - TableSync.single_publishing_job_class_callable&.call + TableSync.single_publishing_job_class_callable.call else raise TableSync::NoCallableError.new("single_publishing_job_class") end diff --git a/lib/table_sync/receiving/config.rb b/lib/table_sync/receiving/config.rb index a7d739b..bee668e 100644 --- a/lib/table_sync/receiving/config.rb +++ b/lib/table_sync/receiving/config.rb @@ -9,15 +9,15 @@ def initialize(model:, events: TableSync::Event::VALID_RESOLVED_EVENTS) @events = [events].flatten.map(&:to_sym) - raise TableSync::UndefinedEvent.new(events) if any_invalid_events? + raise TableSync::UndefinedEvent.new(events) if invalid_events.any? self.class.default_values_for_options.each do |ivar, default_value_generator| instance_variable_set(ivar, default_value_generator.call(self)) end end - def any_invalid_events? - (events - TableSync::Event::VALID_RESOLVED_EVENTS).any? + def invalid_events + events - TableSync::Event::VALID_RESOLVED_EVENTS end class << self From 44ce04d91c86ac435cdab48c06d407c1a233a78f Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Fri, 8 Oct 2021 05:55:49 -0400 Subject: [PATCH 18/20] #publish_async --- lib/table_sync/publishing/batch.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/table_sync/publishing/batch.rb b/lib/table_sync/publishing/batch.rb index 3e6efaf..e67ee1c 100644 --- a/lib/table_sync/publishing/batch.rb +++ b/lib/table_sync/publishing/batch.rb @@ -23,6 +23,8 @@ def message TableSync::Publishing::Message::Batch.new(attributes) end + alias_method :publish_async, :publish_later + private # JOB From 4a827aae3722ae44e93501828a3309f5090b1bdb Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Tue, 12 Oct 2021 05:19:44 -0400 Subject: [PATCH 19/20] docs fixes --- docs/publishing.md | 20 +++++++++++++++++--- docs/publishing/configuration.md | 4 ++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/publishing.md b/docs/publishing.md index 7707be8..5a03f15 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -13,17 +13,31 @@ But demands greater amount of work and data preparation. ## Automatic -Include `TableSync.sync(self)` into a Sequel or ActiveRecord model. `:if` and `:unless` are supported for Sequel and ActiveRecord. +Include `TableSync.sync(self)` into a Sequel or ActiveRecord model. + +Options: + +- `if:` and `unless:` - Runs given proc in the scope of an instance. Skips sync on `false` for `if:` and on `true` for `unless:`. +- `on:` - specify events (`create`, `update`, `destroy`) to trigger sync on. Triggered for all of them without this option. +- `debounce_time` - min time period allowed between synchronizations. Functioning `Rails.cache` is required. -After some change happens, TableSync enqueues a job which then publishes a message. +How it works: + +- `TableSync.sync(self)` - registers new callbacks (for `create`, `update`, `destroy`) for ActiveRecord model, and defines `after_create`, `after_update` and `after_destroy` callback methods for Sequel model. + +- Callbacks call `TableSync::Publishing::Single#publish_later` with given options and object attributes. It enqueues a job which then publishes a message. Example: ```ruby class SomeModel < Sequel::Model - TableSync.sync(self, { if: -> (*) { some_code } }) + TableSync.sync(self, { if: -> (*) { some_code }, unless: -> (*) { some_code }, on: [:create, :update] }) +end + +class SomeOtherModel < Sequel::Model + TableSync.sync(self) end ``` diff --git a/docs/publishing/configuration.md b/docs/publishing/configuration.md index f1b8a66..3f508c9 100644 --- a/docs/publishing/configuration.md +++ b/docs/publishing/configuration.md @@ -38,7 +38,7 @@ It is expected to have `.perform_at(hash_with_options)` and it will be passed a - `original_attributes` - serialized `original_attributes` - `object_class` - model name -- `debounce_time` - pause between pblishing messages +- `debounce_time` - pause between publishing messages - `event` - type of event that happened to synched entity - `perform_at` - time to perform the job at (depends on debounce) @@ -124,7 +124,7 @@ TableSync.routing_key_callable = -> (klass, attributes) { klass.gsub('::', '_'). - `TableSync.headers_callable` is a callable that adds RabbitMQ headers which can be used in routing. It receives object class and published attributes or `#attributes_for_headers` (if defined). -One possible way of using it is defininga headers exchange and routing rules based on key-value pairs (which correspond to sent headers). +One possible way of using it is defining a headers exchange and routing rules based on key-value pairs (which correspond to sent headers). Example: From e039f827fa1c107f9a39b0c0b0357f060851a075 Mon Sep 17 00:00:00 2001 From: VegetableProphet Date: Fri, 15 Oct 2021 11:57:14 +0300 Subject: [PATCH 20/20] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e034928..233db0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog All notable changes to this project will be documented in this file. -## [6.0.0] - 2021-10-01 +## [6.0.0] - 2021-10-15 ### Added - A lot of specs for all the refactoring.