Merge pull request #2498 from internetee/fix-statistics

Refactored market share distribution request

Gemfile

@@ -78,6 +78,8 @@ gem 'wkhtmltopdf-binary', '~> 0.12.5.1'
 
 gem 'directo', github: 'internetee/directo', branch: 'master'
 
+gem 'strong_migrations'
+
 group :development, :test do
   gem 'pry', '0.14.1'
 end

Gemfile.lock

@@ -474,6 +474,8 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     spy (1.0.1)
+    strong_migrations (1.4.0)
+      activerecord (>= 5.2)
     swd (1.3.0)
       activesupport (>= 3)
       attr_required (>= 0.0.5)
@@ -588,6 +590,7 @@ DEPENDENCIES
   simplecov (= 0.17.1)
   simpleidn (= 0.2.1)
   spy
+  strong_migrations
   truemail (~> 3.0)
   uglifier
   validates_email_format_of (= 1.7.2)
@@ -597,4 +600,4 @@ DEPENDENCIES
   wkhtmltopdf-binary (~> 0.12.5.1)
 
 BUNDLED WITH
-   2.3.25
+   2.3.26

app/controllers/repp/v1/stats_controller.rb

@@ -1,21 +1,16 @@
 module Repp
   module V1
-    class StatsController < BaseController
+    class StatsController < BaseController # rubocop:disable Metrics/ClassLength
+      before_action :set_date_params
+
       api :get, '/repp/v1/stats/market_share_distribution'
       desc 'Get market share and distribution of registrars'
       param :q, Hash, required: true, desc: 'Period parameters for data' do
         param :end_date, String, required: true, desc: 'Period end date'
       end
       def market_share_distribution
-        registrars = ::Registrar.where(test_registrar: false).joins(:domains)
-                                .where(from_condition).where(to_condition)
-        grouped = registrars.group(:name).count
-
-        result = grouped.map do |key, value|
-          hash = { name: key.strip, y: value }
-          hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == key
-          hash
-        end
+        domains_by_rar = domains_by_registrar(@date_to, @date_from)
+        result = serialize_distribution_result(domains_by_rar)
         render_success(data: result)
       end
 
@@ -24,60 +19,45 @@ def market_share_distribution
       desc 'Get market share and growth rate of registrars'
       param :q, Hash, required: true, desc: 'Period parameters for data' do
         param :end_date, String, required: true, desc: 'Period end date'
-        param :compare_to_date, String, required: true, desc: 'Comparison date'
+        param :compare_to_end_date, String, required: true, desc: 'Comparison date'
       end
       def market_share_growth_rate
-        registrars = ::Registrar.where(test_registrar: false).joins(:domains)
-
-        domains_by_rar = registrars.where(from_condition).where(to_condition).group(:name).count
-        prev_domains_by_rar = registrars.where(compare_to_condition).group(:name).count
+        domains_by_rar = domains_by_registrar(@date_to, @date_from)
+        prev_domains_by_rar = domains_by_registrar(@date_compare_to, @date_compare_from)
 
         set_zero_values!(domains_by_rar, prev_domains_by_rar)
 
         market_share_by_rar = calculate_market_share(domains_by_rar)
         prev_market_share_by_rar = calculate_market_share(prev_domains_by_rar)
 
-        result = { prev_data: { name: search_params[:compare_to_date],
-                                domains: serialize(prev_domains_by_rar),
-                                market_share: serialize(prev_market_share_by_rar) },
+        result = { prev_data: { name: search_params[:compare_to_end_date],
+                                domains: serialize_growth_rate_result(prev_domains_by_rar),
+                                market_share: serialize_growth_rate_result(prev_market_share_by_rar) },
                    data: { name: search_params[:end_date],
-                           domains: serialize(domains_by_rar),
-                           market_share: serialize(market_share_by_rar) } }
+                           domains: serialize_growth_rate_result(domains_by_rar),
+                           market_share: serialize_growth_rate_result(market_share_by_rar) } }
         render_success(data: result)
       end
       # rubocop:enable Metrics/MethodLength
 
       private
 
       def search_params
-        params.permit(:q, q: %i[start_date end_date compare_to_date])
+        params.permit(:q, q: %i[start_date end_date compare_to_end_date compare_to_start_date])
               .fetch(:q, {}) || {}
       end
 
-      def from_condition
-        return unless search_params[:start_date]
-
-        "domains.created_at >= '#{to_date(search_params[:start_date])}'"
-      end
-
-      def to_condition
-        return unless search_params[:end_date]
-
-        "domains.created_at <= '#{to_date(search_params[:end_date]).end_of_month}'"
-      end
-
-      def compare_to_condition
-        return unless search_params[:compare_to_date]
-
-        "domains.created_at <= '#{to_date(search_params[:compare_to_date]).end_of_month}'"
+      def set_date_params
+        @date_to = to_date(search_params[:end_date]).end_of_month
+        @date_from = to_date(search_params[:start_date] || '01.05')
+        @date_compare_to = to_date(search_params[:compare_to_end_date]).end_of_month
+        @date_compare_from = to_date(search_params[:compare_to_start_date] || '01.05')
       end
 
       def to_date(date_param)
-        Date.strptime(date_param, '%m.%y')
-      end
+        return Time.zone.today if date_param.blank?
 
-      def serialize(rars)
-        rars.map { |key, value| [key, value] }
+        Date.strptime(date_param, '%m.%y')
       end
 
       def set_zero_values!(cur, prev)
@@ -97,6 +77,62 @@ def calculate_market_share(domains_by_rar)
           value < 0.1 ? value.round(3) : value.round(1)
         end
       end
+
+      def domains_by_registrar(date_to, date_from)
+        log_domains_del = log_domains(event: 'destroy', date_to: date_to, date_from: date_from)
+        log_domains_trans = log_domains(event: 'update', date_to: date_to, date_from: date_from)
+        logged_domains = log_domains_trans.map { |ld| ld.object['name'] } +
+                         log_domains_del.map { |ld| ld.object['name'] }
+        domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from)
+                                  .where.not(name: logged_domains.uniq)
+                                  .group(:registrar_id).count.stringify_keys
+        summarize([group(log_domains_del), group(log_domains_trans), domains_grouped])
+      end
+
+      def summarize(arr)
+        arr.inject { |memo, el| memo.merge(el) { |_, old_v, new_v| old_v + new_v } }
+      end
+
+      def log_domains(event:, date_to:, date_from:)
+        domains = ::Version::DomainVersion.where(event: event)
+        domains.where!("object_changes ->> 'registrar_id' IS NOT NULL") if event == 'update'
+        domains.where('created_at > ?', date_to)
+               .where("object ->> 'created_at' <= ?", date_to)
+               .where("object ->> 'created_at' >= ?", date_from)
+               .select("DISTINCT ON (object ->> 'name') object, created_at")
+               .order(Arel.sql("object ->> 'name', created_at desc"))
+      end
+
+      def group(domains)
+        domains.group_by { |ld| ld.object['registrar_id'].to_s }
+               .transform_values(&:count)
+      end
+
+      def registrar_names
+        @registrar_names ||= ::Registrar.where(test_registrar: false)
+                                        .map { |r| { "#{r.id}": r.name }.with_indifferent_access }
+                                        .reduce({}, :merge)
+      end
+
+      def serialize_distribution_result(result)
+        result.map do |key, value|
+          next unless registrar_names.key?(key)
+
+          name = registrar_names[key]
+          hash = { name: name, y: value }
+          hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name
+          hash
+        end.compact
+      end
+
+      def serialize_growth_rate_result(result)
+        result.map do |key, value|
+          next unless registrar_names.key?(key)
+
+          name = registrar_names[key]
+          [name, value]
+        end.compact
+      end
     end
   end
 end

config/initializers/strong_migrations.rb

@@ -0,0 +1,26 @@
+# Mark existing migrations as safe
+StrongMigrations.start_after = 20221206081743
+
+# Set timeouts for migrations
+# If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user
+StrongMigrations.lock_timeout = 10.seconds
+StrongMigrations.statement_timeout = 1.hour
+
+# Analyze tables after indexes are added
+# Outdated statistics can sometimes hurt performance
+StrongMigrations.auto_analyze = true
+
+# Set the version of the production database
+# so the right checks are run in development
+# StrongMigrations.target_version = 10
+
+# Add custom checks
+# StrongMigrations.add_check do |method, args|
+#   if method == :add_index && args[0].to_s == "users"
+#     stop! "No more indexes on the users table"
+#   end
+# end
+
+# Make some operations safe by default
+# See https://github.com/ankane/strong_migrations#safe-by-default
+# StrongMigrations.safe_by_default = true

db/migrate/20221206090120_modify_log_domains_object_changes_data_type.rb

@@ -0,0 +1,21 @@
+class ModifyLogDomainsObjectChangesDataType < ActiveRecord::Migration[6.1]
+  def up
+    add_column :log_domains, :object_changes_jsonb, :jsonb, default: '{}'
+
+    # Copy data from old column to the new one
+    Version::DomainVersion.update_all('object_changes_jsonb = object_changes::jsonb')
+
+    # Rename columns instead of modify their type, it's way faster
+    safety_assured do
+      rename_column :log_domains, :object_changes, :object_changes_json
+      rename_column :log_domains, :object_changes_jsonb, :object_changes
+    end
+  end
+
+  def down
+    safety_assured do
+      rename_column :log_domains, :object_changes, :object_changes_jsonb
+      rename_column :log_domains, :object_changes_json, :object_changes
+    end
+  end
+end

db/migrate/20221206091556_add_indexes_to_log_domains.rb

@@ -0,0 +1,15 @@
+class AddIndexesToLogDomains < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+  def up
+    enable_extension 'btree_gin'
+    add_index :log_domains, :event, algorithm: :concurrently
+    add_index :log_domains, :object, using: :gin, algorithm: :concurrently
+    add_index :log_domains, :object_changes, using: :gin, algorithm: :concurrently
+  end
+
+  def down
+    remove_index :log_domains, :event
+    remove_index :log_domains, :object
+    remove_index :log_domains, :object_changes
+  end
+end

db/migrate/20221207102831_change_log_domains_object_changes_default_value.rb

@@ -0,0 +1,5 @@
+class ChangeLogDomainsObjectChangesDefaultValue < ActiveRecord::Migration[6.1]
+  def change
+    change_column_default :log_domains, :object_changes, nil
+  end
+end