diff --git a/Gemfile b/Gemfile index 75f845477e..0b8b291895 100644 --- a/Gemfile +++ b/Gemfile @@ -78,6 +78,8 @@ gem 'wkhtmltopdf-binary', '~> 0.12.5.1' gem 'directo', github: 'internetee/directo', branch: 'master' +gem 'strong_migrations' + group :development, :test do gem 'pry', '0.14.1' end diff --git a/Gemfile.lock b/Gemfile.lock index ce00e5fdbc..2ebca44722 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -474,6 +474,8 @@ GEM activesupport (>= 4.0) sprockets (>= 3.0.0) spy (1.0.1) + strong_migrations (1.4.0) + activerecord (>= 5.2) swd (1.3.0) activesupport (>= 3) attr_required (>= 0.0.5) @@ -588,6 +590,7 @@ DEPENDENCIES simplecov (= 0.17.1) simpleidn (= 0.2.1) spy + strong_migrations truemail (~> 3.0) uglifier validates_email_format_of (= 1.7.2) @@ -597,4 +600,4 @@ DEPENDENCIES wkhtmltopdf-binary (~> 0.12.5.1) BUNDLED WITH - 2.3.25 + 2.3.26 diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index ae8489c34c..052cdcbc45 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -1,21 +1,16 @@ module Repp module V1 - class StatsController < BaseController + class StatsController < BaseController # rubocop:disable Metrics/ClassLength + before_action :set_date_params + api :get, '/repp/v1/stats/market_share_distribution' desc 'Get market share and distribution of registrars' param :q, Hash, required: true, desc: 'Period parameters for data' do param :end_date, String, required: true, desc: 'Period end date' end def market_share_distribution - registrars = ::Registrar.where(test_registrar: false).joins(:domains) - .where(from_condition).where(to_condition) - grouped = registrars.group(:name).count - - result = grouped.map do |key, value| - hash = { name: key.strip, y: value } - hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == key - hash - end + domains_by_rar = domains_by_registrar(@date_to, @date_from) + result = serialize_distribution_result(domains_by_rar) render_success(data: result) end @@ -24,25 +19,23 @@ def market_share_distribution desc 'Get market share and growth rate of registrars' param :q, Hash, required: true, desc: 'Period parameters for data' do param :end_date, String, required: true, desc: 'Period end date' - param :compare_to_date, String, required: true, desc: 'Comparison date' + param :compare_to_end_date, String, required: true, desc: 'Comparison date' end def market_share_growth_rate - registrars = ::Registrar.where(test_registrar: false).joins(:domains) - - domains_by_rar = registrars.where(from_condition).where(to_condition).group(:name).count - prev_domains_by_rar = registrars.where(compare_to_condition).group(:name).count + domains_by_rar = domains_by_registrar(@date_to, @date_from) + prev_domains_by_rar = domains_by_registrar(@date_compare_to, @date_compare_from) set_zero_values!(domains_by_rar, prev_domains_by_rar) market_share_by_rar = calculate_market_share(domains_by_rar) prev_market_share_by_rar = calculate_market_share(prev_domains_by_rar) - result = { prev_data: { name: search_params[:compare_to_date], - domains: serialize(prev_domains_by_rar), - market_share: serialize(prev_market_share_by_rar) }, + result = { prev_data: { name: search_params[:compare_to_end_date], + domains: serialize_growth_rate_result(prev_domains_by_rar), + market_share: serialize_growth_rate_result(prev_market_share_by_rar) }, data: { name: search_params[:end_date], - domains: serialize(domains_by_rar), - market_share: serialize(market_share_by_rar) } } + domains: serialize_growth_rate_result(domains_by_rar), + market_share: serialize_growth_rate_result(market_share_by_rar) } } render_success(data: result) end # rubocop:enable Metrics/MethodLength @@ -50,34 +43,21 @@ def market_share_growth_rate private def search_params - params.permit(:q, q: %i[start_date end_date compare_to_date]) + params.permit(:q, q: %i[start_date end_date compare_to_end_date compare_to_start_date]) .fetch(:q, {}) || {} end - def from_condition - return unless search_params[:start_date] - - "domains.created_at >= '#{to_date(search_params[:start_date])}'" - end - - def to_condition - return unless search_params[:end_date] - - "domains.created_at <= '#{to_date(search_params[:end_date]).end_of_month}'" - end - - def compare_to_condition - return unless search_params[:compare_to_date] - - "domains.created_at <= '#{to_date(search_params[:compare_to_date]).end_of_month}'" + def set_date_params + @date_to = to_date(search_params[:end_date]).end_of_month + @date_from = to_date(search_params[:start_date] || '01.05') + @date_compare_to = to_date(search_params[:compare_to_end_date]).end_of_month + @date_compare_from = to_date(search_params[:compare_to_start_date] || '01.05') end def to_date(date_param) - Date.strptime(date_param, '%m.%y') - end + return Time.zone.today if date_param.blank? - def serialize(rars) - rars.map { |key, value| [key, value] } + Date.strptime(date_param, '%m.%y') end def set_zero_values!(cur, prev) @@ -97,6 +77,62 @@ def calculate_market_share(domains_by_rar) value < 0.1 ? value.round(3) : value.round(1) end end + + def domains_by_registrar(date_to, date_from) + log_domains_del = log_domains(event: 'destroy', date_to: date_to, date_from: date_from) + log_domains_trans = log_domains(event: 'update', date_to: date_to, date_from: date_from) + logged_domains = log_domains_trans.map { |ld| ld.object['name'] } + + log_domains_del.map { |ld| ld.object['name'] } + domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) + .where.not(name: logged_domains.uniq) + .group(:registrar_id).count.stringify_keys + summarize([group(log_domains_del), group(log_domains_trans), domains_grouped]) + end + + def summarize(arr) + arr.inject { |memo, el| memo.merge(el) { |_, old_v, new_v| old_v + new_v } } + end + + def log_domains(event:, date_to:, date_from:) + domains = ::Version::DomainVersion.where(event: event) + domains.where!("object_changes ->> 'registrar_id' IS NOT NULL") if event == 'update' + domains.where('created_at > ?', date_to) + .where("object ->> 'created_at' <= ?", date_to) + .where("object ->> 'created_at' >= ?", date_from) + .select("DISTINCT ON (object ->> 'name') object, created_at") + .order(Arel.sql("object ->> 'name', created_at desc")) + end + + def group(domains) + domains.group_by { |ld| ld.object['registrar_id'].to_s } + .transform_values(&:count) + end + + def registrar_names + @registrar_names ||= ::Registrar.where(test_registrar: false) + .map { |r| { "#{r.id}": r.name }.with_indifferent_access } + .reduce({}, :merge) + end + + def serialize_distribution_result(result) + result.map do |key, value| + next unless registrar_names.key?(key) + + name = registrar_names[key] + hash = { name: name, y: value } + hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name + hash + end.compact + end + + def serialize_growth_rate_result(result) + result.map do |key, value| + next unless registrar_names.key?(key) + + name = registrar_names[key] + [name, value] + end.compact + end end end end diff --git a/config/initializers/strong_migrations.rb b/config/initializers/strong_migrations.rb new file mode 100644 index 0000000000..11ce430939 --- /dev/null +++ b/config/initializers/strong_migrations.rb @@ -0,0 +1,26 @@ +# Mark existing migrations as safe +StrongMigrations.start_after = 20221206081743 + +# Set timeouts for migrations +# If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user +StrongMigrations.lock_timeout = 10.seconds +StrongMigrations.statement_timeout = 1.hour + +# Analyze tables after indexes are added +# Outdated statistics can sometimes hurt performance +StrongMigrations.auto_analyze = true + +# Set the version of the production database +# so the right checks are run in development +# StrongMigrations.target_version = 10 + +# Add custom checks +# StrongMigrations.add_check do |method, args| +# if method == :add_index && args[0].to_s == "users" +# stop! "No more indexes on the users table" +# end +# end + +# Make some operations safe by default +# See https://github.com/ankane/strong_migrations#safe-by-default +# StrongMigrations.safe_by_default = true diff --git a/db/migrate/20221206090120_modify_log_domains_object_changes_data_type.rb b/db/migrate/20221206090120_modify_log_domains_object_changes_data_type.rb new file mode 100644 index 0000000000..e7308ae594 --- /dev/null +++ b/db/migrate/20221206090120_modify_log_domains_object_changes_data_type.rb @@ -0,0 +1,21 @@ +class ModifyLogDomainsObjectChangesDataType < ActiveRecord::Migration[6.1] + def up + add_column :log_domains, :object_changes_jsonb, :jsonb, default: '{}' + + # Copy data from old column to the new one + Version::DomainVersion.update_all('object_changes_jsonb = object_changes::jsonb') + + # Rename columns instead of modify their type, it's way faster + safety_assured do + rename_column :log_domains, :object_changes, :object_changes_json + rename_column :log_domains, :object_changes_jsonb, :object_changes + end + end + + def down + safety_assured do + rename_column :log_domains, :object_changes, :object_changes_jsonb + rename_column :log_domains, :object_changes_json, :object_changes + end + end +end \ No newline at end of file diff --git a/db/migrate/20221206091556_add_indexes_to_log_domains.rb b/db/migrate/20221206091556_add_indexes_to_log_domains.rb new file mode 100644 index 0000000000..0dbbf944f3 --- /dev/null +++ b/db/migrate/20221206091556_add_indexes_to_log_domains.rb @@ -0,0 +1,15 @@ +class AddIndexesToLogDomains < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + def up + enable_extension 'btree_gin' + add_index :log_domains, :event, algorithm: :concurrently + add_index :log_domains, :object, using: :gin, algorithm: :concurrently + add_index :log_domains, :object_changes, using: :gin, algorithm: :concurrently + end + + def down + remove_index :log_domains, :event + remove_index :log_domains, :object + remove_index :log_domains, :object_changes + end +end diff --git a/db/migrate/20221207102831_change_log_domains_object_changes_default_value.rb b/db/migrate/20221207102831_change_log_domains_object_changes_default_value.rb new file mode 100644 index 0000000000..84a3c32cf8 --- /dev/null +++ b/db/migrate/20221207102831_change_log_domains_object_changes_default_value.rb @@ -0,0 +1,5 @@ +class ChangeLogDomainsObjectChangesDefaultValue < ActiveRecord::Migration[6.1] + def change + change_column_default :log_domains, :object_changes, nil + end +end diff --git a/db/structure.sql b/db/structure.sql index 661b4e66b7..fd26dd07a2 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -9,6 +9,27 @@ SET xmloption = content; SET client_min_messages = warning; SET row_security = off; +-- +-- Name: public; Type: SCHEMA; Schema: -; Owner: - +-- + +-- *not* creating schema, since initdb creates it + + +-- +-- Name: btree_gin; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS btree_gin WITH SCHEMA public; + + +-- +-- Name: EXTENSION btree_gin; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION btree_gin IS 'support for indexing common datatypes in GIN'; + + -- -- Name: btree_gist; Type: EXTENSION; Schema: -; Owner: - -- @@ -956,15 +977,14 @@ CREATE TABLE public.domains ( pending_json jsonb, force_delete_date date, statuses character varying[], - statuses_before_force_delete character varying[] DEFAULT '{}'::character varying[], + status_notes public.hstore, upid integer, up_date timestamp without time zone, uuid uuid DEFAULT public.gen_random_uuid() NOT NULL, locked_by_registrant_at timestamp without time zone, force_delete_start timestamp without time zone, force_delete_data public.hstore, - json_statuses_history jsonb, - status_notes public.hstore + json_statuses_history jsonb ); @@ -1651,11 +1671,12 @@ CREATE TABLE public.log_domains ( event character varying NOT NULL, whodunnit character varying, object jsonb, - object_changes json, + object_changes_json json, created_at timestamp without time zone, session character varying, children jsonb, - uuid character varying + uuid character varying, + object_changes jsonb ); @@ -2287,6 +2308,74 @@ CREATE SEQUENCE public.payment_orders_id_seq ALTER SEQUENCE public.payment_orders_id_seq OWNED BY public.payment_orders.id; +-- +-- Name: pghero_query_stats; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.pghero_query_stats ( + id bigint NOT NULL, + database text, + "user" text, + query text, + query_hash bigint, + total_time double precision, + calls bigint, + captured_at timestamp without time zone +); + + +-- +-- Name: pghero_query_stats_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.pghero_query_stats_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pghero_query_stats_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.pghero_query_stats_id_seq OWNED BY public.pghero_query_stats.id; + + +-- +-- Name: pghero_space_stats; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.pghero_space_stats ( + id bigint NOT NULL, + database text, + schema text, + relation text, + size bigint, + captured_at timestamp without time zone +); + + +-- +-- Name: pghero_space_stats_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.pghero_space_stats_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pghero_space_stats_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.pghero_space_stats_id_seq OWNED BY public.pghero_space_stats.id; + + -- -- Name: prices; Type: TABLE; Schema: public; Owner: - -- @@ -2325,6 +2414,48 @@ CREATE SEQUENCE public.prices_id_seq ALTER SEQUENCE public.prices_id_seq OWNED BY public.prices.id; +-- +-- Name: que_jobs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.que_jobs ( + priority smallint DEFAULT 100 NOT NULL, + run_at timestamp with time zone DEFAULT now() NOT NULL, + job_id bigint NOT NULL, + job_class text NOT NULL, + args json DEFAULT '[]'::json NOT NULL, + error_count integer DEFAULT 0 NOT NULL, + last_error text, + queue text DEFAULT ''::text NOT NULL +); + + +-- +-- Name: TABLE que_jobs; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.que_jobs IS '3'; + + +-- +-- Name: que_jobs_job_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.que_jobs_job_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: que_jobs_job_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.que_jobs_job_id_seq OWNED BY public.que_jobs.job_id; + + -- -- Name: registrant_verifications; Type: TABLE; Schema: public; Owner: - -- @@ -2605,8 +2736,7 @@ CREATE TABLE public.validation_events ( validation_eventable_type character varying, validation_eventable_id bigint, created_at timestamp(6) without time zone NOT NULL, - updated_at timestamp(6) without time zone NOT NULL, - event_type public.validation_type + updated_at timestamp(6) without time zone NOT NULL ); @@ -3148,6 +3278,20 @@ ALTER TABLE ONLY public.notifications ALTER COLUMN id SET DEFAULT nextval('publi ALTER TABLE ONLY public.payment_orders ALTER COLUMN id SET DEFAULT nextval('public.payment_orders_id_seq'::regclass); +-- +-- Name: pghero_query_stats id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pghero_query_stats ALTER COLUMN id SET DEFAULT nextval('public.pghero_query_stats_id_seq'::regclass); + + +-- +-- Name: pghero_space_stats id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pghero_space_stats ALTER COLUMN id SET DEFAULT nextval('public.pghero_space_stats_id_seq'::regclass); + + -- -- Name: prices id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3155,6 +3299,13 @@ ALTER TABLE ONLY public.payment_orders ALTER COLUMN id SET DEFAULT nextval('publ ALTER TABLE ONLY public.prices ALTER COLUMN id SET DEFAULT nextval('public.prices_id_seq'::regclass); +-- +-- Name: que_jobs job_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.que_jobs ALTER COLUMN job_id SET DEFAULT nextval('public.que_jobs_job_id_seq'::regclass); + + -- -- Name: registrant_verifications id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3664,6 +3815,22 @@ ALTER TABLE ONLY public.payment_orders ADD CONSTRAINT payment_orders_pkey PRIMARY KEY (id); +-- +-- Name: pghero_query_stats pghero_query_stats_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pghero_query_stats + ADD CONSTRAINT pghero_query_stats_pkey PRIMARY KEY (id); + + +-- +-- Name: pghero_space_stats pghero_space_stats_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pghero_space_stats + ADD CONSTRAINT pghero_space_stats_pkey PRIMARY KEY (id); + + -- -- Name: prices prices_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -3672,6 +3839,14 @@ ALTER TABLE ONLY public.prices ADD CONSTRAINT prices_pkey PRIMARY KEY (id); +-- +-- Name: que_jobs que_jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.que_jobs + ADD CONSTRAINT que_jobs_pkey PRIMARY KEY (queue, priority, run_at, job_id); + + -- -- Name: registrant_verifications registrant_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -4251,6 +4426,13 @@ CREATE INDEX index_log_domain_contacts_on_item_type_and_item_id ON public.log_do CREATE INDEX index_log_domain_contacts_on_whodunnit ON public.log_domain_contacts USING btree (whodunnit); +-- +-- Name: index_log_domains_on_event; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_log_domains_on_event ON public.log_domains USING btree (event); + + -- -- Name: index_log_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- @@ -4258,6 +4440,20 @@ CREATE INDEX index_log_domain_contacts_on_whodunnit ON public.log_domain_contact CREATE INDEX index_log_domains_on_item_type_and_item_id ON public.log_domains USING btree (item_type, item_id); +-- +-- Name: index_log_domains_on_object; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_log_domains_on_object ON public.log_domains USING gin (object); + + +-- +-- Name: index_log_domains_on_object_changes; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_log_domains_on_object_changes ON public.log_domains USING gin (object_changes); + + -- -- Name: index_log_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- @@ -4426,6 +4622,20 @@ CREATE INDEX index_notifications_on_registrar_id ON public.notifications USING b CREATE INDEX index_payment_orders_on_invoice_id ON public.payment_orders USING btree (invoice_id); +-- +-- Name: index_pghero_query_stats_on_database_and_captured_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_pghero_query_stats_on_database_and_captured_at ON public.pghero_query_stats USING btree (database, captured_at); + + +-- +-- Name: index_pghero_space_stats_on_database_and_captured_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_pghero_space_stats_on_database_and_captured_at ON public.pghero_space_stats USING btree (database, captured_at); + + -- -- Name: index_prices_on_zone_id; Type: INDEX; Schema: public; Owner: - -- @@ -4482,13 +4692,6 @@ CREATE INDEX index_users_on_registrar_id ON public.users USING btree (registrar_ CREATE INDEX index_validation_events_on_event_data ON public.validation_events USING gin (event_data); --- --- Name: index_validation_events_on_event_type; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX index_validation_events_on_event_type ON public.validation_events USING btree (event_type); - - -- -- Name: index_validation_events_on_validation_eventable; Type: INDEX; Schema: public; Owner: - -- @@ -5234,9 +5437,11 @@ INSERT INTO "schema_migrations" (version) VALUES ('20210708131814'), ('20210729131100'), ('20210729134625'), -('20210827185249'), -('20211029073644'), +('20211028122103'), +('20211028125245'), +('20211029082225'), ('20211124071418'), +('20211124084308'), ('20211125181033'), ('20211125184334'), ('20211126085139'), @@ -5245,7 +5450,6 @@ INSERT INTO "schema_migrations" (version) VALUES ('20220113201642'), ('20220113220809'), ('20220124105717'), -('20220216113112'), ('20220228093211'), ('20220316140727'), ('20220406085500'), @@ -5258,6 +5462,10 @@ INSERT INTO "schema_migrations" (version) VALUES ('20220701113409'), ('20220715145808'), ('20220818075833'), -('20221011061840'); +('20221011061840'), +('20221206075912'), +('20221206090120'), +('20221206091556'), +('20221207102831'); diff --git a/test/integration/repp/v1/stats/market_share_test.rb b/test/integration/repp/v1/stats/market_share_test.rb index 614b5868dc..03721cebdf 100644 --- a/test/integration/repp/v1/stats/market_share_test.rb +++ b/test/integration/repp/v1/stats/market_share_test.rb @@ -21,15 +21,15 @@ def test_shows_market_share_distribution_data assert json[:data].is_a? Array assert json[:data][0].is_a? Hash - assert_equal json[:data][0][:name], 'Best Names' - assert json[:data][0][:selected] + assert_equal json[:data][0][:name], 'Good Names' + assert_nil json[:data][0][:selected] end def test_shows_market_share_growth_rate_data prev_date = Time.zone.today.last_month.strftime('%m.%y') get '/repp/v1/stats/market_share_growth_rate', headers: @auth_headers, params: { q: { end_date: @today, - compare_to_date: prev_date } } + compare_to_end_date: prev_date } } json = JSON.parse(response.body, symbolize_names: true) assert_response :ok