From 4c97eb94225a941c693e5c2da0dcf3a280be82aa Mon Sep 17 00:00:00 2001 From: oleghasjanov Date: Wed, 17 Jul 2024 09:28:31 +0300 Subject: [PATCH 1/7] added smtp validator --- Gemfile.lock | 4 +-- app/lib/smtp_validator.rb | 60 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 app/lib/smtp_validator.rb diff --git a/Gemfile.lock b/Gemfile.lock index e4195ae935..5f72133ad0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -293,7 +293,7 @@ GEM activerecord kaminari-core (= 1.2.1) kaminari-core (1.2.1) - libxml-ruby (3.2.1) + libxml-ruby (5.0.3) logger (1.4.3) loofah (2.21.4) crass (~> 1.0.2) @@ -606,4 +606,4 @@ DEPENDENCIES wkhtmltopdf-binary (~> 0.12.6.1) BUNDLED WITH - 2.5.4 + 2.5.15 diff --git a/app/lib/smtp_validator.rb b/app/lib/smtp_validator.rb new file mode 100644 index 0000000000..032cdb2393 --- /dev/null +++ b/app/lib/smtp_validator.rb @@ -0,0 +1,60 @@ +require 'net/smtp' +require 'openssl' + +class SMTPValidator + def self.validate(email, options = {}) + domain = email.split('@').last + smtp_server = options[:smtp_server] || domain + smtp_port = options[:smtp_port] || 587 + helo_domain = options[:helo_domain] || 'localhost' + user_name = options[:user_name] + password = options[:password] + from_address = options[:from_address] || user_name + auth_methods = options[:auth_methods] || [:plain, :login, :cram_md5] + + result = { valid: false, code: nil, message: nil, auth_method: nil } + + begin + smtp = Net::SMTP.new(smtp_server, smtp_port) + smtp.enable_starttls_auto + smtp.open_timeout = 5 + smtp.read_timeout = 5 + + auth_methods.each do |method| + begin + smtp.start(helo_domain, user_name, password, method) do |smtp| + from_response = smtp.mailfrom(from_address) + result[:code], result[:message] = from_response.string.split(' ', 2) + + to_response = smtp.rcptto(email) + result[:code], result[:message] = to_response.string.split(' ', 2) + result[:valid] = result[:code].to_i == 250 + end + result[:auth_method] = method + break + rescue Net::SMTPAuthenticationError => e + puts "Authentication failed with method #{method}: #{e.message}" + result[:code], result[:message] = e.message.split(' ', 2) + next + rescue Net::SMTPFatalError => e + result[:code], result[:message] = e.message.split(' ', 2) + break + end + end + rescue => e + puts "Connection Error: #{e.message}" + result[:message] = e.message + ensure + smtp.finish if smtp && smtp.started? + end + + if result[:auth_method] + puts "Email #{email} validation completed (authenticated with #{result[:auth_method]})" + else + puts "Failed to authenticate with any method" + end + + puts "Valid: #{result[:valid]}, Code: #{result[:code]}, Message: #{result[:message]}" + result + end +end From 501cbf47d04c03cbcb81b569a9a558550dc06564 Mon Sep 17 00:00:00 2001 From: oleghasjanov Date: Wed, 7 Aug 2024 12:14:14 +0300 Subject: [PATCH 2/7] updated greylist validator --- app/lib/greylist_checker.rb | 110 ++++++++++++++++++++++++++++++++++++ app/lib/smtp_validator.rb | 60 -------------------- 2 files changed, 110 insertions(+), 60 deletions(-) create mode 100644 app/lib/greylist_checker.rb delete mode 100644 app/lib/smtp_validator.rb diff --git a/app/lib/greylist_checker.rb b/app/lib/greylist_checker.rb new file mode 100644 index 0000000000..e5585cd6d1 --- /dev/null +++ b/app/lib/greylist_checker.rb @@ -0,0 +1,110 @@ +require 'net/smtp' +require 'resolv' + +class GreylistChecker + GREYLIST_CODES = ['450', '451', '452', '421', '471', '472'] + + DEFAULT_OPTIONS = { + max_attempts: 1, + retry_delay: 60, + sender_email: 'martin@internet.ee', + helo_domain: 'hole.ee', + debug: true + } + + def initialize(email, options = {}) + @email = email + @domain = email.split('@').last + @options = DEFAULT_OPTIONS.merge(options) + @debug = @options[:debug] + end + + def check + mx_servers = get_mx_servers + debug_print("MX servers for #{@domain}: #{mx_servers.join(', ')}") + return { status: :error, message: "Failed to find MX servers" } if mx_servers.empty? + + mx_servers.each do |mx_server| + result = check_server(mx_server) + return result unless result[:status] == :greylisted + end + + { status: :greylisted, message: "All attempts resulted in greylisting" } + end + + private + + def get_mx_servers + Resolv::DNS.open do |dns| + mx_records = dns.getresources(@domain, Resolv::DNS::Resource::IN::MX) + mx_records.sort_by(&:preference).map(&:exchange).map(&:to_s) + end + rescue => e + debug_print("Error getting MX servers: #{e.message}") + [] + end + + def check_server(mx_server) + attempts = 0 + while attempts < @options[:max_attempts] + result = smtp_check(mx_server) + debug_print("Attempt #{attempts + 1} result: #{result}") + + return result unless result[:status] == :greylisted + + attempts += 1 + if attempts < @options[:max_attempts] + debug_print("Waiting before next attempt: #{@options[:retry_delay]} seconds") + sleep @options[:retry_delay] + end + end + result + end + + def smtp_check(mx_server) + debug_print("Starting SMTP check for server: #{mx_server}") + + Net::SMTP.start(mx_server, 25) do |smtp| + smtp.debug_output = method(:debug_print) if @debug + + debug_print("ehlo #{@options[:helo_domain]}") + smtp.ehlo(@options[:helo_domain]) + + debug_print("mail from:<#{@options[:sender_email]}>") + smtp.mailfrom(@options[:sender_email]) + + debug_print("rcpt to:<#{@email}>") + begin + response = smtp.rcptto(@email) + debug_print(response.message) + if response.success? + return { status: :success, message: "Email accepted" } + else + code = response.status.to_s[0..2] + if GREYLIST_CODES.include?(code) + return { status: :greylisted, message: "Greylisting detected: #{response.message}" } + else + return { status: :invalid, message: "Address rejected: #{response.message}" } + end + end + rescue Net::SMTPFatalError => e + debug_print("Received SMTP fatal error: #{e.message}") + return { status: :invalid, message: "Address rejected: #{e.message}" } + rescue Net::SMTPServerBusy => e + debug_print("Received SMTP Server Busy error: #{e.message}") + if GREYLIST_CODES.any? { |code| e.message.start_with?(code) } + return { status: :greylisted, message: "Greylisting detected: #{e.message}" } + else + return { status: :error, message: "Temporary server error: #{e.message}" } + end + end + end + rescue => e + debug_print("Error during SMTP check: #{e.class} - #{e.message}") + { status: :error, message: "Error connecting to SMTP server: #{e.message}" } + end + + def debug_print(message) + puts message if @debug + end +end diff --git a/app/lib/smtp_validator.rb b/app/lib/smtp_validator.rb deleted file mode 100644 index 032cdb2393..0000000000 --- a/app/lib/smtp_validator.rb +++ /dev/null @@ -1,60 +0,0 @@ -require 'net/smtp' -require 'openssl' - -class SMTPValidator - def self.validate(email, options = {}) - domain = email.split('@').last - smtp_server = options[:smtp_server] || domain - smtp_port = options[:smtp_port] || 587 - helo_domain = options[:helo_domain] || 'localhost' - user_name = options[:user_name] - password = options[:password] - from_address = options[:from_address] || user_name - auth_methods = options[:auth_methods] || [:plain, :login, :cram_md5] - - result = { valid: false, code: nil, message: nil, auth_method: nil } - - begin - smtp = Net::SMTP.new(smtp_server, smtp_port) - smtp.enable_starttls_auto - smtp.open_timeout = 5 - smtp.read_timeout = 5 - - auth_methods.each do |method| - begin - smtp.start(helo_domain, user_name, password, method) do |smtp| - from_response = smtp.mailfrom(from_address) - result[:code], result[:message] = from_response.string.split(' ', 2) - - to_response = smtp.rcptto(email) - result[:code], result[:message] = to_response.string.split(' ', 2) - result[:valid] = result[:code].to_i == 250 - end - result[:auth_method] = method - break - rescue Net::SMTPAuthenticationError => e - puts "Authentication failed with method #{method}: #{e.message}" - result[:code], result[:message] = e.message.split(' ', 2) - next - rescue Net::SMTPFatalError => e - result[:code], result[:message] = e.message.split(' ', 2) - break - end - end - rescue => e - puts "Connection Error: #{e.message}" - result[:message] = e.message - ensure - smtp.finish if smtp && smtp.started? - end - - if result[:auth_method] - puts "Email #{email} validation completed (authenticated with #{result[:auth_method]})" - else - puts "Failed to authenticate with any method" - end - - puts "Valid: #{result[:valid]}, Code: #{result[:code]}, Message: #{result[:message]}" - result - end -end From d365c6d9656f1fcd0bc2051b87025460b76c0016 Mon Sep 17 00:00:00 2001 From: oleghasjanov Date: Thu, 8 Aug 2024 11:24:00 +0300 Subject: [PATCH 3/7] added job --- app/jobs/smtp_email_check_job.rb | 13 +++ app/lib/greylist_checker.rb | 154 +++++++++++++++---------------- 2 files changed, 90 insertions(+), 77 deletions(-) create mode 100644 app/jobs/smtp_email_check_job.rb diff --git a/app/jobs/smtp_email_check_job.rb b/app/jobs/smtp_email_check_job.rb new file mode 100644 index 0000000000..5440b78d14 --- /dev/null +++ b/app/jobs/smtp_email_check_job.rb @@ -0,0 +1,13 @@ +class SmtpEmailCheckJob < ApplicationJob + def perform + contact_emails = Contact.all.pluck(:email) + + contact_emails.each do |email| + result = GreylistChecker.new(email).check + + puts '---------' + puts result + puts '---------' + end + end +end \ No newline at end of file diff --git a/app/lib/greylist_checker.rb b/app/lib/greylist_checker.rb index e5585cd6d1..58f7be7741 100644 --- a/app/lib/greylist_checker.rb +++ b/app/lib/greylist_checker.rb @@ -13,98 +13,98 @@ class GreylistChecker } def initialize(email, options = {}) - @email = email - @domain = email.split('@').last - @options = DEFAULT_OPTIONS.merge(options) - @debug = @options[:debug] - end + @email = email + @domain = email.split('@').last + @options = DEFAULT_OPTIONS.merge(options) + @debug = @options[:debug] +end - def check - mx_servers = get_mx_servers - debug_print("MX servers for #{@domain}: #{mx_servers.join(', ')}") - return { status: :error, message: "Failed to find MX servers" } if mx_servers.empty? +def check + mx_servers = get_mx_servers + debug_print("MX servers for #{@domain}: #{mx_servers.join(', ')}") + return { valid: false, status: :no_mx_record, message: "No MX records found for domain" } if mx_servers.empty? - mx_servers.each do |mx_server| - result = check_server(mx_server) - return result unless result[:status] == :greylisted - end - - { status: :greylisted, message: "All attempts resulted in greylisting" } + mx_servers.each do |mx_server| + result = check_server(mx_server) + return result unless result[:status] == :greylisted end - private + { valid: false, status: :greylisted, message: "Email greylisted by all MX servers" } +end + +private - def get_mx_servers - Resolv::DNS.open do |dns| - mx_records = dns.getresources(@domain, Resolv::DNS::Resource::IN::MX) - mx_records.sort_by(&:preference).map(&:exchange).map(&:to_s) - end - rescue => e - debug_print("Error getting MX servers: #{e.message}") - [] +def get_mx_servers + Resolv::DNS.open do |dns| + mx_records = dns.getresources(@domain, Resolv::DNS::Resource::IN::MX) + mx_records.sort_by(&:preference).map(&:exchange).map(&:to_s) end +rescue => e + debug_print("Error getting MX servers: #{e.message}") + [] +end - def check_server(mx_server) - attempts = 0 - while attempts < @options[:max_attempts] - result = smtp_check(mx_server) - debug_print("Attempt #{attempts + 1} result: #{result}") - - return result unless result[:status] == :greylisted - - attempts += 1 - if attempts < @options[:max_attempts] - debug_print("Waiting before next attempt: #{@options[:retry_delay]} seconds") - sleep @options[:retry_delay] - end +def check_server(mx_server) + attempts = 0 + while attempts < @options[:max_attempts] + result = smtp_check(mx_server) + debug_print("Attempt #{attempts + 1} result: #{result}") + + return result unless result[:status] == :greylisted + + attempts += 1 + if attempts < @options[:max_attempts] + debug_print("Waiting before next attempt: #{@options[:retry_delay]} seconds") + sleep @options[:retry_delay] end - result end + result +end - def smtp_check(mx_server) - debug_print("Starting SMTP check for server: #{mx_server}") +def smtp_check(mx_server) + debug_print("Starting SMTP check for server: #{mx_server}") + + Net::SMTP.start(mx_server, 25, @options[:helo_domain]) do |smtp| + smtp.debug_output = method(:debug_print) if @debug - Net::SMTP.start(mx_server, 25) do |smtp| - smtp.debug_output = method(:debug_print) if @debug - - debug_print("ehlo #{@options[:helo_domain]}") - smtp.ehlo(@options[:helo_domain]) - - debug_print("mail from:<#{@options[:sender_email]}>") - smtp.mailfrom(@options[:sender_email]) - - debug_print("rcpt to:<#{@email}>") - begin - response = smtp.rcptto(@email) - debug_print(response.message) - if response.success? - return { status: :success, message: "Email accepted" } - else - code = response.status.to_s[0..2] - if GREYLIST_CODES.include?(code) - return { status: :greylisted, message: "Greylisting detected: #{response.message}" } - else - return { status: :invalid, message: "Address rejected: #{response.message}" } - end - end - rescue Net::SMTPFatalError => e - debug_print("Received SMTP fatal error: #{e.message}") - return { status: :invalid, message: "Address rejected: #{e.message}" } - rescue Net::SMTPServerBusy => e - debug_print("Received SMTP Server Busy error: #{e.message}") - if GREYLIST_CODES.any? { |code| e.message.start_with?(code) } - return { status: :greylisted, message: "Greylisting detected: #{e.message}" } + debug_print("EHLO #{@options[:helo_domain]}") + smtp.ehlo(@options[:helo_domain]) + + debug_print("MAIL FROM:<#{@options[:sender_email]}>") + smtp.mailfrom(@options[:sender_email]) + + debug_print("RCPT TO:<#{@email}>") + begin + response = smtp.rcptto(@email) + debug_print(response.message) + if response.success? + return { valid: true, status: :success, message: "Email address is valid" } + else + code = response.status.to_s[0..2] + if GREYLIST_CODES.include?(code) + return { valid: false, status: :greylisted, message: "Email greylisted: #{response.message}" } else - return { status: :error, message: "Temporary server error: #{e.message}" } + return { valid: false, status: :invalid, message: "Email address is invalid: #{response.message}" } end end + rescue Net::SMTPFatalError => e + debug_print("Received SMTP fatal error: #{e.message}") + return { valid: false, status: :invalid, message: "Email address is invalid: #{e.message}" } + rescue Net::SMTPServerBusy => e + debug_print("Received SMTP Server Busy error: #{e.message}") + if GREYLIST_CODES.any? { |code| e.message.start_with?(code) } + return { valid: false, status: :greylisted, message: "Email greylisted: #{e.message}" } + else + return { valid: false, status: :error, message: "Temporary server error: #{e.message}" } + end end - rescue => e - debug_print("Error during SMTP check: #{e.class} - #{e.message}") - { status: :error, message: "Error connecting to SMTP server: #{e.message}" } end +rescue => e + debug_print("Error during SMTP check: #{e.class} - #{e.message}") + { valid: false, status: :error, message: "Error connecting to SMTP server: #{e.message}" } +end - def debug_print(message) - puts message if @debug - end +def debug_print(message) + puts message if @debug +end end From c5f07d7afa65d2242614ebc704da46dac78599b9 Mon Sep 17 00:00:00 2001 From: oleghasjanov Date: Thu, 8 Aug 2024 15:06:34 +0300 Subject: [PATCH 4/7] added net-fpt --- Gemfile | 6 +++--- Gemfile.lock | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index e51b66ac2b..10ba482478 100644 --- a/Gemfile +++ b/Gemfile @@ -17,7 +17,7 @@ gem 'figaro', '~> 1.2' # model related gem 'paper_trail', '~> 14.0' -gem 'pg', '1.5.7' +gem 'pg', '1.5.6' # 1.8 is for Rails < 5.0 gem 'ransack', '~> 4.0.0' gem 'truemail', '~> 3.0' # validates email by regexp, mail server existence and address existence @@ -43,7 +43,7 @@ gem 'data_migrate', '~> 9.0' gem 'dnsruby', '~> 1.61' gem 'isikukood' # for EE-id validation gem 'money-rails' -gem 'simpleidn', '0.2.3' # For punycode +gem 'simpleidn', '0.2.2' # For punycode gem 'whenever', '1.0.0', require: false # country listing @@ -78,7 +78,7 @@ gem 'rexml' gem 'wkhtmltopdf-binary', '~> 0.12.6.1' gem 'directo', github: 'internetee/directo', branch: 'master' - +gem 'net-ftp' gem 'strong_migrations' group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index 5f72133ad0..8455b17e1d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -211,6 +211,7 @@ GEM activerecord (>= 5.a) database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) + date (3.3.4) devise (4.8.0) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -322,6 +323,9 @@ GEM money (~> 6.13.2) railties (>= 3.0) msgpack (1.7.2) + net-ftp (0.3.7) + net-protocol + time net-protocol (0.1.3) timeout net-smtp (0.3.3) @@ -361,7 +365,7 @@ GEM activerecord (>= 6.0) request_store (~> 1.4) pdfkit (0.8.7.2) - pg (1.5.7) + pg (1.5.6) pg_query (2.1.2) google-protobuf (>= 3.17.1) pghero (3.1.0) @@ -469,7 +473,8 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) - simpleidn (0.2.3) + simpleidn (0.2.2) + unf (~> 0.1.4) sixarm_ruby_unaccent (1.2.0) socksify (1.7.1) sprockets (4.0.3) @@ -490,6 +495,8 @@ GEM temple (0.8.2) thor (1.2.2) tilt (2.0.11) + time (0.3.0) + date timeout (0.3.0) truemail (3.0.3) simpleidn (~> 0.2.1) @@ -572,6 +579,7 @@ DEPENDENCIES minitest (~> 5.17) minitest-stub_any_instance money-rails + net-ftp newrelic-infinite_tracing newrelic_rpm nokogiri (~> 1.16.0) @@ -579,7 +587,7 @@ DEPENDENCIES omniauth-tara! paper_trail (~> 14.0) pdfkit - pg (= 1.5.7) + pg (= 1.5.6) pg_query (>= 0.9.0) pghero pry (= 0.14.2) @@ -595,7 +603,7 @@ DEPENDENCIES selenium-webdriver sidekiq (~> 7.0) simplecov (= 0.17.1) - simpleidn (= 0.2.3) + simpleidn (= 0.2.2) spy strong_migrations truemail (~> 3.0) @@ -606,4 +614,4 @@ DEPENDENCIES wkhtmltopdf-binary (~> 0.12.6.1) BUNDLED WITH - 2.5.15 + 2.5.17 From 1528c8addb2d47f1d07ec5094a5a632fb846a930 Mon Sep 17 00:00:00 2001 From: oleghasjanov Date: Mon, 12 Aug 2024 10:37:42 +0300 Subject: [PATCH 5/7] added net-pop --- Gemfile | 1 + Gemfile.lock | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Gemfile b/Gemfile index 10ba482478..59faf2431b 100644 --- a/Gemfile +++ b/Gemfile @@ -79,6 +79,7 @@ gem 'wkhtmltopdf-binary', '~> 0.12.6.1' gem 'directo', github: 'internetee/directo', branch: 'master' gem 'net-ftp' +gem 'net-pop' gem 'strong_migrations' group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index 8455b17e1d..7912aabcc5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -326,6 +326,8 @@ GEM net-ftp (0.3.7) net-protocol time + net-pop (0.1.2) + net-protocol net-protocol (0.1.3) timeout net-smtp (0.3.3) @@ -580,6 +582,7 @@ DEPENDENCIES minitest-stub_any_instance money-rails net-ftp + net-pop newrelic-infinite_tracing newrelic_rpm nokogiri (~> 1.16.0) From 4526b17848ac80a964e80021d802b36bb5fc2192 Mon Sep 17 00:00:00 2001 From: oleghasjanov Date: Mon, 12 Aug 2024 11:29:13 +0300 Subject: [PATCH 6/7] added net-imap --- Gemfile | 1 + Gemfile.lock | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/Gemfile b/Gemfile index 59faf2431b..5e79a594df 100644 --- a/Gemfile +++ b/Gemfile @@ -80,6 +80,7 @@ gem 'wkhtmltopdf-binary', '~> 0.12.6.1' gem 'directo', github: 'internetee/directo', branch: 'master' gem 'net-ftp' gem 'net-pop' +gem 'net-imap' gem 'strong_migrations' group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index 7912aabcc5..1fcbeaf6cd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -326,6 +326,9 @@ GEM net-ftp (0.3.7) net-protocol time + net-imap (0.4.14) + date + net-protocol net-pop (0.1.2) net-protocol net-protocol (0.1.3) @@ -582,6 +585,7 @@ DEPENDENCIES minitest-stub_any_instance money-rails net-ftp + net-imap net-pop newrelic-infinite_tracing newrelic_rpm From c7b5416fcd7057faeb9f3a778fc59ab6d6f27417 Mon Sep 17 00:00:00 2001 From: oleghasjanov Date: Fri, 16 Aug 2024 14:15:22 +0300 Subject: [PATCH 7/7] removed unnecessary files and added graylist handle job and test for it --- app/jobs/retry_greylisted_emails_job.rb | 82 ++++++++++ app/jobs/smtp_email_check_job.rb | 13 -- app/lib/greylist_checker.rb | 110 -------------- app/models/validation_event.rb | 13 ++ test/jobs/retry_greylisted_emails_job_test.rb | 141 ++++++++++++++++++ 5 files changed, 236 insertions(+), 123 deletions(-) create mode 100644 app/jobs/retry_greylisted_emails_job.rb delete mode 100644 app/jobs/smtp_email_check_job.rb delete mode 100644 app/lib/greylist_checker.rb create mode 100644 test/jobs/retry_greylisted_emails_job_test.rb diff --git a/app/jobs/retry_greylisted_emails_job.rb b/app/jobs/retry_greylisted_emails_job.rb new file mode 100644 index 0000000000..ed2f715dfd --- /dev/null +++ b/app/jobs/retry_greylisted_emails_job.rb @@ -0,0 +1,82 @@ +class RetryGreylistedEmailsJob < ApplicationJob + queue_as :default + + MAX_RETRY_ATTEMPTS = 10 + INITIAL_RETRY_DELAY = 5.minutes + + def perform + unique_greylisted_contacts.each do |contact| + retry_count = 0 + success = false + + while retry_count < MAX_RETRY_ATTEMPTS && !success + success = retry_email_validation(contact, retry_count) + retry_count += 1 + sleep(calculate_delay(retry_count)) unless success + end + + if success + clear_and_save_successful_validation(contact) + else + mark_email_as_invalid(contact) + end + end + end + + private + + def unique_greylisted_contacts + ValidationEvent.greylisted_smtp_errors + .select(:validation_eventable_id, :validation_eventable_type) + .distinct + .map(&:validation_eventable) + end + + def retry_email_validation(contact, retry_count) + result = Truemail.validate(contact.email, with: :smtp).result + + contact.validation_events.create( + event_type: :email_validation, + success: result.success?, + event_data: { + check_level: 'smtp', + error: result.error, + retry_count: retry_count + } + ) + + result.success? + end + + def calculate_delay(retry_count) + INITIAL_RETRY_DELAY * (2 ** (retry_count - 1)) + end + + def clear_and_save_successful_validation(contact) + contact.validation_events.destroy_all + contact.validation_events.create( + event_type: :email_validation, + success: true, + event_data: { check_level: 'smtp' } + ) + end + + def mark_email_as_invalid(contact) + contact.validation_events.destroy_all + contact.validation_events.create( + event_type: :email_validation, + success: false, + event_data: { + check_level: 'smtp', + error: 'Max retry count exceeded' + } + ) + end + + # Prevents sleep in test environment + def sleep(seconds) + return if Rails.env.test? + + super + end +end \ No newline at end of file diff --git a/app/jobs/smtp_email_check_job.rb b/app/jobs/smtp_email_check_job.rb deleted file mode 100644 index 5440b78d14..0000000000 --- a/app/jobs/smtp_email_check_job.rb +++ /dev/null @@ -1,13 +0,0 @@ -class SmtpEmailCheckJob < ApplicationJob - def perform - contact_emails = Contact.all.pluck(:email) - - contact_emails.each do |email| - result = GreylistChecker.new(email).check - - puts '---------' - puts result - puts '---------' - end - end -end \ No newline at end of file diff --git a/app/lib/greylist_checker.rb b/app/lib/greylist_checker.rb deleted file mode 100644 index 58f7be7741..0000000000 --- a/app/lib/greylist_checker.rb +++ /dev/null @@ -1,110 +0,0 @@ -require 'net/smtp' -require 'resolv' - -class GreylistChecker - GREYLIST_CODES = ['450', '451', '452', '421', '471', '472'] - - DEFAULT_OPTIONS = { - max_attempts: 1, - retry_delay: 60, - sender_email: 'martin@internet.ee', - helo_domain: 'hole.ee', - debug: true - } - - def initialize(email, options = {}) - @email = email - @domain = email.split('@').last - @options = DEFAULT_OPTIONS.merge(options) - @debug = @options[:debug] -end - -def check - mx_servers = get_mx_servers - debug_print("MX servers for #{@domain}: #{mx_servers.join(', ')}") - return { valid: false, status: :no_mx_record, message: "No MX records found for domain" } if mx_servers.empty? - - mx_servers.each do |mx_server| - result = check_server(mx_server) - return result unless result[:status] == :greylisted - end - - { valid: false, status: :greylisted, message: "Email greylisted by all MX servers" } -end - -private - -def get_mx_servers - Resolv::DNS.open do |dns| - mx_records = dns.getresources(@domain, Resolv::DNS::Resource::IN::MX) - mx_records.sort_by(&:preference).map(&:exchange).map(&:to_s) - end -rescue => e - debug_print("Error getting MX servers: #{e.message}") - [] -end - -def check_server(mx_server) - attempts = 0 - while attempts < @options[:max_attempts] - result = smtp_check(mx_server) - debug_print("Attempt #{attempts + 1} result: #{result}") - - return result unless result[:status] == :greylisted - - attempts += 1 - if attempts < @options[:max_attempts] - debug_print("Waiting before next attempt: #{@options[:retry_delay]} seconds") - sleep @options[:retry_delay] - end - end - result -end - -def smtp_check(mx_server) - debug_print("Starting SMTP check for server: #{mx_server}") - - Net::SMTP.start(mx_server, 25, @options[:helo_domain]) do |smtp| - smtp.debug_output = method(:debug_print) if @debug - - debug_print("EHLO #{@options[:helo_domain]}") - smtp.ehlo(@options[:helo_domain]) - - debug_print("MAIL FROM:<#{@options[:sender_email]}>") - smtp.mailfrom(@options[:sender_email]) - - debug_print("RCPT TO:<#{@email}>") - begin - response = smtp.rcptto(@email) - debug_print(response.message) - if response.success? - return { valid: true, status: :success, message: "Email address is valid" } - else - code = response.status.to_s[0..2] - if GREYLIST_CODES.include?(code) - return { valid: false, status: :greylisted, message: "Email greylisted: #{response.message}" } - else - return { valid: false, status: :invalid, message: "Email address is invalid: #{response.message}" } - end - end - rescue Net::SMTPFatalError => e - debug_print("Received SMTP fatal error: #{e.message}") - return { valid: false, status: :invalid, message: "Email address is invalid: #{e.message}" } - rescue Net::SMTPServerBusy => e - debug_print("Received SMTP Server Busy error: #{e.message}") - if GREYLIST_CODES.any? { |code| e.message.start_with?(code) } - return { valid: false, status: :greylisted, message: "Email greylisted: #{e.message}" } - else - return { valid: false, status: :error, message: "Temporary server error: #{e.message}" } - end - end - end -rescue => e - debug_print("Error during SMTP check: #{e.class} - #{e.message}") - { valid: false, status: :error, message: "Error connecting to SMTP server: #{e.message}" } -end - -def debug_print(message) - puts message if @debug -end -end diff --git a/app/models/validation_event.rb b/app/models/validation_event.rb index 80327f7c2a..f5dcfe93f1 100644 --- a/app/models/validation_event.rb +++ b/app/models/validation_event.rb @@ -10,6 +10,8 @@ class ValidationEvent < ApplicationRecord VALID_CHECK_LEVELS = %w[regex mx smtp].freeze VALID_EVENTS_COUNT_THRESHOLD = 5 MX_CHECK = 3 + MAX_RETRY_COUNT = 10 + INITIAL_RETRY_DELAY = 5.minutes INVALID_EVENTS_COUNT_BY_LEVEL = { regex: 1, @@ -34,6 +36,13 @@ class ValidationEvent < ApplicationRecord scope :mx, -> { where('event_data @> ?', { 'check_level': 'mx' }.to_json) } scope :smtp, -> { where('event_data @> ?', { 'check_level': 'smtp' }.to_json) } scope :by_object, ->(object) { where(validation_eventable: object) } + scope :greylisted_smtp_errors, -> { + where(success: false) + .where("event_data->>'check_level' = ?", 'smtp') + .where("event_data->'smtp_debug'->0->'response'->'errors'->>'rcptto' LIKE ?", '%Greylisted for%') + .where('created_at > ?', 24.hours.ago) + } + def self.validated_ids_by(klass) old_records @@ -50,6 +59,10 @@ def successful? success end + def greylisted? + event_type == 'smtp' && event_data['error'].include?('Greylisted for') + end + def event_type @event_type ||= ValidationEvent::EventType.new(self[:event_type]) end diff --git a/test/jobs/retry_greylisted_emails_job_test.rb b/test/jobs/retry_greylisted_emails_job_test.rb new file mode 100644 index 0000000000..cc868f08fd --- /dev/null +++ b/test/jobs/retry_greylisted_emails_job_test.rb @@ -0,0 +1,141 @@ +require 'test_helper' + +class RetryGreylistedEmailsJobTest < ActiveJob::TestCase + def setup + @contact = contacts(:john) + @contact.validation_events.destroy_all + @greylisted_event = ValidationEvent.create( + validation_eventable: @contact, + event_type: :email_validation, + success: false, + event_data: { + email: @contact.email, + domain: @contact.email.split('@').last, + mail_servers: ["194.19.134.80", "185.138.56.208"], + errors: { smtp: "smtp error" }, + smtp_debug: [{ + host: "194.19.134.80", + email: @contact.email, + attempts: nil, + response: { + port_opened: true, + connection: true, + helo: true, + mailfrom: { status: "250", string: "250 2.1.0 Ok\n" }, + rcptto: false, + errors: { rcptto: "451 4.7.1 <#{@contact.email}>: Recipient address rejected: Greylisted for 1 minutes\n" } + }, + configuration: { + smtp_port: 25, + verifier_email: "no-reply@example.com", + verifier_domain: "example.com", + response_timeout: 1, + connection_timeout: 1 + } + }], + check_level: "smtp" + } + ) + @contact.reload + end + + test 'performs retry for greylisted emails and succeeds' do + mock_truemail_success do + assert_no_difference 'ValidationEvent.count' do + RetryGreylistedEmailsJob.perform_now + end + end + + @contact.reload + assert @contact.validation_events.last.success? + assert_nil @contact.validation_events.last.event_data.dig('smtp_debug', 0, 'response', 'errors', 'rcptto') + end + + def test_marks_email_as_invalid_after_max_retries + mock_truemail_failure(RetryGreylistedEmailsJob::MAX_RETRY_ATTEMPTS) do + assert_no_difference 'ValidationEvent.count' do + RetryGreylistedEmailsJob.perform_now + end + end + + @contact.reload + last_event = @contact.validation_events.last + refute last_event.success? + + error_message = last_event.event_data['errors']&.dig('smtp') || + last_event.event_data['error'] || + last_event.event_data.dig('smtp_debug', 0, 'response', 'errors', 'rcptto') + + assert_equal 'Max retry count exceeded', error_message + end + + test 'retries until email is not greylisted' do + mock_truemail_success_after_failures(3) do + assert_no_difference 'ValidationEvent.count' do + RetryGreylistedEmailsJob.perform_now + end + end + + @contact.reload + assert @contact.validation_events.last.success? + assert_nil @contact.validation_events.last.event_data.dig('smtp_debug', 0, 'response', 'errors', 'rcptto') + end + + private + + def mock_truemail_success + result = mock_truemail_result(true) + Truemail.stub :validate, result do + yield if block_given? + end +end + +def mock_truemail_failure(times = 1) + results = Array.new(times) { mock_truemail_result(false) } + Truemail.stub :validate, ->(*args) { results.shift || mock_truemail_result(false) } do + yield if block_given? + end +end + +def mock_truemail_success_after_failures(failure_count) + results = Array.new(failure_count) { mock_truemail_result(false) } + results << mock_truemail_result(true) + Truemail.stub :validate, ->(*args) { results.shift || mock_truemail_result(true) } do + yield if block_given? + end +end + +def mock_truemail_result(success) + OpenStruct.new( + result: OpenStruct.new( + success?: success, + email: @contact.email, + domain: @contact.email.split('@').last, + mail_servers: ["194.19.134.80", "185.138.56.208"], + errors: success ? {} : { smtp: "smtp error" }, + smtp_debug: [ + OpenStruct.new( + host: "194.19.134.80", + email: @contact.email, + attempts: nil, + response: OpenStruct.new( + port_opened: true, + connection: true, + helo: true, + mailfrom: OpenStruct.new(status: "250", string: "250 2.1.0 Ok\n"), + rcptto: success, + errors: success ? {} : { rcptto: "451 4.7.1 <#{@contact.email}>: Recipient address rejected: Greylisted for 1 minutes\n" } + ), + configuration: OpenStruct.new( + smtp_port: 25, + verifier_email: "no-reply@example.com", + verifier_domain: "example.com", + response_timeout: 1, + connection_timeout: 1 + ) + ) + ] + ) + ) +end +end \ No newline at end of file