diff --git a/.ruby-version b/.ruby-version index 15a2799817..75a22a26ac 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.3.0 +3.0.3 diff --git a/Gemfile b/Gemfile index 5c6660cb76..f97ef1d250 100644 --- a/Gemfile +++ b/Gemfile @@ -88,12 +88,12 @@ end group :test do gem 'capybara' gem 'database_cleaner' - gem 'minitest', '~> 5.17' + gem 'minitest' gem 'minitest-stub_any_instance' gem 'selenium-webdriver' - # gem 'webdrivers' gem 'simplecov', '0.17.1', require: false # CC last supported v0.17 gem 'spy' + # gem 'webdrivers' gem 'webmock' end diff --git a/Gemfile.lock b/Gemfile.lock index efb8d6f130..eeb70dcf7d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -139,7 +139,7 @@ GEM minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) - addressable (2.8.1) + addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) aes_key_wrap (1.1.0) airbrake (11.0.3) @@ -167,6 +167,7 @@ GEM aws-sigv4 (~> 1.1) aws-sigv4 (1.2.4) aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) bcrypt (3.1.16) bindata (2.4.14) bootsnap (1.17.1) @@ -176,15 +177,15 @@ GEM sassc (>= 2.0.0) builder (3.2.4) cancancan (3.3.0) - capybara (3.35.3) + capybara (3.40.0) addressable + matrix mini_mime (>= 0.1.3) - nokogiri (~> 1.8) + nokogiri (~> 1.11) rack (>= 1.6.0) rack-test (>= 0.6.3) regexp_parser (>= 1.5, < 3.0) xpath (~> 3.2) - childprocess (3.0.0) chronic (0.10.2) coderay (1.1.3) coffee-rails (5.0.0) @@ -235,13 +236,9 @@ GEM thor (>= 0.14.0, < 2) globalid (1.0.1) activesupport (>= 5.0) - google-protobuf (3.25.2) google-protobuf (3.25.2-x86_64-linux) googleapis-common-protos-types (1.3.0) google-protobuf (~> 3.14) - grpc (1.60.0) - google-protobuf (~> 3.25) - googleapis-common-protos-types (~> 1.0) grpc (1.60.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) @@ -301,6 +298,7 @@ GEM mail (2.7.1) mini_mime (>= 0.1.1) marcel (1.0.2) + matrix (0.4.2) method_source (1.0.0) mime-types (3.3.1) mime-types-data (~> 3.2015) @@ -309,7 +307,6 @@ GEM nokogiri (~> 1) rake mini_mime (1.1.5) - mini_portile2 (2.8.5) minitest (5.18.1) minitest-stub_any_instance (1.0.3) monetize (1.9.4) @@ -332,10 +329,7 @@ GEM newrelic_rpm (= 8.1.0) newrelic_rpm (8.1.0) nio4r (2.5.9) - nokogiri (1.16.2) - mini_portile2 (~> 2.8.2) - racc (~> 1.4) - nokogiri (1.16.2-x86_64-linux) + nokogiri (1.16.4-x86_64-linux) racc (~> 1.4) nori (2.6.0) omniauth (2.1.0) @@ -369,11 +363,11 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.0) + public_suffix (5.0.5) puma (5.6.8) nio4r (~> 2.0) racc (1.7.3) - rack (2.2.8.1) + rack (2.2.9) rack-oauth2 (1.21.3) activesupport attr_required @@ -421,7 +415,7 @@ GEM redis-client (>= 0.9.0) redis-client (0.14.1) connection_pool - regexp_parser (2.1.1) + regexp_parser (2.9.0) request_store (1.5.1) rack (>= 1.4) responders (3.0.1) @@ -455,9 +449,11 @@ GEM wasabi (~> 3.4) select2-rails (4.0.13) selectize-rails (0.12.6) - selenium-webdriver (3.142.7) - childprocess (>= 0.5, < 4.0) - rubyzip (>= 1.2.2) + selenium-webdriver (4.20.1) + base64 (~> 0.2) + rexml (~> 3.2, >= 3.2.5) + rubyzip (>= 1.2.2, < 3.0) + websocket (~> 1.0) sidekiq (7.1.4) concurrent-ruby (< 2) connection_pool (>= 2.3.0) @@ -520,7 +516,8 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - websocket-driver (0.7.6) + websocket (1.2.10) + websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) whenever (1.0.0) @@ -531,7 +528,6 @@ GEM zeitwerk (2.6.13) PLATFORMS - ruby x86_64-linux DEPENDENCIES @@ -567,7 +563,7 @@ DEPENDENCIES lhv! mime-types-data mimemagic (= 0.4.3) - minitest (~> 5.17) + minitest minitest-stub_any_instance money-rails newrelic-infinite_tracing @@ -604,4 +600,4 @@ DEPENDENCIES wkhtmltopdf-binary (~> 0.12.6.1) BUNDLED WITH - 2.5.4 + 2.5.10 diff --git a/app/interactions/actions/a_and_aaaa_email_validation.rb b/app/interactions/actions/a_and_aaaa_email_validation.rb index 717d2c1bb0..53fdf59499 100644 --- a/app/interactions/actions/a_and_aaaa_email_validation.rb +++ b/app/interactions/actions/a_and_aaaa_email_validation.rb @@ -10,11 +10,13 @@ def call(email:, value:) def check_for_records_value(email:, value:) email_domain = Mail::Address.new(email).domain + dns_servers = ENV['dnssec_resolver_ips'].to_s.split(',').map(&:strip) resolve_a_and_aaaa_records(dns_servers: dns_servers, email_domain: email_domain, value: value) - rescue Mail::Field::IncompleteParseError => e - Rails.logger.info "Failed to parse email #{email}." + rescue Mail::Field::ParseError => e + Rails.logger.info "Mail parsing error: #{e.message}" + [] end def resolve_a_and_aaaa_records(dns_servers:, email_domain:, value:) diff --git a/app/jobs/verify_emails_job.rb b/app/jobs/verify_emails_job.rb index faefa4ceea..f915cfb8c9 100644 --- a/app/jobs/verify_emails_job.rb +++ b/app/jobs/verify_emails_job.rb @@ -1,31 +1,37 @@ class VerifyEmailsJob < ApplicationJob discard_on StandardError - def perform(email:, check_level: 'mx') - contact = Contact.find_by(email: email) - - return logger.info "Contact #{email} not found!" if contact.nil? + def perform(email:, check_level: 'mx', force: false) + contact = fetch_contact(email) + return unless contact && need_to_verify?(contact, force) - return unless need_to_verify?(contact) - - validate_check_level(check_level) - - logger.info "Trying to verify contact email #{email} with check_level #{check_level}" - contact.verify_email(check_level: check_level) + verify_contact_email(contact, check_level) rescue StandardError => e handle_error(e) end private + def fetch_contact(email) + contact = Contact.find_by(email: email) + logger.info "Contact #{email} not found!" unless contact + contact + end + + def verify_contact_email(contact, check_level) + validate_check_level(check_level) + logger.info "Trying to verify contact email #{contact.email} with check_level #{check_level}" + contact.verify_email(check_level: check_level) + end + def validate_check_level(check_level) return if valid_check_levels.include? check_level raise StandardError, "Check level #{check_level} is invalid" end - def need_to_verify?(contact) - return true if contact.validation_events.empty? + def need_to_verify?(contact, force) + return true if contact.validation_events.empty? || force last_validation = contact.validation_events.last expired_last_validation = last_validation.successful? && last_validation.created_at < validation_expiry_date diff --git a/app/models/concerns/email_verifable.rb b/app/models/concerns/email_verifable.rb index e2d55f2995..5aa8888aaa 100644 --- a/app/models/concerns/email_verifable.rb +++ b/app/models/concerns/email_verifable.rb @@ -79,9 +79,7 @@ def verify(email:, check_level: 'regex') action.call end - # rubocop:disable Metrics/LineLength def process_error(field) errors.add(field, I18n.t('activerecord.errors.models.contact.attributes.email.email_regex_check_error')) end - # rubocop:enable Metrics/LineLength end diff --git a/config/initializers/truemail.rb b/config/initializers/truemail.rb index 10c7374c87..283c67ad68 100644 --- a/config/initializers/truemail.rb +++ b/config/initializers/truemail.rb @@ -62,18 +62,25 @@ # domain only, i.e. if domain whitelisted, validation will passed to Regex, MX or SMTP validators. # Validation of email which not contains whitelisted domain always will return false. # It is equal false by default. - #config.whitelist_validation = true + # config.whitelist_validation = true # Optional parameter. Validation of email which contains blacklisted domain always will # return false. Other validations will not processed even if it was defined in validation_type_for # It is equal to empty array by default. - #config.blacklisted_domains = [] + # config.blacklisted_domains = [] # Optional parameter. This option will provide to use not RFC MX lookup flow. # It means that MX and Null MX records will be cheked on the DNS validation layer only. # By default this option is disabled. # config.not_rfc_mx_lookup_flow = true + # Optional parameter. This option will provide to use smtp fail fast behavior. When + # smtp_fail_fast = true it means that Truemail ends smtp validation session after first + # attempt on the first mx server in any fail cases (network connection/timeout error, + # smtp validation error). This feature helps to reduce total time of SMTP validation + # session up to 1 second. By default this option is disabled. + # config.smtp_fail_fast = true + # Optional parameter. This option will be parse bodies of SMTP errors. It will be helpful # if SMTP server does not return an exact answer that the email does not exist # By default this option is disabled, available for SMTP validation only. diff --git a/lib/tasks/verify_email.rake b/lib/tasks/verify_email.rake index 9aa28612e8..c35fb89427 100644 --- a/lib/tasks/verify_email.rake +++ b/lib/tasks/verify_email.rake @@ -4,17 +4,25 @@ require 'syslog/logger' require 'active_record' SPAM_PROTECT_TIMEOUT = 30.seconds +PATCH_SIZE = 10 +PATCH_INTERVAL = 10.minutes namespace :verify_email do # bundle exec rake verify_email:check_all -- --check_level=mx --spam_protect=true - # bundle exec rake verify_email:check_all -- -dshop.test -cmx -strue + # bundle exec rake verify_email:check_all -- -d shop.test -c mx -s true + # bunlde exec rake verify_email:check_all -- -e email1@example.com,email2@example.com -c mx + # bundle exec rake verify_email:check_all -- --email_regex='^test\d*@example\.com$' --check_level=mx --spam_protect=true desc 'Starts verifying email jobs with optional check level and spam protection' task check_all: :environment do options = { domain_name: nil, check_level: 'mx', spam_protect: false, + emails: [], + email_regex: nil, + force: false } + banner = 'Usage: rake verify_email:check_all -- [options]' options = RakeOptionParserBoilerplate.process_args(options: options, banner: banner, @@ -27,9 +35,11 @@ namespace :verify_email do end def enqueue_email_verification(email_contacts, options) - email_contacts.each do |email| - VerifyEmailsJob.set(wait_until: spam_protect_timeout(options)) - .perform_later(email: email, check_level: options[:check_level]) + email_contacts.each_slice(PATCH_SIZE).with_index do |slice, index| + slice.each do |email| + VerifyEmailsJob.set(wait: spam_protect_timeout(options) + index * PATCH_INTERVAL) + .perform_later(email: email, check_level: options[:check_level], force: options[:force]) + end end end @@ -38,7 +48,11 @@ def spam_protect_timeout(options) end def prepare_contacts(options) - if options[:domain_name].present? + if options[:emails].any? + options[:emails] + elsif options[:email_regex].present? + contacts_by_regex(options[:email_regex]) + elsif options[:domain_name].present? contacts_by_domain(options[:domain_name]) else unvalidated_and_failed_contacts_emails @@ -70,10 +84,17 @@ def contacts_by_domain(domain_name) domain.contacts.pluck(:email).uniq end +def contacts_by_regex(regex) + Contact.where('email ~ ?', regex).pluck(:email).uniq +end + def opts_hash { domain_name: ['-d [DOMAIN_NAME]', '--domain_name [DOMAIN_NAME]', String], check_level: ['-c [CHECK_LEVEL]', '--check_level [CHECK_LEVEL]', String], spam_protect: ['-s [SPAM_PROTECT]', '--spam_protect [SPAM_PROTECT]', FalseClass], + emails: ['-e [EMAILS]', '--emails [EMAILS]', Array], + email_regex: ['-r [EMAIL_REGEX]', '--email_regex [EMAIL_REGEX]', String], + force: ['-f', '--force', FalseClass] } end diff --git a/renovate.json b/renovate.json index 11d9ae9576..86d198b8f3 100644 --- a/renovate.json +++ b/renovate.json @@ -14,7 +14,8 @@ }, { "matchDepTypes": [".ruby-version"], - "addLabels": ["ruby-version"] + "addLabels": ["ruby-version"], + "automerge": false } ], "docker": { diff --git a/test/interactions/email_check_test.rb b/test/interactions/email_check_test.rb index adecd37d48..c7bc547ba3 100644 --- a/test/interactions/email_check_test.rb +++ b/test/interactions/email_check_test.rb @@ -97,7 +97,7 @@ def test_should_remove_old_record_if_multiple_contacts_has_the_same_email end def test_should_test_email_with_punnycode - email = "info@xn--energiathus-mfb.ee" + email = 'info@xn--energiathus-mfb.ee' result = Actions::SimpleMailValidator.run(email: email, level: :mx) assert result diff --git a/test/tasks/emails/verify_email_task_test.rb b/test/tasks/emails/verify_email_task_test.rb index a1a3f138b7..69f668f9d5 100644 --- a/test/tasks/emails/verify_email_task_test.rb +++ b/test/tasks/emails/verify_email_task_test.rb @@ -2,6 +2,15 @@ class VerifyEmailTaskTest < ActiveJob::TestCase def setup + @task_name = 'verify_email:check_all' + @default_options = { + domain_name: nil, + check_level: 'mx', + spam_protect: false, + emails: [], + email_regex: nil, + force: false + } @contact = contacts(:john) @invalid_contact = contacts(:invalid_email) @registrar = registrars(:bestnames) @@ -37,31 +46,30 @@ def test_should_skip_duplicate_emails assert_equal william_contacts_count, 2 assert_equal Contact.count, 9 run_task - assert_equal ValidationEvent.count, Contact.count - 1 + assert_equal ValidationEvent.count, Contact.count end def test_should_not_affect_successfully_verified_emails assert_equal ValidationEvent.count, 0 run_task - assert_equal ValidationEvent.count, Contact.count - 1 + assert_equal ValidationEvent.count, Contact.count assert_equal ValidationEvent.where(success: true).count, 5 - assert_equal ValidationEvent.where(success: false).count, 3 + assert_equal ValidationEvent.where(success: false).count, 4 run_task assert_equal ValidationEvent.where(success: true).count, 5 - assert_equal ValidationEvent.where(success: false).count, 6 + assert_equal ValidationEvent.where(success: false).count, 8 end def test_should_verify_contact_email_which_was_not_verified - assert_equal ValidationEvent.count, 0 - + run_task - - assert_equal ValidationEvent.count, Contact.count - 1 + + assert_equal ValidationEvent.count, Contact.count assert_equal Contact.count, 9 - + assert_difference 'Contact.count', 1 do create_valid_contact end @@ -112,15 +120,72 @@ def test_should_remove_old_validation_records assert_predicate ValidationEvent.old_records.count, :zero? end + def test_task_invocation_with_emails_option + options = @default_options.merge( + emails: [@contact.email, @invalid_contact.email] + ) + + assert_equal ValidationEvent.count, 0 + + assert_difference 'ValidationEvent.where(success: true).count', 1 do + RakeOptionParserBoilerplate.stub(:process_args, options) do + run_task + end + end + + assert_equal ValidationEvent.count, 2 + assert_equal ValidationEvent.where(success: true).count, 1 + assert_equal ValidationEvent.where(success: false).count, 1 + end + + def test_task_invocation_with_regex_option + options = @default_options.merge( + email_regex: '@inbox\.test$' + ) + + assert_equal ValidationEvent.count, 0 + + assert_difference 'ValidationEvent.where(success: true).count', 5 do + RakeOptionParserBoilerplate.stub(:process_args, options) do + run_task + end + end + end + + def test_task_invocation_with_force_option_and_failed_last_regex_validation + options = @default_options.merge( + emails: [@contact.email, @invalid_contact.email], + check_level: 'regex' + ) + assert_equal ValidationEvent.count, 0 + + assert_difference 'ValidationEvent.count', 2 do + RakeOptionParserBoilerplate.stub(:process_args, options) do + run_task + end + end + + last_failed_validation = @invalid_contact.validation_events.last + + options[:force] = true + assert_difference 'ValidationEvent.count', 0 do + RakeOptionParserBoilerplate.stub(:process_args, options) do + run_task + end + end + + assert_not_equal last_failed_validation.id, @invalid_contact.validation_events.last.id + end + def run_task perform_enqueued_jobs do - Rake::Task['verify_email:check_all'].execute + Rake::Task[@task_name].execute end end def create_valid_contact Contact.create!(name: 'Jeembo', - email: 'heey@jeembo.com', + email: 'heey@inbox.test', phone: '+555.555', ident: '1234', ident_type: 'priv',