Merge pull request #11288 from 18F/stages/rc-2024-09-26

Deploy RC 417 to Production

.gitlab-ci.yml

@@ -63,7 +63,7 @@ stages:
   - after_test
   - review
   - scan
-  - deploy_production
+  - deploy_eks
 
 workflow:
   rules:
@@ -604,21 +604,6 @@ stop-review-app:
     - if: $CI_PIPELINE_SOURCE != "merge_request_event"
       when: never
 
-deploy_production:
-  stage: deploy_production
-  allow_failure: true
-  needs:
-    - job: build-review-image
-  resource_group: $CI_ENVIRONMENT_SLUG.reviewapps.identitysandbox.gov
-  extends: .deploy
-  environment:
-    name: production
-    deployment_tier: production
-    url: https://$CI_ENVIRONMENT_SLUG.reviewapps.identitysandbox.gov
-  rules:
-    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
-
-
 include:
   - template: Jobs/SAST.gitlab-ci.yml
   - template: Jobs/Dependency-Scanning.gitlab-ci.yml
@@ -867,3 +852,15 @@ audit_packages_scheduled:
       fi
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
+
+# EKS deployment
+deploy_eks:
+  trigger:
+    project: lg-public/identity-eks-control
+    branch: main
+  stage: deploy_eks
+  variables:
+    APP: idp
+    IMAGE_TAG: $CI_COMMIT_SHA
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"

app/helpers/ipp_helper.rb

@@ -8,8 +8,14 @@ def scrub_message(message)
   def scrub_body(body)
     return nil if body.nil?
 
-    body = body.with_indifferent_access
-    body[:responseMessage] = scrub_message(body[:responseMessage])
-    body
+    if body.is_a?(String)
+      scrub_message(body)
+    else
+      body = body.with_indifferent_access
+      if body[:responseMessage].present?
+        body[:responseMessage] = scrub_message(body[:responseMessage])
+      end
+      body
+    end
   end
 end

app/jobs/reports/duplicate_ssn_report.rb

@@ -38,10 +38,12 @@ def report_body
 
       ssn_signatures = todays_profiles.map(&:ssn_signature).uniq
 
-      profiles_connected_by_ssn = Profile.
-        includes(:user).
-        where(ssn_signature: ssn_signatures).
-        to_a
+      profiles_connected_by_ssn = ssn_signatures.each_slice(1000).flat_map do |ssn_signature_slice|
+        Profile.
+          includes(:user).
+          where(ssn_signature: ssn_signature_slice).
+          to_a
+      end
 
       profiles_connected_by_ssn.sort_by!(&:id).reverse!
 

app/jobs/reports/idv_legacy_conversion_supplement_report.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Reports
+  class IdvLegacyConversionSupplementReport < BaseReport
+    REPORT_NAME = 'idv-legacy-conversion-supplement-report'
+
+    def perform(_date)
+      csv = build_csv
+      save_report(REPORT_NAME, csv, extension: 'csv')
+    end
+
+    # @return [String] CSV report
+    def build_csv
+      sql = <<~SQL
+        SELECT
+	          iaa_orders.start_date
+	        , iaa_orders.end_date
+	        , iaa_orders.order_number
+	        , iaa_gtcs.gtc_number AS gtc_number
+	        , upgrade.issuer AS issuer
+	        , sp.friendly_name AS friendly_name
+	        , DATE_TRUNC('month', upgrade.upgraded_at) AS year_month
+	        , COUNT(DISTINCT upgrade.user_id) AS user_count
+        FROM iaa_orders
+        INNER JOIN integration_usages iu ON iu.iaa_order_id = iaa_orders.id
+        INNER JOIN integrations ON integrations.id = iu.integration_id
+        INNER JOIN iaa_gtcs ON iaa_gtcs.id = iaa_orders.iaa_gtc_id
+        INNER JOIN service_providers sp ON sp.issuer = integrations.issuer
+        INNER JOIN (
+          SELECT DISTINCT ON (user_id) *
+          FROM sp_upgraded_biometric_profiles
+        ) upgrade ON upgrade.issuer = integrations.issuer
+        WHERE upgrade.upgraded_at BETWEEN iaa_orders.start_date AND iaa_orders.end_date
+        GROUP BY iaa_orders.id, upgrade.issuer, year_month, iaa_gtcs.gtc_number, sp.friendly_name
+        ORDER BY iaa_orders.id, year_month
+      SQL
+
+      results = transaction_with_timeout do
+        ActiveRecord::Base.connection.select_all(sql)
+      end
+
+      CSV.generate do |csv|
+        csv << [
+          'iaa_order_number',
+          'iaa_start_date',
+          'iaa_end_date',
+          'issuer',
+          'friendly_name',
+          'year_month',
+          'year_month_readable',
+          'user_count',
+        ]
+
+        results.each do |iaa|
+          csv << [
+            IaaReportingHelper.key(
+              gtc_number: iaa['gtc_number'],
+              order_number: iaa['order_number'],
+            ),
+            iaa['start_date'],
+            iaa['end_date'],
+            iaa['issuer'],
+            iaa['friendly_name'],
+            iaa['year_month'].strftime('%Y%m'),
+            iaa['year_month'].strftime('%B %Y'),
+            iaa['user_count'],
+          ]
+        end
+      end
+    end
+  end
+end

app/models/profile.rb

@@ -92,7 +92,9 @@ def activate(reason_deactivated: nil)
     confirm_that_profile_can_be_activated!
 
     now = Time.zone.now
-    is_reproof = Profile.find_by(user_id: user_id, active: true)
+    profile_to_deactivate = Profile.find_by(user_id: user_id, active: true)
+    is_reproof = profile_to_deactivate.present?
+    is_biometric_upgrade = is_reproof && biometric? && !profile_to_deactivate.biometric?
 
     attrs = {
       active: true,
@@ -105,6 +107,7 @@ def activate(reason_deactivated: nil)
       Profile.where(user_id: user_id).update_all(active: false)
       update!(attrs)
     end
+    track_biometric_reproof if is_biometric_upgrade
     send_push_notifications if is_reproof
   end
   # rubocop:enable Rails/SkipsModelValidations
@@ -199,8 +202,8 @@ def deactivate_due_to_gpo_expiration
   def deactivate_due_to_in_person_verification_cancelled
     update!(
       active: false,
-      deactivation_reason: :verification_cancelled,
       in_person_verification_pending_at: nil,
+      deactivation_reason: deactivation_reason.presence || :verification_cancelled,
     )
   end
 
@@ -306,6 +309,10 @@ def profile_age_in_seconds
     (Time.zone.now - created_at).round
   end
 
+  def biometric?
+    ::User::BIOMETRIC_COMPARISON_IDV_LEVELS.include?(idv_level)
+  end
+
   private
 
   def confirm_that_profile_can_be_activated!
@@ -333,4 +340,13 @@ def send_push_notifications
     event = PushNotification::ReproofCompletedEvent.new(user: user)
     PushNotification::HttpPush.deliver(event)
   end
+
+  def track_biometric_reproof
+    SpUpgradedBiometricProfile.create(
+      user: user,
+      upgraded_at: Time.zone.now,
+      idv_level: idv_level,
+      issuer: initiating_service_provider_issuer,
+    )
+  end
 end

app/models/sp_upgraded_biometric_profile.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class SpUpgradedBiometricProfile < ApplicationRecord
+  belongs_to :user
+end

app/services/iaa_reporting_helper.rb

@@ -13,7 +13,7 @@ module IaaReportingHelper
   ) do
     # ex LG123567-0001
     def key
-      "#{gtc_number}-#{format('%04d', order_number)}"
+      IaaReportingHelper.key(gtc_number:, order_number:)
     end
   end
 
@@ -74,4 +74,8 @@ def partner_accounts
         end
       end.compact
   end
+
+  def key(gtc_number:, order_number:)
+    "#{gtc_number}-#{format('%04d', order_number)}"
+  end
 end

app/services/proofing/aamva/proofer.rb

@@ -21,6 +21,13 @@ class Proofer
         ],
       ).freeze
 
+      REQUIRED_VERIFICATION_ATTRIBUTES = %i[
+        state_id_number
+        dob
+        last_name
+        first_name
+      ].freeze
+
       ADDRESS_ATTRIBUTES = [
         :address1,
         :address2,
@@ -63,7 +70,7 @@ def proof(applicant)
 
       def build_result_from_response(verification_response, jurisdiction)
         Proofing::StateIdResult.new(
-          success: verification_response.success?,
+          success: successful?(verification_response),
           errors: parse_verification_errors(verification_response),
           exception: nil,
           vendor_name: 'aamva:state_id',
@@ -77,7 +84,7 @@ def build_result_from_response(verification_response, jurisdiction)
       def parse_verification_errors(verification_response)
         errors = Hash.new { |h, k| h[k] = [] }
 
-        return errors if verification_response.success?
+        return errors if successful?(verification_response)
 
         verification_response.verification_results.each do |attribute, v_result|
           attribute_key = attribute.to_sym
@@ -121,6 +128,14 @@ def send_to_new_relic(result)
         NewRelic::Agent.notice_error(result.exception)
       end
 
+      def successful?(verification_response)
+        REQUIRED_VERIFICATION_ATTRIBUTES.each do |verification_attribute|
+          return false unless verification_response.verification_results[verification_attribute]
+        end
+
+        true
+      end
+
       def jurisdiction_in_maintenance_window?(state)
         Idv::AamvaStateMaintenanceWindow.in_maintenance_window?(state)
       end

app/services/proofing/aamva/response/verification_response.rb

@@ -22,13 +22,6 @@ class VerificationResponse
           'AddressZIP5MatchIndicator' => :zipcode,
         }.freeze
 
-        REQUIRED_VERIFICATION_ATTRIBUTES = %i[
-          state_id_number
-          dob
-          last_name
-          first_name
-        ].freeze
-
         attr_reader :verification_results, :transaction_locator_id
 
         def initialize(http_response)
@@ -48,26 +41,6 @@ def initialize(http_response)
           raise VerificationError.new(error_message)
         end
 
-        def reasons
-          REQUIRED_VERIFICATION_ATTRIBUTES.map do |verification_attribute|
-            verification_result = verification_results[verification_attribute]
-            case verification_result
-            when false
-              "Failed to verify #{verification_attribute}"
-            when nil
-              "Response was missing #{verification_attribute}"
-            end
-          end.compact
-        end
-
-        def success?
-          REQUIRED_VERIFICATION_ATTRIBUTES.each do |verification_attribute|
-            return false unless verification_results[verification_attribute]
-          end
-
-          true
-        end
-
         private
 
         attr_reader :http_response, :missing_attributes

config/application.yml.default

@@ -62,10 +62,12 @@ component_previews_enabled: false
 compromised_password_randomizer_threshold: 900
 compromised_password_randomizer_value: 1000
 country_phone_number_overrides: '{}'
+database_advisory_locks_enabled: false
 database_host: ''
 database_name: ''
 database_password: ''
 database_pool_idp: 5
+database_prepared_statements_enabled: false
 database_read_replica_host: ''
 database_readonly_password: ''
 database_readonly_username: ''

config/database.yml

@@ -86,8 +86,8 @@ production:
     host: <%= IdentityConfig.store.database_socket.present? ?  IdentityConfig.store.database_socket : IdentityConfig.store.database_host %>
     password: <%= IdentityConfig.store.database_password %>
     pool: <%= primary_pool %>
-    advisory_locks: <%= !IdentityConfig.store.database_socket.present? %>
-    prepared_statements: <%= !IdentityConfig.store.database_socket.present? %>
+    advisory_locks: <%= IdentityConfig.store.database_advisory_locks_enabled %>
+    prepared_statements: <%= IdentityConfig.store.database_prepared_statements_enabled %>
     sslmode: <%= IdentityConfig.store.database_sslmode %>
     sslrootcert: '/usr/local/share/aws/rds-combined-ca-bundle.pem'
     migrations_paths: db/primary_migrate

config/initializers/job_configurations.rb

@@ -59,6 +59,12 @@
         cron: cron_24h,
         args: -> { [Time.zone.today] },
       },
+      # Idv Legacy Conversion Supplement Report to S3
+      idv_legacy_conversion_supplement_report: {
+        class: 'Reports::IdvLegacyConversionSupplementReport',
+        cron: cron_24h,
+        args: -> { [Time.zone.today] },
+      },
       agreement_summary_report: {
         class: 'Reports::AgreementSummaryReport',
         cron: cron_24h,

db/primary_migrate/20240916202940_create_sp_upgraded_biometric_profiles.rb

@@ -0,0 +1,14 @@
+class CreateSpUpgradedBiometricProfiles < ActiveRecord::Migration[7.1]
+  def change
+    create_table :sp_upgraded_biometric_profiles do |t|
+      t.datetime :upgraded_at, null: false
+      t.references :user, null: false
+      t.string :idv_level, null: false
+      t.string :issuer, null: false
+
+      t.timestamps
+
+      t.index [:issuer, :upgraded_at]
+    end
+  end
+end