Import nav22 external trainings

app/domain/sac_imports/csv_source.rb

@@ -18,6 +18,7 @@ class SacImports::CsvSource
     NAV17: Nav17,
     NAV18: Nav18,
     NAV21: Nav21,
+    NAV22: Nav22,
     WSO21: Wso2,
     CHIMP_1: Chimp,
     CHIMP_2: Chimp,

app/domain/sac_imports/csv_source/nav22.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+class SacImports::CsvSource
+  # ExternalTraining
+  # !!! DO NOT CHANGE THE ORDER OF THE KEYS !!!
+  # they must match the order of the columns in the CSV files
+  Nav22 = Data.define(
+    :person_id,
+    :name,
+    :provider,
+    :start_at,
+    :finish_at,
+    :event_kind,
+    :training_days,
+    :link,
+    :remarks,
+    :unused
+  )
+end

app/domain/sac_imports/events/external_training_entry.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+module SacImports
+  module Events
+    class ExternalTrainingEntry
+      ATTRS_REGULAR = [:person_id, :name, :provider, :start_at, :finish_at, :training_days, :link, :remarks]
+      ATTRS_BELONGS_TO = [:event_kind]
+
+      attr_reader :row, :associations, :warnings
+
+      delegate :errors, to: :external_training
+
+      def initialize(row, associations)
+        @row = row
+        @associations = associations
+        @warnings = []
+        build_external_training
+      end
+
+      def import!
+        external_training.save!
+      end
+
+      def valid?
+        external_training.valid?
+      end
+
+      def error_messages
+        errors.full_messages.join(", ")
+      end
+
+      def external_training
+        @external_training ||= ExternalTraining.new
+      end
+
+      def build_external_training
+        external_training.attributes = regular_attrs
+        external_training.event_kind_id = association_id(:event_kind, value(:event_kind))
+      end
+
+      def regular_attrs
+        ATTRS_REGULAR.each_with_object({}) do |attr, hash|
+          hash[attr] = value(attr)
+        end
+      end
+
+      def association_id(attr, value)
+        return nil if value.nil?
+
+        associations.fetch(attr.to_s.pluralize.to_sym).fetch(value) do
+          @warnings << "#{attr} with value #{value} couldn't be found"
+          nil
+        end
+      end
+
+      def value(attr)
+        row.public_send(attr)
+      end
+    end
+  end
+end

app/domain/sac_imports/events/participation_entry.rb

@@ -8,7 +8,6 @@
 module SacImports
   module Events
     class ParticipationEntry
-      LOCALES = [:de, :fr, :it].freeze
       ATTRS_REGULAR = [:person_id, :state, :additional_information, :canceled_at, :cancel_statement, :actual_days, :price]
       ATTRS_BOOLEAN = [:qualified, :subsidy]
       ATTRS_BELONGS_TO = [:event_number]

app/domain/sac_imports/nav22_external_trainings_importer.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+module SacImports
+  class Nav22ExternalTrainingsImporter
+    include LogCounts
+
+    REPORT_HEADERS = [
+      :person_id,
+      :start_at,
+      :status,
+      :errors
+    ]
+
+    def initialize(output: $stdout, import_spec_fixture: false)
+      @output = output
+      # spec fixture includes all sections and it's public data
+      @import_spec_fixture = import_spec_fixture
+      @source_file = source_file
+      @csv_report = SacImports::CsvReport.new("nav22-external-trainings", REPORT_HEADERS, output:)
+    end
+
+    def create
+      ExternalTraining.skip_callback(:save, :after, :issue_qualifications)
+
+      @csv_report.log("The file contains #{@source_file.lines_count} rows.")
+      progress = Progress.new(@source_file.lines_count, title: "NAV22 External Trainings")
+
+      log_counts_delta(@csv_report, ExternalTraining.unscoped) do
+        @source_file.rows do |row|
+          progress.step
+          process_row(row)
+        end
+      end
+
+      @csv_report.finalize
+    end
+
+    private
+
+    def source_file
+      if @import_spec_fixture
+        CsvSource.new(:NAV22, source_dir: spec_fixture_dir)
+      else
+        CsvSource.new(:NAV22)
+      end
+    end
+
+    def spec_fixture_dir
+      Pathname.new(HitobitoSacCas::Wagon.root.join("spec", "fixtures", "files", "sac_imports_src"))
+    end
+
+    def process_row(row)
+      entry = Events::ExternalTrainingEntry.new(row, associations)
+      entry.import! if entry.valid?
+      report_warnings(entry)
+      report_errors(entry)
+    end
+
+    def report_errors(entry)
+      return if entry.errors.blank?
+
+      @output.puts("#{entry.row.person_id} - #{entry.row.start_at}: ❌ #{entry.error_messages}")
+      @csv_report.add_row(
+        person_id: entry.row.person_id,
+        start_at: entry.row.start_at,
+        status: "error",
+        errors: entry.error_messages
+      )
+    end
+
+    def report_warnings(entry)
+      return if entry.warnings.blank?
+
+      @csv_report.add_row(
+        person_id: entry.row.person_id,
+        start_at: entry.row.start_at,
+        status: "warning",
+        errors: entry.warnings.join(", ")
+      )
+    end
+
+    def associations
+      @associations ||= {
+        event_kinds: Event::Kind::Translation.where(locale: :de).pluck(:short_name, :event_kind_id).to_h
+      }
+    end
+  end
+end

app/models/external_training.rb

@@ -28,15 +28,13 @@
 #  index_external_trainings_on_person_id      (person_id)
 #
 class ExternalTraining < ActiveRecord::Base
-  validates_by_schema
+  attr_accessor :other_people_ids
 
   belongs_to :person
   belongs_to :event_kind, class_name: "Event::Kind"
 
-  attr_accessor :other_people_ids
-
+  validates_by_schema
   validates_date :finish_at, on_or_after: :start_at, allow_blank: true
-
   validates_date :finish_at, on_or_before: lambda { Date.current }
 
   scope :list, -> { order(created_at: :desc) }

db/seeds/course_master_data.rb

@@ -141,7 +141,8 @@ def seed_event_kind_categories
     {order: 8500, cost_center_id: cost_centers.fetch("2100023"), cost_unit_id: cost_units.fetch("A8500")},
     {order: 9000, cost_center_id: cost_centers.fetch("2100027"), cost_unit_id: cost_units.fetch("A9000")},
     {order: 9100, cost_center_id: cost_centers.fetch("2100001"), cost_unit_id: cost_units.fetch("A9100")},
-    {order: 9500, cost_center_id: cost_centers.fetch("2100028"), cost_unit_id: cost_units.fetch("A9500")})
+    {order: 9500, cost_center_id: cost_centers.fetch("2100028"), cost_unit_id: cost_units.fetch("A9500")},
+    {order: 9999, cost_center_id: cost_centers.fetch("10000"), cost_unit_id: cost_units.fetch("A0000")})
 
   kind_categories = Event::KindCategory.pluck(:order, :id).to_h
   Event::KindCategory::Translation.seed_once(:event_kind_category_id, :locale,
@@ -173,7 +174,8 @@ def seed_event_kind_categories
     {event_kind_category_id: kind_categories.fetch(8500), locale: "de", label: "SAC - Tourenangebote Sommer"},
     {event_kind_category_id: kind_categories.fetch(9000), locale: "de", label: "SAC - Hüttenwartsausbildung"},
     {event_kind_category_id: kind_categories.fetch(9100), locale: "de", label: "SAC - Kurskaderausbildung"},
-    {event_kind_category_id: kind_categories.fetch(9500), locale: "de", label: "Sektionen - Angebote"})
+    {event_kind_category_id: kind_categories.fetch(9500), locale: "de", label: "Sektionen - Angebote"},
+    {event_kind_category_id: kind_categories.fetch(9999), locale: "de", label: "Externe Ausbildungen"})
 end
 
 ##### Kursstufen NAV 14

lib/tasks/sac_imports.rake

@@ -73,6 +73,7 @@ namespace :sac_imports do
     "nav17-1_event_kinds",
     "nav18-1_events",
     "nav21-1_event-participations",
+    "nav22-1_external-trainings",
     :cleanup,
     :check_data_quality
   ] do
@@ -175,6 +176,12 @@ namespace :sac_imports do
     Rake::Task["sac_imports:dump_database"].execute(dump_name: "nav21-event-participations")
   end
 
+  desc "Imports external trainings"
+  task "nav22-1_external-trainings": :setup do
+    SacImports::Nav22ExternalTrainingsImporter.new.create
+    Rake::Task["sac_imports:dump_database"].execute(dump_name: "nav22-external-trainings")
+  end
+
   desc "NAV1 Imports subscriptions from Navision"
   task "chimp-subscriptions": :setup do
     SacImports::ChimpImporter.new.create

spec/domain/sac_imports/nav22_external_trainings_importer_spec.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+require "spec_helper"
+
+describe SacImports::Nav22ExternalTrainingsImporter do
+  let(:sac_imports_src) { file_fixture("sac_imports_src").expand_path }
+  let(:output) { double(puts: nil, print: nil) }
+  let(:report) { described_class.new(output: output) }
+  let(:report_file) { Rails.root.join("log", "sac_imports", "nav22-external-trainings_2024-01-23-1142.csv") }
+  let(:report_headers) {
+    %w[person_id start_at status errors]
+  }
+  let(:csv_report) { CSV.read(report_file, col_sep: ";") }
+
+  before do
+    File.delete(report_file) if File.exist?(report_file)
+    stub_const("SacImports::CsvSource::SOURCE_DIR", sac_imports_src)
+
+    create_event_kinds
+    Person.find(600001).qualifications.create!(
+      qualification_kind: qualification_kinds(:snowboard_leader),
+      start_at: Date.new(2018, 7, 3),
+      qualified_at: Date.new(2018, 7, 3),
+      finish_at: Date.new(2024, 12, 31)
+    )
+  end
+
+  after do
+    # re-register callback
+    ExternalTraining.after_save :issue_qualifications
+  end
+
+  it "creates report for entries in source file" do
+    expected_output = []
+    expected_output << "600001 - 2023-03-22: ❌ Kursart muss ausgefüllt werden"
+
+    expect(output).to receive(:puts).with("The file contains 6 rows.")
+    expected_output.flatten.each do |output_line|
+      expect(output).to receive(:puts).with(output_line)
+    end
+    expect(output).to receive(:puts).with("\n\n\nReport generated in 0.0 minutes.")
+    expect(output).to receive(:puts).with("Thank you for flying with SAC Imports.")
+    expect(output).to receive(:puts).with("Report written to #{report_file}")
+
+    travel_to DateTime.new(2024, 1, 23, 10, 42)
+
+    expect { report.create }
+      .to change { ExternalTraining.count }.by(4).and \
+        change { Qualification.count }.by(0) # make sure no qualifications are issued
+
+    t1 = ExternalTraining.first
+    expect(t1.attributes.symbolize_keys).to include(
+      person_id: 600000,
+      name: "Leiterfortbildung Skifahren",
+      provider: "extern",
+      start_at: Date.new(2022, 4, 8),
+      finish_at: Date.new(2022, 4, 10),
+      training_days: 3,
+      link: nil,
+      remarks: nil
+    )
+    expect(t1.event_kind.label).to eq("Leiterfortbildung Skifahren")
+
+    t3 = ExternalTraining.third
+    expect(t3.attributes.symbolize_keys).to include(
+      person_id: 600002,
+      name: "Leiterfortbildung Snowboard",
+      provider: "extern",
+      start_at: Date.new(2022, 4, 15),
+      finish_at: Date.new(2022, 4, 16),
+      training_days: 2.5,
+      link: nil,
+      remarks: nil
+    )
+
+    expect(File.exist?(report_file)).to be_truthy
+
+    expect(csv_report.size).to eq(4)
+    expect(csv_report.first).to eq(report_headers)
+    expect(csv_report[1..]).to eq(
+      [["699999", "2023-03-22", "error", "Person muss ausgefüllt werden"],
+        ["600001", "2023-03-22", "warning", "event_kind with value X42 couldn't be found"],
+        ["600001", "2023-03-22", "error", "Kursart muss ausgefüllt werden"]]
+    )
+
+    File.delete(report_file)
+    expect(File.exist?(report_file)).to be(false)
+  end
+
+  def create_event_kinds
+    default_attrs = {
+      cost_unit: CostUnit.first,
+      cost_center: CostCenter.first,
+      kind_category: Event::KindCategory.first,
+      level: Event::Level.first
+    }
+    ski = Fabricate(:event_kind, default_attrs.merge(label: "Leiterfortbildung Skifahren", short_name: "X01"))
+    ski.event_kind_qualification_kinds.create!(qualification_kind: qualification_kinds(:ski_leader), role: "participant", category: "qualification")
+    Fabricate(:event_kind, default_attrs.merge(label: "Leiterfortbildung Klettern", short_name: "X02"))
+    sb = Fabricate(:event_kind, default_attrs.merge(label: "Leiterfortbildung Snowboard", short_name: "X03"))
+    sb.event_kind_qualification_kinds.create!(qualification_kind: qualification_kinds(:snowboard_leader), role: "participant", category: "prolongation")
+  end
+end

spec/fixtures/files/sac_imports_src/NAV22_fixture.csv

@@ -0,0 +1,6 @@
+600000,Leiterfortbildung Skifahren,extern,2022-04-08,2022-04-10,X01,3.00000,NULL,NULL,NULL
+600001,Leiterfortbildung Skifahren,extern,2022-04-08,2022-04-10,X01,3,NULL,NULL,NULL
+600002,Leiterfortbildung Snowboard,extern,2022-04-15,2022-04-16,X02,2.5,NULL,NULL,NULL
+699999,Leiterfortbildung Klettern,extern,2023-03-22,2023-03-22,X03,1.000,NULL,NULL,NULL
+600000,Leiterfortbildung Klettern,extern,2022-04-08,2022-04-16,X03,4.0,NULL,NULL,NULL
+600001,Leiterfortbildung Schwimmen,extern,2023-03-22,2023-03-22,X42,1.000,NULL,NULL,NULL