Merge pull request #483 from slovensko-digital/GO-128/download_messag…

…es_from_fs

GO-128 Download messages from FS

.env

@@ -11,6 +11,7 @@ SITE_ADMIN_EMAILS=
 GOOGLE_CLIENT_ID=
 GOOGLE_CLIENT_SECRET=
 GOOD_JOB_EXECUTION_MODE=
+GROVER_NO_SANDBOX=true  # must be true for running chromium as root in Docker container
 ADMIN_IDS=
 LOG_LEVEL=
 DB_HOST=
@@ -24,3 +25,4 @@ UPVS_SSO_SUBJECT=
 UPVS_SSO_SP_CERTIFICATE=
 UPVS_SSO_SP_PRIVATE_KEY=
 UPVS_ENV=
+PDF_DISPLAY_URL=

.env.test

@@ -11,3 +11,5 @@ ZwIDAQAB
 ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=1
 ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=2
 ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=3
+GROVER_NO_SANDBOX=true  # must be true for running chromium as root in Docker container
+PDF_DISPLAY_URL=

Dockerfile

@@ -1,7 +1,8 @@
 FROM ruby:3.3.0
 
 # Install packages
-RUN apt-get update && apt-get install -y build-essential nodejs libpq-dev npm fop=1:2.* libsaxon-java libsaxonb-java
+RUN apt-get update && apt-get install -y build-essential nodejs libpq-dev npm fop=1:2.* libsaxon-java libsaxonb-java chromium \
+    && apt-get -y install libx11-xcb1 libxcomposite1 libasound2 libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6
 
 # Setup FOP to use saxon xslt parser
 RUN sed -i '/find_jars/i \

Gemfile

@@ -39,6 +39,7 @@ gem 'jwt'
 gem 'stimulus-rails'
 gem 'jsbundling-rails'
 gem 'pdf-reader'
+gem 'grover'
 
 # Monitoring
 gem 'rollbar'

Gemfile.lock

@@ -117,6 +117,9 @@ GEM
       activesupport
       tzinfo
     coderay (1.1.3)
+    combine_pdf (1.0.26)
+      matrix
+      ruby-rc4 (>= 0.1.5)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
     crack (1.0.0)
@@ -164,6 +167,9 @@ GEM
       fugit (>= 1.1)
       railties (>= 6.0.0)
       thor (>= 0.14.1)
+    grover (1.1.11)
+      combine_pdf (~> 1.0)
+      nokogiri (~> 1.0)
     hashdiff (1.1.0)
     hashery (2.1.2)
     hashie (5.0.0)
@@ -512,6 +518,7 @@ DEPENDENCIES
   erb_lint
   foreman
   good_job
+  grover
   htmlbeautifier
   importmap-rails
   jbuilder

app/components/message_draft_body_component.html.erb

@@ -93,7 +93,7 @@
 
       <% if @message.html_visualization.present? %>
         <div class="w-full">
-          <%= tag.iframe class: "relative border-none overflow-hidden h-full w-full", srcdoc: @message.format_html_visualization, onload: "(
+          <%= tag.iframe class: "relative border-none overflow-hidden h-full w-full", srcdoc: @message.html_visualization, onload: "(
               function(iframe) {
                 iframe.contentWindow.document.body.style['height'] = 'unset';
                 iframe.contentWindow.document.body.style['min-height'] = 'unset';

app/components/settings/rules/rule_header_form_component.rb

@@ -6,6 +6,7 @@ def before_render
     @trigger_events_list = [
       [t('message_created'), 'message_created'],
       [t('message_draft_submitted'), 'message_draft_submitted'],
+      [t('message_object_downloaded'), 'message_object_downloaded'],
     ]
   end
 end

app/controllers/message_objects_controller.rb

@@ -26,6 +26,9 @@ def show
 
   def download
     authorize @message_object
+
+    EventBus.publish(:message_object_downloaded, @message_object)
+
     send_data @message_object.content, filename: MessageObjectHelper.displayable_name(@message_object), type: @message_object.mimetype, disposition: :download
   end
 
@@ -34,6 +37,8 @@ def download_pdf
 
     pdf_content = @message_object.prepare_pdf_visualization
     if pdf_content
+      EventBus.publish(:message_object_downloaded, @message_object)
+
       send_data pdf_content, filename: MessageObjectHelper.pdf_name(@message_object), type: 'application/pdf', disposition: :download
     else
       redirect_back fallback_location: message_thread_path(@message_object.message.thread), alert: "Obsah nie je možné stiahnuť."

app/helpers/application_helper.rb

@@ -9,10 +9,10 @@ def nice_datetime(datetime)
     end
   end
 
-  def nice_datetime_with_time(datetime)
-    if datetime.today?
+  def nice_datetime_with_time(datetime, full_date: false)
+    if datetime.today? && !full_date
       l(datetime, format: '%H:%M')
-    elsif datetime.year == Date.current.year
+    elsif datetime.year == Date.current.year && !full_date
       l(datetime, format: '%e. %b %H:%M')
     else
       l(datetime, format: '%e. %b %Y %H:%M')

app/helpers/fs/message_helper.rb

@@ -0,0 +1,23 @@
+module Fs::MessageHelper
+  def self.build_html_visualization(message)
+    return [ActionController::Base.new.render_to_string('fs/messages/_submission', layout: false, locals: { message: message }), build_html_visualization_from_form(message)].compact.join('<hr>') if message.outbox?
+
+    # TODO: Vieme aj lepsie identifikovat? Nejake dalsie typy v tejto kategorii neexistuju?
+    template = if message.title.in?(['Informácia o podaní', 'Informácia o odmietnutí podania'])
+                 'fs/messages/_delivery_report'
+               else
+                 'fs/messages/_generic_message'
+               end
+
+    ActionController::Base.new.render_to_string(template, layout: false, locals: { message: message })
+  end
+
+  def self.build_html_visualization_from_form(message)
+    raise 'Missing Fs::Form XSLT' unless message.form&.xslt_txt
+    return unless message.form_object&.unsigned_content
+
+    template = Nokogiri::XSLT(message.form.xslt_txt)
+
+    ActionController::Base.new.render_to_string('fs/messages/_style', layout: false, locals: { message: message }) + ActionController::Base.helpers.simple_format(template.transform(message.form_object.xml_unsigned_content).to_s)
+  end
+end

app/helpers/message_helper.rb

@@ -2,10 +2,4 @@ module MessageHelper
   def self.export_filename(message)
     "#{message.delivered_at.to_date}-sprava-#{message.id}.zip"
   end
-
-  def format_html_visualization
-    return ActionController::Base.helpers.simple_format(html_visualization) if is_a?(Fs::MessageDraft)
-
-    html_visualization
-  end
 end

app/jobs/fs/download_received_message_job.rb

@@ -0,0 +1,18 @@
+module Fs
+  class DownloadReceivedMessageJob < ApplicationJob
+    def perform(fs_message_id, box:, fs_client: FsEnvironment.fs_client)
+      raise unless box.is_a?(Fs::Box)
+      return unless box.syncable?
+
+      return if box.messages.where("metadata ->> 'fs_message_id' = ?", fs_message_id).any?
+
+      ActiveRecord::Base.transaction do
+        fs_api = fs_client.api(api_connection: box.api_connection, box: box)
+
+        raw_message = fs_api.fetch_received_message(fs_message_id)
+
+        Fs::Message.create_inbox_message_with_thread!(raw_message, box: box)
+      end
+    end
+  end
+end

app/jobs/fs/download_sent_message_job.rb

@@ -0,0 +1,20 @@
+module Fs
+  class DownloadSentMessageJob < ApplicationJob
+    def perform(fs_message_id, box:, fs_client: FsEnvironment.fs_client)
+      raise unless box.is_a?(Fs::Box)
+      return unless box.syncable?
+
+      return if box.messages.where(type: [nil, 'Message']).where("metadata ->> 'fs_message_id' = ?", fs_message_id).any?
+
+      ActiveRecord::Base.transaction do
+        fs_api = fs_client.api(api_connection: box.api_connection, box: box)
+
+        raw_message = fs_api.fetch_sent_message(fs_message_id)
+
+        message = Fs::Message.create_outbox_message_with_thread!(raw_message, box: box)
+
+        DownloadSentMessageRelatedMessagesJob.set(wait: 3.minutes).perform_later(message)
+      end
+    end
+  end
+end

app/jobs/fs/download_sent_message_related_messages_job.rb

@@ -0,0 +1,20 @@
+module Fs
+  class DownloadSentMessageRelatedMessagesJob < ApplicationJob
+    def perform(outbox_message, from: nil, to: nil, fs_client: FsEnvironment.fs_client, batch_size: 25)
+      raise unless outbox_message.box.is_a?(Fs::Box)
+      return unless outbox_message.box.syncable?
+
+      fs_api = fs_client.api(api_connection: outbox_message.box.api_connection, box: outbox_message.box)
+
+      0.step do |k|
+        received_messages = fs_api.fetch_received_messages(sent_message_id: outbox_message.metadata['fs_message_id'], page: k + 1, count: batch_size, from: from, to: to)
+
+        received_messages['messages'].each do |received_message|
+          ::Fs::DownloadReceivedMessageJob.perform_later(received_message['message_id'], box: outbox_message.box)
+        end
+
+        break if received_messages['messages'].size < batch_size
+      end
+    end
+  end
+end

app/jobs/fs/fetch_forms_job.rb

@@ -7,6 +7,7 @@ def perform(fs_client: FsEnvironment.fs_client, download_related_documents_job:
         identifier: fs_form_data['identifier'],
       ).tap do |form|
         form.update(
+          submission_type_identifier: fs_form_data['submission_type_identifier'],
           name: fs_form_data['name'],
           group_name: fs_form_data['form_group_name'],
           subtype_name: fs_form_data['subtype_name'],

app/jobs/fs/submit_message_draft_job.rb

@@ -7,6 +7,8 @@ def perform(message_draft, bulk_submit: false, fs_client: FsEnvironment.fs_clien
     response = fs_api.post_submission(
       message_draft.form.identifier,
       Base64.strict_encode64(message_draft.form_object.content),
+      message_uuid: message_draft.uuid,
+      form_object_uuid: message_draft.form_object.uuid,
       allow_warn_status: true,
       is_signed: message_draft.form_object.is_signed,
       mime_type: message_draft.form_object.mimetype

app/jobs/fs/submit_message_draft_result_job.rb

@@ -4,7 +4,10 @@ def perform(message_draft, location_header, fs_client: FsEnvironment.fs_client)
 
     if 200 == response[:status]
       message_draft.metadata[:status] = 'submitted'
+      message_draft.metadata[:fs_message_id] = response[:body]['sent_message_id']
       message_draft.save
+
+      ::Fs::DownloadSentMessageJob.perform_later(response[:body]['sent_message_id'], box: message_draft.box)
     elsif [400, 422].include?(response[:status])
       message_draft.metadata[:status] = 'submit_fail'
       message_draft.add_cascading_tag(message_draft.tenant.submission_error_tag)

app/jobs/fs/sync_all_boxes_job.rb

@@ -0,0 +1,12 @@
+module Fs
+  class SyncAllBoxesJob < ApplicationJob
+    def perform
+      Fs::Box.where(syncable: true).find_each do |box|
+        SyncBoxJob.perform_later(box)
+      end
+
+      # TODO ponastavovat v BetterUptime
+      BetterUptimeApi.ping_heartbeat('FS_SYNC')
+    end
+  end
+end

app/jobs/fs/sync_box_job.rb

@@ -0,0 +1,12 @@
+module Fs
+  class SyncBoxJob < ApplicationJob
+    def perform(box, from: Date.today - 1.week, to: Date.tomorrow)
+      raise unless box.is_a?(Fs::Box)
+      return unless box.syncable?
+
+      box.messages.outbox.find_each do |outbox_message|
+        DownloadSentMessageRelatedMessagesJob.perform_later(outbox_message, from: from, to: to)
+      end
+    end
+  end
+end

app/lib/event_bus.rb

@@ -28,7 +28,7 @@ def self.reset!
 # wiring
 
 # automation
-[:message_thread_created, :message_created, :message_draft_submitted].each do |event|
+[:message_thread_created, :message_created, :message_draft_submitted, :message_object_downloaded].each do |event|
   EventBus.subscribe_job event, Automation::ApplyRulesForEventJob
 end
 

app/lib/fs/api.rb

@@ -29,19 +29,27 @@ def get_subjects
     end
 
     def fetch_sent_messages(page: 1, count: 100, obo: @obo)
-      request(:get, "sent-messages", {}, jwt_header(obo).merge(fs_credentials_header))[:body]
+      request(:get, "sent-messages?page=#{page}&per_page=#{count}", {}, jwt_header(obo).merge(fs_credentials_header))[:body]
     end
 
     def fetch_sent_message(message_id, obo: @obo)
-      request(:get, "sent-messages/#{message_id}", {}, jwt_header(obo).merge(fs_credentials_header))[:body]
+      request(:get, "sent-messages/#{CGI.escape(message_id)}", {}, jwt_header(obo).merge(fs_credentials_header))[:body]
     end
 
-    def fetch_received_messages(page: 1, count: 100, obo: @obo)
-      request(:get, "received-messages", {}, jwt_header(obo).merge(fs_credentials_header))[:body]
+    def fetch_received_messages(sent_message_id: nil, page: 1, count: 100, from: nil, to: nil, obo: @obo)
+      query = {
+        sent_message_id: sent_message_id,
+        page: page,
+        per_page: count,
+        from: from,
+        to: to
+      }.compact.to_query
+
+      request(:get, "received-messages?#{query}", {}, jwt_header(obo).merge(fs_credentials_header))[:body]
     end
 
     def fetch_received_message(message_id, obo: @obo)
-      request(:get, "received-messages/#{message_id}", {}, jwt_header(obo).merge(fs_credentials_header))[:body]
+      request(:get, "received-messages/#{CGI.escape(message_id)}", {}, jwt_header(obo).merge(fs_credentials_header))[:body]
     end
 
     def post_validation(form_identifier, content)
@@ -52,8 +60,10 @@ def delete_validation(validation_id)
       request(:delete, "validations/#{validation_id}", {}, jwt_header, accept_negative: true)
     end
 
-    def post_submission(form_identifier, content, allow_warn_status: true, is_signed: true, mime_type: 'application/vnd.etsi.asic-e+zip', obo: @obo)
+    def post_submission(form_identifier, content, allow_warn_status: true, message_uuid:, form_object_uuid:, is_signed: true, mime_type: 'application/vnd.etsi.asic-e+zip', obo: @obo)
       request(:post, "submissions", {
+        message_container_message_id: message_uuid,
+        message_container_form_object_id: form_object_uuid,
         is_signed: is_signed,
         mime_type: mime_type,
         form_identifier: form_identifier,

app/lib/utils.rb

@@ -100,7 +100,6 @@ def file_extension_by_mimetype(mimetype)
 
   # TODO use UPVS API to detect if document is signed
   def is_signed?(entry_name:, content:)
-
     case File.extname(entry_name).downcase
     when '.asice', '.asics', '.xzep'
       true

app/models/automation/condition.rb

@@ -21,7 +21,7 @@ class Condition < ApplicationRecord
     attr_accessor :delete_record
 
     # when adding items, check defaults in condition_form_component.rb
-    ATTR_LIST = %i[box sender_name recipient_name title sender_uri recipient_uri attachment].freeze
+    ATTR_LIST = %i[box sender_name recipient_name title sender_uri recipient_uri attachment fs_submission_status fs_message_type object_type].freeze
 
     def valid_condition_type_list_for_attr
       Automation::Condition.subclasses.map do |subclass|
@@ -36,7 +36,7 @@ def box_list
 
   class ContainsCondition < Automation::Condition
     validates :value, presence: true
-    VALID_ATTR_LIST = %w[sender_name recipient_name title].freeze
+    VALID_ATTR_LIST = %w[sender_name recipient_name title object_type].freeze
     validates :attr, inclusion: { in: VALID_ATTR_LIST }
 
     def satisfied?(thing)
@@ -50,7 +50,7 @@ def cleanup_record
 
   class MetadataValueCondition < Automation::Condition
     validates :value, presence: true
-    VALID_ATTR_LIST = %w[sender_uri recipient_uri].freeze
+    VALID_ATTR_LIST = %w[sender_uri recipient_uri fs_submission_status fs_message_type].freeze
     validates :attr, inclusion: { in: VALID_ATTR_LIST }
 
     def satisfied?(thing)
@@ -81,6 +81,20 @@ def cleanup_record
     end
   end
 
+  class MessageMetadataValueCondition < Automation::Condition
+    validates :value, presence: true
+    VALID_ATTR_LIST = %w[fs_message_type].freeze
+    validates :attr, inclusion: { in: VALID_ATTR_LIST }
+
+    def satisfied?(thing)
+      thing.message.metadata && thing.message.metadata[attr]&.match?(value)
+    end
+
+    def cleanup_record
+      self.condition_object = nil
+    end
+  end
+
   class AttachmentContentContainsCondition < Automation::Condition
     validates :value, presence: true
     VALID_ATTR_LIST = ['attachment'].freeze