Avoid multiple report indexing

app/controllers/reports_controller.rb

@@ -60,7 +60,7 @@ def destroy
     if @report.destroy
       head :no_content
     else
-      Rails.logger.warn @report.errors.inspect
+      Rails.logger.error @report.errors.inspect
       render jsonapi: serialize(@report.errors), status: :unprocessable_entity
     end
   end
@@ -108,7 +108,7 @@ def create
     if @report.save
       render json: @report, status: :created
     else
-      Rails.logger.warn @report.errors.inspect
+      Rails.logger.error @report.errors.inspect
       render json: @report.errors, status: :unprocessable_entity
     end
   end

app/jobs/validation_job.rb

@@ -13,8 +13,8 @@ def perform(id, options={})
 
     if is_valid
       message = "[ValidationJob] Subset #{id} of Usage Report #{subset.report.uid} successfully validated."
-      subset.push_report
       subset.update_column(:aasm, "valid")
+      subset.push_report unless options[:validate_only]
       Rails.logger.info message
       true
     else

app/models/concerns/queueable.rb

@@ -22,7 +22,9 @@ def send_message(body, options={})
         message_body: body.to_json,
       }
 
-      sqs.send_message(options)
+      sent_message = sqs.send_message(options)
+      Rails.logger.info "[UsageUpdateImportWorker] Report " + report_id + "  has been queued." if sent_message.respond_to?("successful")
+      sent_message
     end
 
     # def queue_report(options={})

app/models/report.rb

@@ -1,13 +1,13 @@
-require 'base64'
-require 'digest'
+require "base64"
+require "digest"
 
 class Report < ApplicationRecord
   self.primary_key = :uid
 
   has_many :report_subsets, autosave: true, dependent: :destroy
 
   # has_one_attached :report
-  COMPRESSED_HASH_MESSAGE = { "code"=>69, "severity"=>"warning", "message"=>"report is compressed using gzip", "help-url"=>"https://github.com/datacite/sashimi", "data"=>"usage data needs to be uncompressed"}
+  COMPRESSED_HASH_MESSAGE = { "code" => 69, "severity" => "warning", "message" => "report is compressed using gzip", "help-url" => "https://github.com/datacite/sashimi", "data" => "usage data needs to be uncompressed" }.freeze
 
   # include validation methods for sushi
   include Metadatable
@@ -16,20 +16,20 @@ class Report < ApplicationRecord
   include Queueable
 
   # attr_accessor :month, :year, :compressed
-  validates_presence_of :report_id, :created_by, :client_id, :provider_id, :created, :reporting_period 
+  validates_presence_of :report_id, :created_by, :client_id, :provider_id, :created, :reporting_period
   validates_presence_of :report_datasets, if: :normal_report?
-  #, :report_datasets
+  # , :report_datasets
   validates :uid, uniqueness: true
   validates :validate_sushi, sushi: { presence: true }, if: :normal_report?
   attr_readonly :created_by, :month, :year, :client_id, :report_id, :uid
 
   # serialize :exceptions, Array
   before_validation :set_uid, on: :create
-  after_save :to_compress 
+  after_save :to_compress
   after_validation :clean_datasets
   # before_create :set_id
   after_commit :push_report, if: :normal_report?
-  after_destroy_commit :destroy_report_events,  on: :delete
+  after_destroy_commit :destroy_report_events, on: :delete
 
   # after_commit :validate_report_job, unless: :normal_report?
 
@@ -41,8 +41,8 @@ def destroy_report_events
     DestroyEventsJob.perform_later(uid)
   end
 
-  def self.destroy_events(uid, options={})
-    url = "#{ENV["API_URL"]}/events?" + URI.encode_www_form("subj-id" =>"#{ENV["API_URL"]}/reports/#{uid}")
+  def self.destroy_events(uid, _options = {})
+    url = "#{ENV['API_URL']}/events?" + URI.encode_www_form("subj-id" => "#{ENV['API_URL']}/reports/#{uid}")
     response = Maremma.get url
     events = response.fetch("data", [])
     # TODO add error class
@@ -70,51 +70,47 @@ def compress
     json_report = {
       "report-header":
       {
-        "report-name": self.report_name,
-        "report-id": self.report_id,
-        "release": self.release,
-        "created": self.created,
-        "created-by": self.created_by,
-        "reporting-period": self.reporting_period,
-        "report-filters": self.report_filters,
-        "report-attributes": self.report_attributes,
-        "exceptions": self.exceptions,
+        "report-name": report_name,
+        "report-id": report_id,
+        "release": release,
+        "created": created,
+        "created-by": created_by,
+        "reporting-period": reporting_period,
+        "report-filters": report_filters,
+        "report-attributes": report_attributes,
+        "exceptions": exceptions,
       },
-      "report-datasets": self.report_datasets,
+      "report-datasets": report_datasets,
     }
 
     ActiveSupport::Gzip.compress(json_report.to_json)
   end
-  
+
   def self.compressed_report?
-    return nil if self.exceptions.empty? || self.compressed.nil?
+    return nil if exceptions.empty? || compressed.nil?
 
     # self.exceptions.include?(COMPRESSED_HASH_MESSAGE)
-    code = self.exceptions.first.fetch("code", "")
+    code = exceptions.first.fetch("code", "")
 
     if code == 69
       true
-    else
-      nil
     end
   end
 
   def normal_report?
-    return nil if compressed_report? || self.report_datasets.nil?
+    return nil if compressed_report? || report_datasets.nil?
 
     true
   end
 
   def compressed_report?
-    return nil if self.exceptions && self.exceptions.empty? || self.compressed.nil?
+    return nil if exceptions&.empty? || compressed.nil?
 
     # self.exceptions.include?(COMPRESSED_HASH_MESSAGE)
-    code = self.exceptions.first.fetch("code","")
+    code = exceptions.first.fetch("code", "")
 
     if code == 69
       true
-    else
-      nil
     end
   end
 
@@ -126,9 +122,9 @@ def set_id
   end
 
   def to_compress
-    if self.compressed.nil? && self.report_subsets.empty?
-      ReportSubset.create(compressed: compress, report_id: self.uid)
-    elsif self.report_subsets.empty?
+    if compressed.nil? && report_subsets.empty?
+      ReportSubset.create(compressed: compress, report_id: uid)
+    elsif report_subsets.empty?
       ReportSubset.create(compressed: compressed, report_id: uid)
       # ReportSubset.create(compressed: self.compressed, report_id: self.report_id)
     end
@@ -139,15 +135,15 @@ def clean_datasets
   end
 
   def set_uid
-    return ActionController::ParameterMissing if self.reporting_period.nil?
+    return ActionController::ParameterMissing if reporting_period.nil?
 
     self.uid = SecureRandom.uuid if uid.blank?
     self.report_filters = report_filters.nil? ? [] : report_filters
     self.report_attributes = report_attributes.nil? ? [] : report_attributes
     self.exceptions = exceptions.nil? ? [] : exceptions
-    # self.report_id = self.uid 
-    month = Date.strptime(self.reporting_period["begin_date"],"%Y-%m-%d").month.to_s 
-    year = Date.strptime(self.reporting_period["begin_date"],"%Y-%m-%d").year.to_s 
+    # self.report_id = self.uid
+    month = Date.strptime(reporting_period["begin_date"], "%Y-%m-%d").month.to_s
+    year = Date.strptime(reporting_period["begin_date"], "%Y-%m-%d").year.to_s
     write_attribute(:month, month)
     write_attribute(:year, year)
   end

app/models/report_subset.rb

@@ -1,6 +1,7 @@
-require 'digest'
-require 'base64'
+require "digest"
+require "base64"
 
+### Subset are never updated. there will be always deleted and recreated
 class ReportSubset < ApplicationRecord
   # include validation methods for sushi
   include Queueable
@@ -25,7 +26,9 @@ def convert_report_job
   end
 
   def push_report
-    Rails.logger.debug "[UsageReports] calling queue for #{id}"
+    return false if aasm != "valid"
+
+    Rails.logger.info "[UsageReports] calling queue for #{id}"
     body = { report_id: report_subset_url }
 
     send_message(body) if ENV["AWS_REGION"].present?
@@ -50,7 +53,7 @@ def report_header
       "report-filters": report.report_filters,
       "report-attributes": report.report_attributes,
       "exceptions": report.exceptions,
-     }
+    }
   end
 
   def make_checksum

config/application.rb

@@ -72,6 +72,26 @@ class Application < Rails::Application
     # secret_key_base is not used by Rails API, as there are no sessions
     config.secret_key_base = 'blipblapblup'
 
+    config.lograge.enabled = true
+    config.lograge.formatter = Lograge::Formatters::Logstash.new
+    config.lograge.logger = LogStashLogger.new(type: :stdout)
+    config.logger = config.lograge.logger        ## LogStashLogger needs to be pass to rails logger, see roidrage/lograge#26
+    config.log_level = ENV["LOG_LEVEL"].to_sym   ## Log level in a config level configuration
+
+    config.lograge.ignore_actions = ["HeartbeatController#index", "IndexController#index"]
+    config.lograge.ignore_custom = lambda do |event|
+      event.payload.inspect.length > 100000
+    end
+    config.lograge.base_controller_class = "ActionController::API"
+
+    config.lograge.custom_options = lambda do |event|
+      exceptions = %w(controller action format id)
+      {
+        params: event.payload[:params].except(*exceptions),
+        uid: event.payload[:uid],
+      }
+    end
+
     # configure caching
     config.cache_store = :dalli_store, nil, { :namespace => ENV['APPLICATION'] }