refactor: datacollector

Module Datacollector
separate business logic
  * collector.rb: file/folder crawler for a given device
  * correspondence.rb: set the sender and recipient
     encompass AR Model logic to create the attachments into the proper
     containers
  * configuration.rb: validate the datacollector device config
     also instantiate the SFTP client if applicable
  * collector_file: file/folder class for basic file operations
      independent of whether the file is on local fs or over sftp

  AdminApi device connection testing:
    * now rely on Datacollector::Configuration

.env.development

@@ -1,8 +1,9 @@
-SFTP_HOST=sftp
+SFTP_HOST=localhost
 SFTP_PORT=22
 SFTP_UPLOAD_FOLDER=uploads
-SFTP_USER=sftp_test
+SFTP_USER=$USER
 SFTP_PASSWORD=sftp_test
+SFTP_AUTH_METHODS=publickey
 
 MESSAGE_ENABLE=false
 MESSAGE_AUTO_INTERNAL=6000

app/api/chemotion/admin_api.rb

@@ -8,7 +8,6 @@
 module Chemotion
   # Publish-Subscription MessageAPI
   class AdminAPI < Grape::API
-    helpers AdminHelpers
     resource :admin do
       before do
         error!(401) unless current_user.is_a?(Admin)

app/api/chemotion/admin_device_api.rb

@@ -2,7 +2,6 @@
 
 module Chemotion
   class AdminDeviceAPI < Grape::API
-    helpers AdminHelpers
     resource :admin_devices do
       before do
         error!(401) unless current_user.is_a?(Admin)
@@ -127,7 +126,7 @@ class AdminDeviceAPI < Grape::API
         end
       end
 
-      # test datacollector sftp
+      # test datacollector sftp connection
       params do
         requires :id, type: Integer
         optional :datacollector_method, type: String
@@ -138,21 +137,9 @@ class AdminDeviceAPI < Grape::API
       end
       route_param :test_sftp do
         post do
-          case params[:datacollector_authentication]
-          when 'password'
-            credentials = Rails.configuration.datacollectors.sftpusers.find do |e|
-              e[:user] == params[:datacollector_user]
-            end
-            raise 'No match user credentials!' unless credentials
-
-            connect_sftp_with_password(
-              host: params[:datacollector_host],
-              user: credentials[:user],
-              password: credentials[:password],
-            )
-          when 'keyfile'
-            connect_sftp_with_key(params)
-          end
+          # make options hashie compatible
+          options = Hashie::Mash.new declared(params, include_missing: false).merge(info: id)
+          Datacollector::Configuration.new!(options)
 
           { status: 'success', message: 'Test connection successfully.' }
         rescue StandardError => e

app/clients/sftp_client.rb

@@ -1,27 +1,57 @@
+# frozen_string_literal: true
+
 class SFTPClient
+  ALLOWED_OPTIONS = %i[
+    port password keys keys_only auth_methods timeout verbose key_data number_of_password_prompts
+  ].freeze
+
   def self.with_default_settings
-    @with_default_settings ||= self.new(
-      {
-        host: ENV['SFTP_HOST'],
-        port: ENV['SFTP_PORT'],
-        username: ENV['SFTP_USER'],
-        password: (ENV['SFTP_PASSWORD'] if ENV['SFTP_PASSWORD']),
-        keys: (ENV['SFTP_KEYS'] if ENV['SFTP_KEYS'])
-      }
+    @with_default_settings ||= new(
+      host: ENV.fetch('SFTP_HOST', nil),
+      user: ENV.fetch('SFTP_USER', nil),
+
+      port: ENV.fetch('SFTP_PORT', nil),
+      password: ENV.fetch('SFTP_PASSWORD', nil),
+      keys: ENV.fetch('SFTP_KEYS', nil),
+      auth_methods: ENV.fetch('SFTP_AUTH_METHODS', nil),
     )
   end
 
-  def initialize(sftp_config)
-    @host = sftp_config.fetch(:host, nil)
-    @port = sftp_config.fetch(:port, nil)
-    @username = sftp_config.fetch(:username, nil)
-    @password = sftp_config.fetch(:password, nil)
+  attr_reader :host, :user, :session_options
 
-    # This specifies the list of private key files to use instead of the
-    # defaults ($HOME/.ssh/id_dsa, $HOME/.ssh2/id_dsa, $HOME/.ssh/id_rsa, and $HOME/.ssh2/id_rsa).
-    # The value of this option should be an array of strings.
-    # See http://net-ssh.github.io/ssh/v1/chapter-2.html
-    @keys = sftp_config.fetch(:keys, [])
+  # Initialize the SFTP parameters with the given host, username and options
+  # to be passed to the Net::SFTP.start method.
+  #
+  # @param host [String] the host to connect to
+  # @param user [String] the username to use for the connection
+  # @param options [Hash] the options to pass to the Net::SFTP.start method
+  # @options options [String] :port the port to connect to
+  # @options options [String] :password the password to use for the connection
+  # @options options [Array<String>] :keys the key files to use for the connection
+  # @options options [Boolean] :keys_only whether to use only the keys for the connection
+  # @options options [Array<String>] :auth_methods the authentication methods to use
+  # @options options [Integer] :timeout the timeout to use for the connection
+  # @options options [Boolean] :verbose whether to be verbose
+  # @options options [Array<String>] :key_data the key data to use for the connection
+  # @example new('example.com', 'user', { port: '2222', password: 'password' })
+  # @example new('sfpt://[email protected]:1234/unrelevant')
+  # @note see ALLOWED_OPTIONS for the allowed options
+  # @note optional user and port params have precedence over the parsed ones from the host
+  def initialize(host = nil, user = nil, **options)
+    extract_host_and_user(host, user, **options)
+    raise ArgumentError, 'No host or user given' unless @host && @user
+
+    %i[auth_methods keys].each do |key|
+      options[key] = options[key].split(',') if options[key].is_a?(String)
+    end
+    default_options(options)
+    @session_options = @session_options.merge(options.slice(*ALLOWED_OPTIONS)).compact
+  end
+
+  # Test the connection to the SFTP server
+  # @return [Boolean] whether the connection was successful
+  def open?
+    with_session(&:open?)
   end
 
   def upload_file!(local_path, remote_path)
@@ -47,6 +77,12 @@ def download_file!(remote_path, local_path)
     end
   end
 
+  def download_directory!(remote_path, local_path)
+    with_session(remote_path, local_path) do |sftp|
+      sftp.download!(remote_path, local_path, requests: 5, recursive: true)
+    end
+  end
+
   def move_file!(remote_src_path, remote_target_path)
     with_session(remote_src_path, remote_target_path) do |sftp|
       sftp.rename(remote_src_path, remote_target_path).wait
@@ -56,7 +92,13 @@ def move_file!(remote_src_path, remote_target_path)
 
   def remove_file!(remote_path)
     with_session(remote_path) do |sftp|
-      sftp.remove(remote_path).wait
+      sftp.remove!(remote_path)
+    end
+  end
+
+  def remove_dir!(remote_path)
+    with_session(remote_path) do |sftp|
+      sftp.session.exec!("rm -rf #{remote_path}")
     end
   end
 
@@ -66,40 +108,80 @@ def read_file(remote_path)
     end
   end
 
-  def file_exists?(remote_path)
+  def exist?(remote_path)
+    with_session do |sftp|
+      sftp.stat!(remote_path) && true
+    rescue Net::SFTP::StatusException => e
+      return false if e.code == 2 # No such file
+
+      raise e
+    end
+  end
+
+  def file?(remote_path)
+    parent = File.dirname(remote_path)
+    name = File.basename(remote_path)
+    result = nil
+    with_session do |sftp|
+      result = sftp.dir.glob(parent, name).find(&:file?).present?
+    end
+    result
+  end
+
+  def directory?(remote_path)
+    parent = File.dirname(remote_path)
+    name = File.basename(remote_path)
+    result = nil
+    with_session(parent) do |sftp|
+      result = sftp.dir.glob(parent, name).find(&:directory?).present?
+    end
+    result
+  end
+
+  def entries(remote_path)
     with_session(remote_path) do |sftp|
-      begin
-        sftp.stat!(remote_path)
-      rescue Net::SFTP::StatusException => e
-        if e.code == 2 # No such file
-          return false
-        else
-          raise e
-        end
-      end
-      true
+      sftp.dir.entries(remote_path)
+    end
+  end
+
+  def glob(remote_path, pattern, flags = 0)
+    with_session do |sftp|
+      sftp.dir.glob(remote_path, pattern, flags)
     end
   end
 
   private
 
-    def with_session(*args_of_caller)
-      begin
-        if @keys.blank?
-          Net::SFTP.start(@host, @username, port: @port, password: @password) do |sftp|
-            data = yield(sftp)
-            return data if data
-          end
-        else
-          Net::SFTP.start(@host, @username, port: @port, keys: @keys) do |sftp|
-            data = yield(sftp)
-            return data if data
-          end
-        end
-      rescue Exception => e
-        # for usage of caller, see Kernel#caller
-        raise SFTPClientError.new(e, caller[1], args_of_caller)
-      end
-    end
+  # Extract Host, User and Port from the given URI
+  def extract_host_and_user(host, user, **options)
+    # extract host information from the given host, remove protocol prefix if present
+    host = (host.presence || options.delete(:host)).sub(%r{^[a-z]+://}, '')
+    uri = URI.parse("ssh://#{host}")
+    @host = uri.host
+    @user = user || uri.user || options.delete(:user)
+    @root_path = options.delete(:root_path) || (uri.path != '/' && uri.path.presence)
+    @session_options = { port: uri.port }
+  end
+
+  # Set some default options
+  def default_options(options)
+    @session_options = @session_options.merge(
+      timeout: 5, verbose: :warn, keys_only: true, auth_methods: [], number_of_password_prompts: 0,
+    )
+    @session_options[:auth_methods] = %w[publickey] if options[:keys].present? || options[:key_data].present?
+    @session_options[:auth_methods] += %w[password] if options[:password].present?
+    @session_options[:auth_methods] = %w[publickey] if options[:keys_only].present?
+  end
 
+  # rubocop:disable Lint/RescueException
+  def with_session(*args_of_caller)
+    Net::SFTP.start(@host, @username, @session_options) do |sftp|
+      data = yield(sftp)
+      return data unless data.nil?
+    end
+  rescue Exception => e
+    # for usage of caller, see Kernel#caller
+    raise SFTPClientError.new(e, caller(2..2).first, args_of_caller)
+  end
+  # rubocop:enable Lint/RescueException
 end

app/errors/errors.rb

@@ -14,4 +14,6 @@ def initialize(message = 'Forbidden')
       super(message)
     end
   end
+
+  class DatacollectorError < ApplicationError; end
 end

app/jobs/collect_data_from_local_job.rb

@@ -2,7 +2,7 @@ class CollectDataFromLocalJob < ApplicationJob
   queue_as :collect_data
 
   def perform
-    collector = Foldercollector.new
-    collector.execute(false)
+    devices = Device.where(datacollector_method: 'folderwatcherlocal')
+    Datacollector::Collectors.bulk_execute(devices)
   end
 end

app/jobs/collect_data_from_mail_job.rb

@@ -1,8 +1,8 @@
 class CollectDataFromMailJob < ApplicationJob
   queue_as :collect_data
-  
+
   def perform
-    collector = Mailcollector.new
+    collector = Datacollector::Mailcollector.new
     collector.execute
   end
 end

app/jobs/collect_data_from_sftp_job.rb

@@ -2,7 +2,7 @@ class CollectDataFromSftpJob < ApplicationJob
   queue_as :collect_data
 
   def perform
-    collector = Foldercollector.new
-    collector.execute(true)
+    devices = Device.where(datacollector_method: 'folderwatchersftp')
+    Datacollector::Collectors.bulk_execute(devices)
   end
 end

app/jobs/collect_file_from_local_job.rb

@@ -2,7 +2,7 @@ class CollectFileFromLocalJob < ApplicationJob
   queue_as :collect_data
 
   def perform
-    collector = Filecollector.new
-    collector.execute(false)
+    devices = Device.where(datacollector_method: 'filewatcherlocal')
+    Datacollector::Collectors.bulk_execute(devices)
   end
 end

app/jobs/collect_file_from_sftp_job.rb

@@ -1,8 +1,8 @@
 class CollectFileFromSftpJob < ApplicationJob
   queue_as :collect_data
-  
+
   def perform
-    collector = Filecollector.new
-    collector.execute(true)
+    devices = Device.where(datacollector_method: 'filewatchersftp')
+    Datacollector::Collectors.bulk_execute(devices)
   end
 end