Merge pull request #1078 from SUSE/rmt_deb_codesprint

Debian support on RMT

.rubocop.yml

@@ -50,3 +50,11 @@ Style/OpenStructUse:
 
 RSpec/MessageChain:
   Enabled: false
+
+# We disable this check since we are stubing class methods
+# that are already tested in their respective tests.
+# This way we circumvent tight coupling between tests and
+# implementation.
+# Count: 107 offenses
+RSpec/SubjectStub:
+  Enabled: false

.rubocop_todo.yml

@@ -16,7 +16,6 @@ Layout/ArgumentAlignment:
     - 'app/models/product.rb'
     - 'engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb'
     - 'spec/factories/products.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
     - 'spec/models/migration_engine_spec.rb'
 
 # Offense count: 3
@@ -40,7 +39,6 @@ Layout/EmptyLineAfterGuardClause:
     - 'lib/rmt/cli/repos_custom.rb'
     - 'lib/rmt/cli/smt_importer.rb'
     - 'lib/rmt/fiber_request.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
@@ -102,7 +100,6 @@ Layout/LineEndStringConcatenationIndentation:
 # Cop supports --auto-correct.
 Lint/AmbiguousOperatorPrecedence:
   Exclude:
-    - 'lib/rmt/cli/mirror.rb'
     - 'lib/rmt/cli/products.rb'
     - 'lib/rmt/cli/repos.rb'
     - 'lib/rmt/cli/repos_custom.rb'
@@ -241,15 +238,13 @@ Performance/RegexpMatch:
     - 'engines/registration_sharing/lib/registration_sharing.rb'
     - 'engines/strict_authentication/app/controllers/strict_authentication/authentication_controller.rb'
     - 'lib/rmt/cli/base.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
 
 # Offense count: 4
 # Cop supports --auto-correct.
 # Configuration parameters: AutoCorrect.
 Performance/StringInclude:
   Exclude:
     - 'engines/strict_authentication/app/controllers/strict_authentication/authentication_controller.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
 
 # Offense count: 167
 # Configuration parameters: Prefixes.
@@ -279,7 +274,6 @@ RSpec/EmptyLineAfterHook:
     - 'engines/strict_authentication/spec/requests/services_controller_spec.rb'
     - 'engines/strict_authentication/spec/requests/strict_authentication/authentication_controller_spec.rb'
     - 'spec/lib/rmt/config_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
     - 'spec/models/product_spec.rb'
     - 'spec/requests/api/connect/v3/subscriptions/systems_controller_spec.rb'
     - 'spec/requests/api/connect/v3/systems/activations_controller_spec.rb'
@@ -321,7 +315,6 @@ RSpec/ExpectInHook:
     - 'spec/lib/rmt/cli/main_spec.rb'
     - 'spec/lib/rmt/cli/repos_custom_spec.rb'
     - 'spec/lib/rmt/cli/repos_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
     - 'spec/lib/rmt/scc_spec.rb'
 
 # Offense count: 3
@@ -358,11 +351,6 @@ RSpec/ImplicitSubject:
     - 'spec/models/migration_engine_spec.rb'
     - 'spec/models/product_spec.rb'
 
-# Offense count: 34
-# Configuration parameters: AssignmentOnly.
-RSpec/InstanceVariable:
-  Exclude:
-    - 'spec/lib/rmt/mirror_spec.rb'
 
 # Offense count: 1
 RSpec/IteratedExpectation:
@@ -430,7 +418,6 @@ RSpec/Rails/HttpStatus:
 RSpec/ReceiveCounts:
   Exclude:
     - 'spec/lib/rmt/lockfile_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
     - 'spec/suse/connect/api_spec.rb'
 
 # Offense count: 3
@@ -454,7 +441,6 @@ RSpec/StubbedMock:
     - 'spec/lib/rmt/cli/main_spec.rb'
     - 'spec/lib/rmt/cli/smt_importer_spec.rb'
     - 'spec/lib/rmt/cli/systems_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
     - 'spec/lib/rmt/scc_spec.rb'
     - 'spec/support/shared_examples/cli.rb'
     - 'spec/suse/connect/api_spec.rb'

config/rmt.yml

@@ -27,7 +27,6 @@ scc:
 
 mirroring:
   mirror_src: false
-  verify_rpm_checksums: false
   dedup_method: hardlink
 
 http_client:

lib/rmt.rb

@@ -1,5 +1,5 @@
 module RMT
-  VERSION ||= '2.15'.freeze
+  VERSION ||= '2.16'.freeze
 
   DEFAULT_USER = '_rmt'.freeze
   DEFAULT_GROUP = 'nginx'.freeze

lib/rmt/cli/base.rb

@@ -117,8 +117,6 @@ def logger
     @logger
   end
 
-  private
-
   def needs_path(path, writable: false)
     # expand the path to make it easier to work with
     path = File.expand_path(path)
@@ -137,6 +135,8 @@ def needs_path(path, writable: false)
     path
   end
 
+  private
+
   # Allows to have any type of multi input that you want:
   #
   # 1575 (alone)

lib/rmt/cli/export.rb

@@ -29,26 +29,30 @@ def settings(path)
   REPOS
   def repos(path)
     path = needs_path(path, writable: true)
-
-    mirror = RMT::Mirror.new(mirroring_base_dir: path, logger: logger, airgap_mode: true)
+    suma_product_tree = RMT::Mirror::SumaProductTree.new(mirroring_base_dir: path, logger: logger)
 
     begin
-      mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/')
+      suma_product_tree.mirror
     rescue RMT::Mirror::Exception => e
-      logger.warn(e.message)
+      logger.warn(_('Exporting SUSE Manager product tree failed: %{error_message}') % { error_message: e.message })
     end
 
     repos_file = File.join(path, 'repos.json')
     raise RMT::CLI::Error.new(_('%{file} does not exist.') % { file: repos_file }) unless File.exist?(repos_file)
 
     repos = JSON.parse(File.read(repos_file))
-    repos.each do |repo|
+    repos.each do |repo_json|
+      repo = Repository.find_by(external_url: repo_json['url'])
+
       begin
-        mirror.mirror(
-          repository_url: repo['url'],
-          local_path: Repository.make_local_path(repo['url']),
-          auth_token: repo['auth_token']
-        )
+        configuration = {
+          repository: repo,
+          logger: logger,
+          mirroring_base_dir: path,
+          mirror_sources: RMT::Config.mirror_src_files?,
+          is_airgapped: true
+        }
+        RMT::Mirror.new(**configuration).mirror_now
       rescue RMT::Mirror::Exception => e
         warn e.to_s
       end

lib/rmt/cli/import.rb

@@ -12,23 +12,26 @@ def data(path)
   def repos(path)
     RMT::Lockfile.lock do
       path = needs_path(path)
-
-      mirror = RMT::Mirror.new(logger: logger, airgap_mode: true)
-
       repos_file = File.join(path, 'repos.json')
+
       raise RMT::CLI::Error.new(_('%{file} does not exist.') % { file: repos_file }) unless File.exist?(repos_file)
 
       begin
-        exported_suma_path = File.join(path, '/suma/')
-        suma_repo_url = URI.join('file://', exported_suma_path).to_s
-        mirror.mirror_suma_product_tree(repository_url: suma_repo_url)
+        suma_repo_url = URI.join('file://', File.join(path, '/suma/')).to_s
+
+        RMT::Mirror::SumaProductTree.new(
+          mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR,
+          logger: logger,
+          url: suma_repo_url
+        ).mirror
       rescue RMT::Mirror::Exception => e
-        logger.warn(e.message)
+        logger.warn(_('Importing suma product tree failed: %{error_message}') % { error_message: e.message })
       end
 
       repos = JSON.parse(File.read(repos_file))
       repos.each do |repo_json|
         repo = Repository.find_by(external_url: repo_json['url'])
+
         if repo.nil?
           warn _('repository by URL %{url} does not exist in database') % { url: repo_json['url'] }
           next
@@ -37,12 +40,20 @@ def repos(path)
         begin
           exported_repo_path = File.join(path, Repository.make_local_path(repo_json['url']))
           repo_url = URI.join('file://', exported_repo_path).to_s
-          mirror.mirror(
-            repository_url: repo_url,
-            local_path: Repository.make_local_path(repo.external_url),
-            auth_token: repo.auth_token,
-            repo_name: repo.name
-          )
+
+          # We temporary alter the external_url to point to files on this. This is
+          # a bit of a hack to make sure we import from the disk rather the a real
+          # remote origin
+          repo.external_url = repo_url
+
+          configuration = {
+            repository: repo,
+            logger: logger,
+            mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR,
+            mirror_sources: RMT::Config.mirror_src_files?,
+            is_airgapped: true
+          }
+          RMT::Mirror.new(**configuration).mirror_now
 
           repo.refresh_timestamp!
         rescue RMT::Mirror::Exception => e

lib/rmt/cli/mirror.rb

@@ -5,11 +5,12 @@ class RMT::CLI::Mirror < RMT::CLI::Base
   def all
     RMT::Lockfile.lock('mirror') do
       begin
-        mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/')
+        suma_product_tree.mirror
       rescue RMT::Mirror::Exception => e
-        errors << _('Mirroring SUMA product tree failed: %{error_message}') % { error_message: e.message }
+        errors << (_('Mirroring SUMA product tree failed: %{error_message}') % { error_message: e.message })
       end
 
+
       raise RMT::CLI::Error.new(_('There are no repositories marked for mirroring.')) if Repository.only_mirroring_enabled.empty?
 
       # The set of repositories to be mirrored can change while the command is
@@ -19,7 +20,7 @@ def all
       # could lead to different Repositories sets where it's used.
       mirrored_repo_ids = []
       until (repos = Repository.only_mirroring_enabled.where.not(id: mirrored_repo_ids).load).empty?
-        mirror_repos!(repos)
+        mirror_repositories!(repos)
         mirrored_repo_ids.concat(repos.pluck(:id))
       end
 
@@ -39,11 +40,11 @@ def repository(*ids)
       repos = ids.map do |id|
         repo = Repository.find_by(friendly_id: id)
         errored_repos_id << id if options[:do_not_raise_unpublished] && repo.nil?
-        errors << _('Repository with ID %{repo_id} not found') % { repo_id: id } if repo.nil?
+        errors << (_('Repository with ID %{repo_id} not found') % { repo_id: id }) if repo.nil?
         repo
       end
 
-      mirror_repos!(repos)
+      mirror_repositories!(repos)
       finish_execution
     end
   end
@@ -57,29 +58,29 @@ def product(*targets)
       repos = []
       targets.each do |target|
         products = Product.get_by_target!(target)
-        errors << _('Product for target %{target} not found') % { target: target } if products.empty?
+        errors << (_('Product for target %{target} not found') % { target: target }) if products.empty?
         products.each do |product|
           product_repos = product.repositories.only_mirroring_enabled
           if product_repos.empty?
-            errors << _('Product %{target} has no repositories enabled') % { target: target }
+            errors << (_('Product %{target} has no repositories enabled') % { target: target })
             errored_products_id << target if options[:do_not_raise_unpublished]
           end
           repos += product_repos.to_a
         end
       rescue ActiveRecord::RecordNotFound
-        errors << _('Product with ID %{target} not found') % { target: target }
+        errors << (_('Product with ID %{target} not found') % { target: target })
         errored_products_id << target if options[:do_not_raise_unpublished]
       end
 
-      mirror_repos!(repos)
+      mirror_repositories!(repos)
       finish_execution
     end
   end
 
-  protected
+  private
 
-  def mirror
-    @mirror ||= RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?)
+  def suma_product_tree
+    RMT::Mirror::SumaProductTree.new(logger: logger, mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR)
   end
 
   def errors
@@ -123,24 +124,27 @@ def in_alpha_or_beta?
     true
   end
 
-  def mirror_repos!(repos)
+  def mirror_repositories!(repos)
     repos.compact.each do |repo|
       unless repo.mirroring_enabled
-        errors << _('Mirroring of repository with ID %{repo_id} is not enabled') % { repo_id: repo.friendly_id }
+        errors << (_('Mirroring of repository with ID %{repo_id} is not enabled') % { repo_id: repo.friendly_id })
         next
       end
 
-      mirror.mirror(
-        repository_url: repo.external_url,
-        local_path: Repository.make_local_path(repo.external_url),
-        auth_token: repo.auth_token,
-        repo_name: repo.name
-      )
+      configuration = {
+        repository: repo,
+        logger: logger,
+        mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR,
+        mirror_sources: RMT::Config.mirror_src_files?,
+        is_airgapped: false
+      }
+
+      RMT::Mirror.new(**configuration).mirror_now
       repo.refresh_timestamp!
     rescue RMT::Mirror::Exception => e
-      errors << _("Repository '%{repo_name}' (%{repo_id}): %{error_message}") % {
+      errors << (_("Repository '%{repo_name}' (%{repo_id}): %{error_message}") % {
         repo_id: repo.friendly_id, repo_name: repo.name, error_message: e.message
-      }
+      })
       errored_repos_id << repo.id if options[:do_not_raise_unpublished]
     end
   end

lib/rmt/deduplicator.rb

@@ -16,7 +16,7 @@ def deduplicate(target_file)
     make_file_dir(target_file.local_path)
     source_file_path = source_files.first.local_path
 
-    if RMT::Config.deduplication_by_hardlink? && !airgap_mode
+    if RMT::Config.deduplication_by_hardlink? && !is_airgapped
       hardlink(source_file_path, target_file.local_path)
       logger.info("← #{File.basename(target_file.local_path)}")
     else
@@ -25,7 +25,7 @@ def deduplicate(target_file)
     end
 
     # we don't want to track airgap files in our database
-    unless airgap_mode
+    unless is_airgapped
       DownloadedFile.track_file(checksum: target_file.checksum,
                                 checksum_type: target_file.checksum_type,
                                 local_path: target_file.local_path,