diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index be8aaf3dc..fceb878eb 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -16,7 +16,7 @@ Layout/ArgumentAlignment: - 'app/models/product.rb' - 'engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb' - 'spec/factories/products.rb' - - 'spec/lib/rmt/mirror/repomd_spec.rb' + - 'spec/lib/rmt/mirror_spec.rb' - 'spec/models/migration_engine_spec.rb' # Offense count: 3 @@ -40,7 +40,7 @@ Layout/EmptyLineAfterGuardClause: - 'lib/rmt/cli/repos_custom.rb' - 'lib/rmt/cli/smt_importer.rb' - 'lib/rmt/fiber_request.rb' - - 'spec/lib/rmt/mirror/repomd_spec.rb' + - 'spec/lib/rmt/mirror_spec.rb' # Offense count: 2 # Cop supports --auto-correct. @@ -241,7 +241,7 @@ Performance/RegexpMatch: - 'engines/registration_sharing/lib/registration_sharing.rb' - 'engines/strict_authentication/app/controllers/strict_authentication/authentication_controller.rb' - 'lib/rmt/cli/base.rb' - - 'spec/lib/rmt/mirror/repomd_spec.rb' + - 'spec/lib/rmt/mirror_spec.rb' # Offense count: 4 # Cop supports --auto-correct. @@ -249,7 +249,7 @@ Performance/RegexpMatch: Performance/StringInclude: Exclude: - 'engines/strict_authentication/app/controllers/strict_authentication/authentication_controller.rb' - - 'spec/lib/rmt/mirror/repomd_spec.rb' + - 'spec/lib/rmt/mirror_spec.rb' # Offense count: 167 # Configuration parameters: Prefixes. @@ -279,7 +279,7 @@ RSpec/EmptyLineAfterHook: - 'engines/strict_authentication/spec/requests/services_controller_spec.rb' - 'engines/strict_authentication/spec/requests/strict_authentication/authentication_controller_spec.rb' - 'spec/lib/rmt/config_spec.rb' - - 'spec/lib/rmt/mirror/repomd_spec.rb' + - 'spec/lib/rmt/mirror_spec.rb' - 'spec/models/product_spec.rb' - 'spec/requests/api/connect/v3/subscriptions/systems_controller_spec.rb' - 'spec/requests/api/connect/v3/systems/activations_controller_spec.rb' @@ -321,7 +321,7 @@ RSpec/ExpectInHook: - 'spec/lib/rmt/cli/main_spec.rb' - 'spec/lib/rmt/cli/repos_custom_spec.rb' - 'spec/lib/rmt/cli/repos_spec.rb' - - 'spec/lib/rmt/mirror/repomd_spec.rb' + - 'spec/lib/rmt/mirror_spec.rb' - 'spec/lib/rmt/scc_spec.rb' # Offense count: 3 @@ -362,7 +362,7 @@ RSpec/ImplicitSubject: # Configuration parameters: AssignmentOnly. RSpec/InstanceVariable: Exclude: - - 'spec/lib/rmt/mirror/repomd_spec.rb' + - 'spec/lib/rmt/mirror_spec.rb' # Offense count: 1 RSpec/IteratedExpectation: @@ -430,7 +430,7 @@ RSpec/Rails/HttpStatus: RSpec/ReceiveCounts: Exclude: - 'spec/lib/rmt/lockfile_spec.rb' - - 'spec/lib/rmt/mirror/repomd_spec.rb' + - 'spec/lib/rmt/mirror_spec.rb' - 'spec/suse/connect/api_spec.rb' # Offense count: 3 @@ -454,7 +454,7 @@ RSpec/StubbedMock: - 'spec/lib/rmt/cli/main_spec.rb' - 'spec/lib/rmt/cli/smt_importer_spec.rb' - 'spec/lib/rmt/cli/systems_spec.rb' - - 'spec/lib/rmt/mirror/repomd_spec.rb' + - 'spec/lib/rmt/mirror_spec.rb' - 'spec/lib/rmt/scc_spec.rb' - 'spec/support/shared_examples/cli.rb' - 'spec/suse/connect/api_spec.rb' diff --git a/lib/rmt/cli/export.rb b/lib/rmt/cli/export.rb index fc515cd28..395892252 100644 --- a/lib/rmt/cli/export.rb +++ b/lib/rmt/cli/export.rb @@ -30,7 +30,7 @@ def settings(path) def repos(path) path = needs_path(path, writable: true) - mirror = RMT::Mirror::Repomd.new(mirroring_base_dir: path, logger: logger, airgap_mode: true) + mirror = RMT::Mirror.new(mirroring_base_dir: path, logger: logger, airgap_mode: true) begin mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/') diff --git a/lib/rmt/cli/import.rb b/lib/rmt/cli/import.rb index 99cd09d73..b065b79d7 100644 --- a/lib/rmt/cli/import.rb +++ b/lib/rmt/cli/import.rb @@ -13,7 +13,7 @@ def repos(path) RMT::Lockfile.lock do path = needs_path(path) - mirror = RMT::Mirror::Repomd.new(logger: logger, airgap_mode: true) + mirror = RMT::Mirror.new(logger: logger, airgap_mode: true) repos_file = File.join(path, 'repos.json') raise RMT::CLI::Error.new(_('%{file} does not exist.') % { file: repos_file }) unless File.exist?(repos_file) diff --git a/lib/rmt/cli/mirror.rb b/lib/rmt/cli/mirror.rb index aa3643b3a..39dbd01fb 100644 --- a/lib/rmt/cli/mirror.rb +++ b/lib/rmt/cli/mirror.rb @@ -79,13 +79,7 @@ def product(*targets) protected def mirror - config = { - logger: logger, - mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR, - mirror_src: RMT::Config.mirror_src_files?, - airgap_mode: false - } - @mirror ||= RMT::Mirror::Repomd.new(**config) + @mirror ||= RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?) end def errors diff --git a/lib/rmt/mirror.rb b/lib/rmt/mirror.rb index 7e0dce51e..6e28510ba 100644 --- a/lib/rmt/mirror.rb +++ b/lib/rmt/mirror.rb @@ -1,38 +1,205 @@ +require 'rmt/downloader' +require 'rmt/gpg' +require 'repomd_parser' +require 'time' + class RMT::Mirror - RPM_FILE_NEEDLE = 'repodata/repomd.xml'.freeze - DEB_FILE_NEEDLE = 'Release'.freeze + class RMT::Mirror::Exception < RuntimeError + end - attr_reader :logger, :base_dir, :mirror_sources, :is_airgapped, :repository + include RMT::Deduplicator + include RMT::FileValidator - def initialize(repository:, base_dir:, logger:, mirror_sources: false, is_airgapped: false) - @repository = repository + def initialize(mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR, logger:, mirror_src: false, airgap_mode: false) + @mirroring_base_dir = mirroring_base_dir @logger = logger - @base_dir = base_dir - @mirror_sources = mirror_sources - @is_airgapped = is_airgapped + @mirror_src = mirror_src + @airgap_mode = airgap_mode + @deep_verify = false + + # don't save files for deduplication when in offline mode + @downloader = RMT::Downloader.new(logger: logger, track_files: !airgap_mode) end - def detect_repository_type - search = { - repomd: File.join(repository.external_url, RPM_FILE_NEEDLE), - debian: File.join(repository.external_url, DEB_FILE_NEEDLE) + def mirror_suma_product_tree(repository_url:) + # we have an inconsistency in how we mirror in offline mode + # in normal mode we mirror in the following way: + # base_dir/repo/... + # however, in offline mode we mirror in the following way + # base_dir/... + # we need this extra step to ensure that we write to the public directory + base_dir = mirroring_base_dir + base_dir = File.expand_path(File.join(mirroring_base_dir, '/../')) if mirroring_base_dir == RMT::DEFAULT_MIRROR_DIR + + repository_dir = File.join(base_dir, '/suma/') + mirroring_paths = { + base_url: URI.join(repository_url), + base_dir: repository_dir, + cache_dir: repository_dir } - search.each do |key, url| - # Current CDN authenticates via a key append to the request path - # e.g. - # https://update.suse.com/SUSE/product/some-product - # becomes - # https://update.suse.com/SUSE/product/some-product?authenication_tokensiduhashasdyashdaysdasud - uri = URI.join(url) - uri.query = @repository.auth_token if @repository.auth_token - - request = RMT::HttpRequest.new(uri, method: :head, followlocation: true) - request.on_success do - return key + logger.info _('Mirroring SUSE Manager product tree to %{dir}') % { dir: repository_dir } + downloader.download_multi([FileReference.new(relative_path: 'product_tree.json', **mirroring_paths)]) + rescue RMT::Downloader::Exception => e + raise RMT::Mirror::Exception.new(_('Could not mirror SUSE Manager product tree with error: %{error}') % { error: e.message }) + end + + def mirror(repository_url:, local_path:, auth_token: nil, repo_name: nil) + repository_dir = File.join(mirroring_base_dir, local_path) + + logger.info _('Mirroring repository %{repo} to %{dir}') % { repo: repo_name || repository_url, dir: repository_dir } + + create_repository_dir(repository_dir) + temp_licenses_dir = create_temp_dir + # downloading license doesn't require an auth token + mirror_license(repository_dir, repository_url, temp_licenses_dir) + + downloader.auth_token = auth_token + temp_metadata_dir = create_temp_dir + metadata_files = mirror_metadata(repository_dir, repository_url, temp_metadata_dir) + mirror_packages(metadata_files, repository_dir, repository_url) + + replace_directory(temp_licenses_dir, repository_dir.chomp('/') + '.license/') if Dir.exist?(temp_licenses_dir) + replace_directory(File.join(temp_metadata_dir, 'repodata'), File.join(repository_dir, 'repodata')) + ensure + [temp_licenses_dir, temp_metadata_dir].each { |dir| FileUtils.remove_entry(dir, true) } + end + + protected + + attr_reader :airgap_mode, :deep_verify, :downloader, :logger, :mirroring_base_dir, :mirror_src + + def create_repository_dir(repository_dir) + FileUtils.mkpath(repository_dir) unless Dir.exist?(repository_dir) + rescue StandardError => e + raise RMT::Mirror::Exception.new( + _('Could not create local directory %{dir} with error: %{error}') % { dir: repository_dir, error: e.message } + ) + end + + def create_temp_dir + Dir.mktmpdir + rescue StandardError => e + raise RMT::Mirror::Exception.new(_('Could not create a temporary directory: %{error}') % { error: e.message }) + end + + def mirror_metadata(repository_dir, repository_url, temp_metadata_dir) + mirroring_paths = { + base_url: URI.join(repository_url), + base_dir: temp_metadata_dir, + cache_dir: repository_dir + } + + repomd_xml = FileReference.new(relative_path: 'repodata/repomd.xml', **mirroring_paths) + downloader.download_multi([repomd_xml]) + + begin + signature_file = FileReference.new(relative_path: 'repodata/repomd.xml.asc', **mirroring_paths) + key_file = FileReference.new(relative_path: 'repodata/repomd.xml.key', **mirroring_paths) + # mirror repomd.xml.asc first, because there are repos with repomd.xml.asc but without repomd.xml.key + downloader.download_multi([signature_file]) + downloader.download_multi([key_file]) + + RMT::GPG.new( + metadata_file: repomd_xml.local_path, + key_file: key_file.local_path, + signature_file: signature_file.local_path, + logger: logger + ).verify_signature + rescue RMT::Downloader::Exception => e + if (e.http_code == 404) + logger.info(_('Repository metadata signatures are missing')) + else + raise(_('Downloading repo signature/key failed with: %{message}, HTTP code %{http_code}') % { message: e.message, http_code: e.http_code }) end - request.run end - nil + + metadata_files = RepomdParser::RepomdXmlParser.new(repomd_xml.local_path).parse + .map { |reference| FileReference.build_from_metadata(reference, **mirroring_paths) } + + downloader.download_multi(metadata_files.dup) + + metadata_files + rescue StandardError => e + raise RMT::Mirror::Exception.new(_('Error while mirroring metadata: %{error}') % { error: e.message }) + end + + def mirror_license(repository_dir, repository_url, temp_licenses_dir) + mirroring_paths = { + base_url: repository_url.chomp('/') + '.license/', + base_dir: temp_licenses_dir, + cache_dir: repository_dir.chomp('/') + '.license/' + } + + begin + directory_yast = FileReference.new(relative_path: 'directory.yast', **mirroring_paths) + downloader.download_multi([directory_yast]) + rescue RMT::Downloader::Exception + logger.debug("No license directory found for repository '#{repository_url}'") + FileUtils.remove_entry(temp_licenses_dir) # the repository would have an empty licenses directory unless removed + return + end + + license_files = File.readlines(directory_yast.local_path) + .map(&:strip).reject { |item| item == 'directory.yast' } + .map { |relative_path| FileReference.new(relative_path: relative_path, **mirroring_paths) } + downloader.download_multi(license_files) + rescue StandardError => e + raise RMT::Mirror::Exception.new(_('Error while mirroring license files: %{error}') % { error: e.message }) + end + + def mirror_packages(metadata_files, repository_dir, repository_url) + package_references = parse_packages_metadata(metadata_files) + + package_file_references = package_references.map do |reference| + FileReference.build_from_metadata(reference, + base_dir: repository_dir, + base_url: repository_url) + end + + failed_downloads = download_package_files(package_file_references) + + raise _('Failed to download %{failed_count} files') % { failed_count: failed_downloads.size } unless failed_downloads.empty? + rescue StandardError => e + raise RMT::Mirror::Exception.new(_('Error while mirroring packages: %{error}') % { error: e.message }) + end + + def parse_packages_metadata(metadata_references) + xml_parsers = { deltainfo: RepomdParser::DeltainfoXmlParser, + primary: RepomdParser::PrimaryXmlParser } + + metadata_references + .map { |file| xml_parsers[file.type]&.new(file.local_path) }.compact + .map(&:parse).flatten + end + + def download_package_files(file_references) + files_to_download = file_references.select { |file| need_to_download?(file) } + return [] if files_to_download.empty? + + downloader.download_multi(files_to_download, ignore_errors: true) + end + + def need_to_download?(file) + return false if file.arch == 'src' && !mirror_src + return false if validate_local_file(file) + return false if deduplicate(file) + + true + end + + def replace_directory(source_dir, destination_dir) + old_directory = File.join(File.dirname(destination_dir), '.old_' + File.basename(destination_dir)) + + FileUtils.remove_entry(old_directory) if Dir.exist?(old_directory) + FileUtils.mv(destination_dir, old_directory) if Dir.exist?(destination_dir) + FileUtils.mv(source_dir, destination_dir, force: true) + FileUtils.chmod(0o755, destination_dir) + rescue StandardError => e + raise RMT::Mirror::Exception.new(_('Error while moving directory %{src} to %{dest}: %{error}') % { + src: source_dir, + dest: destination_dir, + error: e.message + }) end end diff --git a/lib/rmt/mirror/exception.rb b/lib/rmt/mirror/exception.rb deleted file mode 100644 index 89a9c28cd..000000000 --- a/lib/rmt/mirror/exception.rb +++ /dev/null @@ -1,2 +0,0 @@ -class RMT::Mirror::Exception < RuntimeError -end diff --git a/lib/rmt/mirror/repomd.rb b/lib/rmt/mirror/repomd.rb deleted file mode 100644 index 199511de4..000000000 --- a/lib/rmt/mirror/repomd.rb +++ /dev/null @@ -1,202 +0,0 @@ -require 'rmt/downloader' -require 'rmt/gpg' -require 'repomd_parser' -require 'time' - -class RMT::Mirror::Repomd - include RMT::Deduplicator - include RMT::FileValidator - - def initialize(logger:, mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR, mirror_src: false, airgap_mode: false) - @mirroring_base_dir = mirroring_base_dir - @logger = logger - @mirror_src = mirror_src - @airgap_mode = airgap_mode - @deep_verify = false - - # don't save files for deduplication when in offline mode - @downloader = RMT::Downloader.new(logger: logger, track_files: !airgap_mode) - end - - def mirror_suma_product_tree(repository_url:) - # we have an inconsistency in how we mirror in offline mode - # in normal mode we mirror in the following way: - # base_dir/repo/... - # however, in offline mode we mirror in the following way - # base_dir/... - # we need this extra step to ensure that we write to the public directory - base_dir = mirroring_base_dir - base_dir = File.expand_path(File.join(mirroring_base_dir, '/../')) if mirroring_base_dir == RMT::DEFAULT_MIRROR_DIR - - repository_dir = File.join(base_dir, '/suma/') - mirroring_paths = { - base_url: URI.join(repository_url), - base_dir: repository_dir, - cache_dir: repository_dir - } - - logger.info _('Mirroring SUSE Manager product tree to %{dir}') % { dir: repository_dir } - downloader.download_multi([RMT::Mirror::FileReference.new(relative_path: 'product_tree.json', **mirroring_paths)]) - rescue RMT::Downloader::Exception => e - raise RMT::Mirror::Exception.new(_('Could not mirror SUSE Manager product tree with error: %{error}') % { error: e.message }) - end - - def mirror(repository_url:, local_path:, auth_token: nil, repo_name: nil) - repository_dir = File.join(mirroring_base_dir, local_path) - - logger.info _('Mirroring repository %{repo} to %{dir}') % { repo: repo_name || repository_url, dir: repository_dir } - - create_repository_dir(repository_dir) - temp_licenses_dir = create_temp_dir - # downloading license doesn't require an auth token - mirror_license(repository_dir, repository_url, temp_licenses_dir) - - downloader.auth_token = auth_token - temp_metadata_dir = create_temp_dir - metadata_files = mirror_metadata(repository_dir, repository_url, temp_metadata_dir) - mirror_packages(metadata_files, repository_dir, repository_url) - - replace_directory(temp_licenses_dir, repository_dir.chomp('/') + '.license/') if Dir.exist?(temp_licenses_dir) - replace_directory(File.join(temp_metadata_dir, 'repodata'), File.join(repository_dir, 'repodata')) - ensure - [temp_licenses_dir, temp_metadata_dir].each { |dir| FileUtils.remove_entry(dir, true) } - end - - protected - - attr_reader :airgap_mode, :deep_verify, :downloader, :logger, :mirroring_base_dir, :mirror_src - - def create_repository_dir(repository_dir) - FileUtils.mkpath(repository_dir) unless Dir.exist?(repository_dir) - rescue StandardError => e - raise RMT::Mirror::Exception.new( - _('Could not create local directory %{dir} with error: %{error}') % { dir: repository_dir, error: e.message } - ) - end - - def create_temp_dir - Dir.mktmpdir - rescue StandardError => e - raise RMT::Mirror::Exception.new(_('Could not create a temporary directory: %{error}') % { error: e.message }) - end - - def mirror_metadata(repository_dir, repository_url, temp_metadata_dir) - mirroring_paths = { - base_url: URI.join(repository_url), - base_dir: temp_metadata_dir, - cache_dir: repository_dir - } - - repomd_xml = RMT::Mirror::FileReference.new(relative_path: 'repodata/repomd.xml', **mirroring_paths) - downloader.download_multi([repomd_xml]) - - begin - signature_file = RMT::Mirror::FileReference.new(relative_path: 'repodata/repomd.xml.asc', **mirroring_paths) - key_file = RMT::Mirror::FileReference.new(relative_path: 'repodata/repomd.xml.key', **mirroring_paths) - # mirror repomd.xml.asc first, because there are repos with repomd.xml.asc but without repomd.xml.key - downloader.download_multi([signature_file]) - downloader.download_multi([key_file]) - - RMT::GPG.new( - metadata_file: repomd_xml.local_path, - key_file: key_file.local_path, - signature_file: signature_file.local_path, - logger: logger - ).verify_signature - rescue RMT::Downloader::Exception => e - if (e.http_code == 404) - logger.info(_('Repository metadata signatures are missing')) - else - raise(_('Downloading repo signature/key failed with: %{message}, HTTP code %{http_code}') % { message: e.message, http_code: e.http_code }) - end - end - - metadata_files = RepomdParser::RepomdXmlParser.new(repomd_xml.local_path).parse - .map { |reference| RMT::Mirror::FileReference.build_from_metadata(reference, **mirroring_paths) } - - downloader.download_multi(metadata_files.dup) - - metadata_files - rescue StandardError => e - raise RMT::Mirror::Exception.new(_('Error while mirroring metadata: %{error}') % { error: e.message }) - end - - def mirror_license(repository_dir, repository_url, temp_licenses_dir) - mirroring_paths = { - base_url: repository_url.chomp('/') + '.license/', - base_dir: temp_licenses_dir, - cache_dir: repository_dir.chomp('/') + '.license/' - } - - begin - directory_yast = RMT::Mirror::FileReference.new(relative_path: 'directory.yast', **mirroring_paths) - downloader.download_multi([directory_yast]) - rescue RMT::Downloader::Exception - logger.debug("No license directory found for repository '#{repository_url}'") - FileUtils.remove_entry(temp_licenses_dir) # the repository would have an empty licenses directory unless removed - return - end - - license_files = File.readlines(directory_yast.local_path) - .map(&:strip).reject { |item| item == 'directory.yast' } - .map { |relative_path| RMT::Mirror::FileReference.new(relative_path: relative_path, **mirroring_paths) } - downloader.download_multi(license_files) - rescue StandardError => e - raise RMT::Mirror::Exception.new(_('Error while mirroring license files: %{error}') % { error: e.message }) - end - - def mirror_packages(metadata_files, repository_dir, repository_url) - package_references = parse_packages_metadata(metadata_files) - - package_file_references = package_references.map do |reference| - RMT::Mirror::FileReference.build_from_metadata(reference, - base_dir: repository_dir, - base_url: repository_url) - end - - failed_downloads = download_package_files(package_file_references) - - raise _('Failed to download %{failed_count} files') % { failed_count: failed_downloads.size } unless failed_downloads.empty? - rescue StandardError => e - raise RMT::Mirror::Exception.new(_('Error while mirroring packages: %{error}') % { error: e.message }) - end - - def parse_packages_metadata(metadata_references) - xml_parsers = { deltainfo: RepomdParser::DeltainfoXmlParser, - primary: RepomdParser::PrimaryXmlParser } - - metadata_references - .map { |file| xml_parsers[file.type]&.new(file.local_path) }.compact - .map(&:parse).flatten - end - - def download_package_files(file_references) - files_to_download = file_references.select { |file| need_to_download?(file) } - return [] if files_to_download.empty? - - downloader.download_multi(files_to_download, ignore_errors: true) - end - - def need_to_download?(file) - return false if file.arch == 'src' && !mirror_src - return false if validate_local_file(file) - return false if deduplicate(file) - - true - end - - def replace_directory(source_dir, destination_dir) - old_directory = File.join(File.dirname(destination_dir), '.old_' + File.basename(destination_dir)) - - FileUtils.remove_entry(old_directory) if Dir.exist?(old_directory) - FileUtils.mv(destination_dir, old_directory) if Dir.exist?(destination_dir) - FileUtils.mv(source_dir, destination_dir, force: true) - FileUtils.chmod(0o755, destination_dir) - rescue StandardError => e - raise RMT::Mirror::Exception.new(_('Error while moving directory %{src} to %{dest}: %{error}') % { - src: source_dir, - dest: destination_dir, - error: e.message - }) - end -end diff --git a/spec/lib/rmt/cli/export_spec.rb b/spec/lib/rmt/cli/export_spec.rb index 95a7520ea..91b1cfd49 100644 --- a/spec/lib/rmt/cli/export_spec.rb +++ b/spec/lib/rmt/cli/export_spec.rb @@ -67,7 +67,7 @@ include_examples 'handles non-writable path' let(:command) { described_class.start(['repos', path]) } - let(:mirror_double) { instance_double('RMT::Mirror::Repomd') } + let(:mirror_double) { instance_double('RMT::Mirror') } let(:repo_settings) do [ { url: 'http://foo.bar/repo1', auth_token: 'foobar' }, @@ -77,8 +77,8 @@ context 'suma product tree mirror with exception' do it 'outputs exception message' do - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree).and_raise(RMT::Mirror::Exception, 'black mirror') - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror).twice + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree).and_raise(RMT::Mirror::Exception, 'black mirror') + expect_any_instance_of(RMT::Mirror).to receive(:mirror).twice FileUtils.mkdir_p path File.write("#{path}/repos.json", repo_settings.to_json) @@ -92,7 +92,7 @@ it 'outputs a warning' do FileUtils.mkdir_p path - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) expect { command }.to raise_error(SystemExit).and(output("#{File.join(path, 'repos.json')} does not exist.\n").to_stderr) end end @@ -101,7 +101,7 @@ let(:path) { '/mnt/usb/../usb' } before do - expect(RMT::Mirror::Repomd).to receive(:new).with( + expect(RMT::Mirror).to receive(:new).with( mirroring_base_dir: '/mnt/usb', logger: instance_of(RMT::Logger), airgap_mode: true @@ -131,7 +131,7 @@ context 'with valid repo ids' do before do - expect(RMT::Mirror::Repomd).to receive(:new).with( + expect(RMT::Mirror).to receive(:new).with( mirroring_base_dir: path, logger: instance_of(RMT::Logger), airgap_mode: true diff --git a/spec/lib/rmt/cli/import_spec.rb b/spec/lib/rmt/cli/import_spec.rb index e3fe88f00..ed5944117 100644 --- a/spec/lib/rmt/cli/import_spec.rb +++ b/spec/lib/rmt/cli/import_spec.rb @@ -47,7 +47,7 @@ let(:repo2) { create :repository, mirroring_enabled: true } let(:repo1_local_path) { repo_url_to_local_path(path, repo1.external_url) } let(:repo2_local_path) { repo_url_to_local_path(path, repo2.external_url) } - let(:mirror_double) { instance_double RMT::Mirror::Repomd } + let(:mirror_double) { instance_double RMT::Mirror } let(:repo_settings) do [ { url: repo1.external_url, auth_token: repo1.auth_token.to_s }, @@ -59,7 +59,7 @@ FileUtils.mkdir_p path File.write("#{path}/repos.json", repo_settings.to_json) - expect(RMT::Mirror::Repomd).to receive(:new).with( + expect(RMT::Mirror).to receive(:new).with( logger: instance_of(RMT::Logger), airgap_mode: true ).and_return(mirror_double) @@ -91,7 +91,7 @@ FileUtils.mkdir_p path File.write("#{path}/repos.json", repo_settings.to_json) - expect(RMT::Mirror::Repomd).to receive(:new).with( + expect(RMT::Mirror).to receive(:new).with( logger: instance_of(RMT::Logger), airgap_mode: true ).and_return(mirror_double) @@ -124,8 +124,8 @@ context 'suma product tree mirror with exception' do it 'outputs exception message' do - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree).and_raise(RMT::Mirror::Exception, 'black mirror') - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror).twice + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree).and_raise(RMT::Mirror::Exception, 'black mirror') + expect_any_instance_of(RMT::Mirror).to receive(:mirror).twice FileUtils.mkdir_p path File.write("#{path}/repos.json", repo_settings.to_json) @@ -148,7 +148,7 @@ FileUtils.mkdir_p path File.write("#{path}/repos.json", repo_settings.to_json) - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) expect { command }.to output(/repository by URL #{missing_repo_url} does not exist in database/).to_stderr.and output('').to_stdout end end @@ -165,7 +165,7 @@ FileUtils.mkdir_p path File.write("#{path}/repos.json", repo_settings.to_json) - expect(RMT::Mirror::Repomd).to receive(:new).with( + expect(RMT::Mirror).to receive(:new).with( logger: instance_of(RMT::Logger), airgap_mode: true ).and_return(mirror_double) diff --git a/spec/lib/rmt/cli/mirror_spec.rb b/spec/lib/rmt/cli/mirror_spec.rb index a49882ae6..cb1f90ac8 100644 --- a/spec/lib/rmt/cli/mirror_spec.rb +++ b/spec/lib/rmt/cli/mirror_spec.rb @@ -26,10 +26,10 @@ let(:error_messages) { "Mirroring SUMA product tree failed: #{suma_error}." } it 'handles the exception and raises an error after mirroring all repos' do - expect_any_instance_of(RMT::Mirror::Repomd) + expect_any_instance_of(RMT::Mirror) .to receive(:mirror_suma_product_tree) .and_raise(RMT::Mirror::Exception, suma_error) - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror) + expect_any_instance_of(RMT::Mirror).to receive(:mirror) expect { command } .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } @@ -44,8 +44,8 @@ end it 'raises an error' do - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree) - expect_any_instance_of(RMT::Mirror::Repomd).not_to receive(:mirror) + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).not_to receive(:mirror) expect { command } .to raise_error(SystemExit) @@ -58,8 +58,8 @@ let!(:repository) { create :repository, :with_products, mirroring_enabled: true } it 'updates repository mirroring timestamp' do - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree) - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror) + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).to receive(:mirror) Timecop.freeze(Time.utc(2018)) do expect { command } @@ -70,7 +70,7 @@ context 'with exceptions during mirroring' do before do - allow_any_instance_of(RMT::Mirror::Repomd) + allow_any_instance_of(RMT::Mirror) .to receive(:mirror) .and_raise(RMT::Mirror::Exception, mirroring_error) end @@ -79,7 +79,7 @@ let(:error_messages) { /Repository '#{repository.name}' \(#{repository.friendly_id}\): #{mirroring_error}\./ } it 'raises an error' do - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) expect { command } .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } @@ -91,7 +91,7 @@ context 'with repositories in alpha or beta stage' do before do - allow_any_instance_of(RMT::Mirror::Repomd) + allow_any_instance_of(RMT::Mirror) .to receive(:mirror) .and_raise(RMT::Mirror::Exception, mirroring_error) end @@ -115,7 +115,7 @@ let(:argv) { ['all', '--do-not-raise-unpublished'] } it 'log the warning and does not raise an error' do - allow_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree) + allow_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) expect { command }.to output(error_log).to_stdout end end @@ -126,8 +126,8 @@ let!(:additional_repository) { create :repository, :with_products, mirroring_enabled: false } it 'mirrors additional repositories' do - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree) - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror).with( + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( repository_url: repository.external_url, local_path: anything, repo_name: anything, @@ -138,7 +138,7 @@ additional_repository.save! end - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror).with( + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( repository_url: additional_repository.external_url, local_path: anything, repo_name: anything, @@ -156,8 +156,8 @@ let(:error_messages) { /Repository '#{repository.name}' \(#{repository.friendly_id}\): #{mirroring_error}\./ } it 'handles exceptions and mirrors additional repositories' do - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree) - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror).with( + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( repository_url: repository.external_url, local_path: anything, repo_name: anything, @@ -169,7 +169,7 @@ raise(RMT::Mirror::Exception, mirroring_error) end - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror).with( + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( repository_url: additional_repository.external_url, local_path: anything, repo_name: anything, @@ -194,7 +194,7 @@ let(:argv) { ['repository', repository.friendly_id] } it 'mirrors the repository' do - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror).with( + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( repository_url: repository.external_url, local_path: anything, repo_name: anything, @@ -212,7 +212,7 @@ let(:error_messages) { /Repository '#{repository.name}' \(#{repository.friendly_id}\): #{mirroring_error}\./ } it 'handles the exception and raises an error after mirroring all repos' do - expect_any_instance_of(RMT::Mirror::Repomd) + expect_any_instance_of(RMT::Mirror) .to receive(:mirror).at_least(:once) .and_raise(RMT::Mirror::Exception, mirroring_error) @@ -271,7 +271,7 @@ it 'mirrors repos' do product.repositories.each do |repo| - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror).with( + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( repository_url: repo.external_url, local_path: anything, repo_name: anything, @@ -289,7 +289,7 @@ it 'mirrors repos' do product.repositories.each do |repo| - expect_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror).with( + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( repository_url: repo.external_url, local_path: anything, repo_name: anything, @@ -312,7 +312,7 @@ end it 'handles the exception and raises an error after mirroring all repos' do - expect_any_instance_of(RMT::Mirror::Repomd) + expect_any_instance_of(RMT::Mirror) .to receive(:mirror).at_least(:once) .and_raise(RMT::Mirror::Exception, mirroring_error) @@ -381,7 +381,7 @@ context 'mirror product using --do-not-raise-unpublished flag' do it 'log the warning and does not raise an error' do - allow_any_instance_of(RMT::Mirror::Repomd).to receive(:mirror_suma_product_tree) + allow_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) expect { command }.to output(error_log).to_stdout end end diff --git a/spec/lib/rmt/mirror/repomd_spec.rb b/spec/lib/rmt/mirror/repomd_spec.rb deleted file mode 100644 index 2def94af4..000000000 --- a/spec/lib/rmt/mirror/repomd_spec.rb +++ /dev/null @@ -1,853 +0,0 @@ -require 'rails_helper' - -RSpec.describe RMT::Mirror::Repomd do - RSpec::Matchers.define :file_reference_containing_path do |expected| - match do |actual| - actual.local_path.include?(expected) - end - - failure_message do |actual| - "expected that file path #{actual.local_path} would contain #{expected}" - end - end - - let(:logger) { RMT::Logger.new('/dev/null') } - - describe '#mirror_suma_product_tree' do - subject(:command) { rmt_mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/') } - - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: logger, - mirror_src: false - ) - end - - around do |example| - @tmp_dir = Dir.mktmpdir('rmt') - example.run - FileUtils.remove_entry(@tmp_dir) - end - - context 'all is well', vcr: { cassette_name: 'mirroring_suma_product_tree' } do - before do - expect(logger).to receive(:info).with(/Mirroring SUSE Manager product tree to/).once - expect(logger).to receive(:info).with(/↓ product_tree.json/).once - end - - it 'downloads the suma product tree' do - command - content = File.read(File.join(@tmp_dir, 'suma/product_tree.json')) - expect(Digest::SHA256.hexdigest(content)).to eq('7486026e9c1181affae5b21c9aa64637aa682fcdeacb099e213f0e8c7e86d85d') - end - end - - context 'with download exception' do - before do - expect_any_instance_of(RMT::Downloader).to receive(:download_multi).and_raise(RMT::Downloader::Exception, "418 - I'm a teapot") - end - - it 'raises mirroring exception' do - expect { command }.to raise_error(RMT::Mirror::Exception, "Could not mirror SUSE Manager product tree with error: 418 - I'm a teapot") - end - end - end - - describe '#mirror' do - around do |example| - @tmp_dir = Dir.mktmpdir('rmt') - example.run - FileUtils.remove_entry(@tmp_dir) - end - - before do - allow_any_instance_of(RMT::GPG).to receive(:verify_signature) - end - - after do - Dir.glob(File.join(Dir.tmpdir, 'rmt_mirror_*', '**')) - .each { |tmpdir| FileUtils.remove_entry(tmpdir, true) } - end - - context 'without auth_token', vcr: { cassette_name: 'mirroring' } do - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: logger, - mirror_src: false - ) - end - - let(:mirror_params) do - { - repository_url: 'http://localhost/dummy_repo/', - local_path: '/dummy_repo' - } - end - - before do - rmt_mirror.mirror(**mirror_params) - end - - it 'downloads rpm files' do - rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_repo')).select { |entry| entry =~ /\.rpm$/ } - expect(rpm_entries.length).to eq(4) - end - - it 'downloads drpm files' do - rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_repo')).select { |entry| entry =~ /\.drpm$/ } - expect(rpm_entries.length).to eq(2) - end - end - - context 'importing local repo' do - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: logger, - mirror_src: false - ) - end - - let(:mirror_params) do - { - repository_url: URI.join('file://', File.expand_path(file_fixture('dummy_repo'))).to_s + '/', - local_path: Repository.make_local_path('dummy_repo/'), - repo_name: 'dummy_repo' - } - end - - before do - rmt_mirror.mirror(**mirror_params) - end - - it 'copies rpm files' do - expect(Dir.entries(File.join(@tmp_dir, 'dummy_repo'))).to match_array(Dir.entries(file_fixture('dummy_repo'))) - end - - it 'copies metadata' do - expect(Dir.entries(File.join(@tmp_dir, 'dummy_repo/repodata'))).to match_array(Dir.entries(file_fixture('dummy_repo/repodata'))) - end - end - - context 'without auth_token and with source packages', vcr: { cassette_name: 'mirroring_with_src' } do - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: logger, - mirror_src: mirror_src - ) - end - - let(:mirror_params) do - { - repository_url: 'http://localhost/dummy_repo_with_src/', - local_path: '/dummy_repo' - } - end - - before do - rmt_mirror.mirror(**mirror_params) - end - - context 'when mirror_src is false' do - let(:mirror_src) { false } - - it 'downloads rpm files' do - rpm_entries = Dir.glob(File.join(@tmp_dir, 'dummy_repo', '**', '*.rpm')) - expect(rpm_entries.length).to eq(2) - end - - it 'downloads drpm files' do - rpm_entries = Dir.glob(File.join(@tmp_dir, 'dummy_repo', '**', '*.drpm')) - expect(rpm_entries.length).to eq(1) - end - end - - context 'when mirror_src is true' do - let(:mirror_src) { true } - - it 'downloads rpm files' do - rpm_entries = Dir.glob(File.join(@tmp_dir, 'dummy_repo', '**', '*.rpm')) - expect(rpm_entries.length).to eq(4) - end - - it 'downloads drpm files' do - rpm_entries = Dir.glob(File.join(@tmp_dir, 'dummy_repo', '**', '*.drpm')) - expect(rpm_entries.length).to eq(1) - end - end - end - - context 'with auth_token', vcr: { cassette_name: 'mirroring_with_auth_token' } do - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: logger, - mirror_src: false - ) - end - - let(:mirror_params) do - { - repository_url: 'http://localhost/dummy_repo/', - local_path: '/dummy_repo', - auth_token: 'repo_auth_token' - } - end - - before do - expect(logger).to receive(:info).with(/Mirroring repository/).once - expect(logger).to receive(:info).with('Repository metadata signatures are missing').once - expect(logger).to receive(:info).with(/↓/).at_least(:once) - rmt_mirror.mirror(**mirror_params) - end - - it 'downloads rpm files' do - rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_repo')).select { |entry| entry =~ /\.rpm$/ } - expect(rpm_entries.length).to eq(4) - end - - it 'downloads drpm files' do - rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_repo')).select { |entry| entry =~ /\.drpm$/ } - expect(rpm_entries.length).to eq(2) - end - end - - context 'product with license and signatures', vcr: { cassette_name: 'mirroring_product' } do - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: logger, - mirror_src: false - ) - end - - let(:mirror_params) do - { - repository_url: 'http://localhost/dummy_product/product/', - local_path: '/dummy_product/product/', - auth_token: 'repo_auth_token' - } - end - - before do - expect(logger).to receive(:info).with(/Mirroring repository/).once - expect(logger).to receive(:info).with(/↓/).at_least(:once) - rmt_mirror.mirror(**mirror_params) - end - - it 'downloads rpm files' do - rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_product/product/')).select { |entry| entry =~ /\.rpm$/ } - expect(rpm_entries.length).to eq(4) - end - - it 'downloads drpm files' do - rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_product/product/')).select { |entry| entry =~ /\.drpm$/ } - expect(rpm_entries.length).to eq(2) - end - - it 'downloads repomd.xml signatures' do - ['repomd.xml.key', 'repomd.xml.asc'].each do |file| - expect(File.size(File.join(@tmp_dir, 'dummy_product/product/repodata/', file))).to be > 0 - end - end - - it 'downloads product license' do - ['directory.yast', 'license.txt', 'license.de.txt', 'license.ru.txt'].each do |file| - expect(File.size(File.join(@tmp_dir, 'dummy_product/product.license/', file))).to be > 0 - end - end - end - - context 'when an error occurs' do - let(:mirroring_dir) { @tmp_dir } - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: mirroring_dir, - logger: logger, - mirror_src: false - ) - end - - let(:mirror_params) do - { - repository_url: 'http://localhost/dummy_product/product/', - local_path: '/dummy_product/product/', - auth_token: 'repo_auth_token' - } - end - - context 'when mirroring_base_dir is not writable' do - let(:mirroring_dir) { '/non/existent/path' } - - it 'raises exception', vcr: { cassette_name: 'mirroring_product' } do - expect { rmt_mirror.mirror(**mirror_params) }.to raise_error(RMT::Mirror::Exception) - end - end - - context "when can't create tmp dir", vcr: { cassette_name: 'mirroring_product' } do - before { allow(Dir).to receive(:mktmpdir).and_raise('mktmpdir exception') } - - it 'handles the exception' do - expect { rmt_mirror.mirror(**mirror_params) }.to raise_error(RMT::Mirror::Exception) - end - end - - context "when can't download metadata", vcr: { cassette_name: 'mirroring_product' } do - before do - allow_any_instance_of(RMT::Downloader).to receive(:download_multi).and_call_original - expect_any_instance_of(RMT::Downloader) - .to receive(:download_multi) - .with([file_reference_containing_path('repodata/repomd.xml')]) - .and_raise(RMT::Downloader::Exception, "418 - I'm a teapot") - end - - it 'handles RMT::Downloader::Exception' do - expect { rmt_mirror.mirror(**mirror_params) } - .to raise_error(RMT::Mirror::Exception, "Error while mirroring metadata: 418 - I'm a teapot") - end - end - - context "when there's no licenses to download", vcr: { cassette_name: 'mirroring' } do - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: logger, - mirror_src: false - ) - end - - let(:mirror_params) do - { - repository_url: 'http://localhost/dummy_repo/', - local_path: '/dummy_product/product/' - } - end - - it 'does not error out' do - expect { rmt_mirror.mirror(**mirror_params) }.not_to raise_error - end - - it 'does not create a product.licenses directory' do - rmt_mirror.mirror(**mirror_params) - expect(Dir).not_to exist(File.join(@tmp_dir, 'dummy_product', 'product.license')) - end - - it 'removes the temporary licenses directory' do - rmt_mirror.mirror(**mirror_params) - tmp_dir_glob = Dir.glob(File.join(Dir.tmpdir, 'rmt_mirror_*', '**')) - expect(tmp_dir_glob.length).to eq(0) - end - end - - context "when can't download some of the license files" do - before do - allow_any_instance_of(RMT::Downloader).to receive(:download_multi).and_wrap_original do |klass, *args| - raise RMT::Downloader::Exception.new('') if /license/.match?(args[0][0].local_path) - - klass.call(*args) - end - end - - it 'handles RMT::Downloader::Exception', vcr: { cassette_name: 'mirroring_product' } do - expect { rmt_mirror.mirror(**mirror_params) }.to raise_error(RMT::Mirror::Exception, /Error while mirroring license files:/) - end - end - - context "when can't parse metadata", vcr: { cassette_name: 'mirroring_product' } do - before { allow_any_instance_of(RepomdParser::RepomdXmlParser).to receive(:parse).and_raise('Parse error') } - - it 'removes the temporary metadata directory' do - expect { rmt_mirror.mirror(**mirror_params) } - .to raise_error(RMT::Mirror::Exception, 'Error while mirroring metadata: Parse error') - - tmp_dir_glob = Dir.glob(File.join(Dir.tmpdir, 'rmt_mirror_*', '**')) - expect(tmp_dir_glob.length).to eq(0) - end - end - - context 'when Interrupt is raised', vcr: { cassette_name: 'mirroring_product' } do - before { allow_any_instance_of(RepomdParser::RepomdXmlParser).to receive(:parse).and_raise(Interrupt.new) } - - it 'removes the temporary metadata directory' do - expect { rmt_mirror.mirror(**mirror_params) }.to raise_error(Interrupt) - - tmp_dir_glob = Dir.glob(File.join(Dir.tmpdir, 'rmt_mirror_*', '**')) - expect(tmp_dir_glob.length).to eq(0) - end - end - - context "when can't download data", vcr: { cassette_name: 'mirroring_product' } do - it 'handles RMT::Downloader::Exception' do - allow_any_instance_of(RMT::Downloader).to receive(:make_request).and_wrap_original do |klass, *args| - # raise the exception only for the RPMs/DRPMs - raise(RMT::Downloader::Exception, "418 - I'm a teapot") if /rpm$/.match?(args[0].local_path) - - klass.call(*args) - end - - expect do - rmt_mirror.mirror(**mirror_params) - end.to raise_error(RMT::Mirror::Exception, 'Error while mirroring packages: Failed to download 6 files') - end - - it 'handles RMT::ChecksumVerifier::Exception' do - allow_any_instance_of(RMT::Downloader).to receive(:make_request).and_wrap_original do |klass, *args| - # raise the exception only for the RPMs/DRPMs - raise(RMT::ChecksumVerifier::Exception, "Checksum doesn't match") if /rpm$/.match?(args[0].local_path) - - klass.call(*args) - end - - expect do - rmt_mirror.mirror(**mirror_params) - end.to raise_error(RMT::Mirror::Exception, 'Error while mirroring packages: Failed to download 6 files') - end - end - end - - context 'deduplication' do - let(:rmt_source_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: RMT::Logger.new('/dev/null'), - mirror_src: false - ) - end - - let(:rmt_dedup_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: RMT::Logger.new('/dev/null'), - mirror_src: false - ) - end - - let(:rmt_dedup_airgap_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: RMT::Logger.new('/dev/null'), - mirror_src: false, - airgap_mode: true - ) - end - - let(:mirror_params_source) do - { - repository_url: 'http://localhost/dummy_product/product/', - local_path: '/dummy_product/product/', - auth_token: 'repo_auth_token' - } - end - - let(:mirror_params_dedup) do - { - repository_url: 'http://localhost/dummy_deduped_product/product/', - local_path: '/dummy_deduped_product/product/', - auth_token: 'repo_auth_token' - } - end - - let(:dedup_path) { File.join(@tmp_dir, 'dummy_deduped_product/product/') } - let(:source_path) { File.join(@tmp_dir, 'dummy_product/product/') } - - shared_examples_for 'a deduplicated run' do |source_nlink, dedup_nlink, has_same_content| - it 'downloads source rpm files' do - rpm_entries = Dir.entries(File.join(source_path)).select { |entry| entry =~ /\.rpm$/ } - expect(rpm_entries.length).to eq(4) - end - - it 'deduplicates rpm files' do - rpm_entries = Dir.entries(File.join(dedup_path)).select { |entry| entry =~ /\.rpm$/ } - expect(rpm_entries.length).to eq(4) - end - - it 'has correct content for deduplicated rpm files' do - Dir.entries(File.join(dedup_path)).select { |entry| entry =~ /\.rpm$/ }.each do |file| - if has_same_content - expect(File.read(dedup_path + file)).to eq(File.read(source_path + file)) - else - expect(File.read(dedup_path + file)).not_to eq(File.read(source_path + file)) - end - end - end - - it "source rpms have #{source_nlink} nlink" do - Dir.entries(source_path).select { |entry| entry =~ /\.rpm$/ }.each do |file| - expect(File.stat(source_path + file).nlink).to eq(source_nlink) - end - end - - it "dedup rpms have #{dedup_nlink} nlink" do - Dir.entries(dedup_path).select { |entry| entry =~ /\.rpm$/ }.each do |file| - expect(File.stat(dedup_path + file).nlink).to eq(dedup_nlink) - end - end - - it 'downloads source drpm files' do - rpm_entries = Dir.entries(File.join(source_path)).select { |entry| entry =~ /\.drpm$/ } - expect(rpm_entries.length).to eq(2) - end - - it 'deduplicates drpm files' do - rpm_entries = Dir.entries(File.join(dedup_path)).select { |entry| entry =~ /\.drpm$/ } - expect(rpm_entries.length).to eq(2) - end - - it 'has correct content for deduplicated drpm files' do - Dir.entries(File.join(dedup_path)).select { |entry| entry =~ /\.drpm$/ }.each do |file| - if has_same_content - expect(File.read(dedup_path + file)).to eq(File.read(source_path + file)) - else - expect(File.read(dedup_path + file)).not_to eq(File.read(source_path + file)) - end - end - end - - it "source drpms have #{source_nlink} nlink" do - Dir.entries(source_path).select { |entry| entry =~ /\.drpm$/ }.each do |file| - expect(File.stat(source_path + file).nlink).to eq(source_nlink) - end - end - - it "dedup drpms have #{dedup_nlink} nlink" do - Dir.entries(dedup_path).select { |entry| entry =~ /\.drpm$/ }.each do |file| - expect(File.stat(dedup_path + file).nlink).to eq(dedup_nlink) - end - end - end - - context 'by copy' do - before do - deduplication_method(:copy) - VCR.use_cassette 'mirroring_product_with_dedup' do - rmt_source_mirror.mirror(**mirror_params_source) - rmt_dedup_mirror.mirror(**mirror_params_dedup) - end - end - - it_behaves_like 'a deduplicated run', 1, 1, true - end - - context 'by hardlink' do - before do - deduplication_method(:hardlink) - VCR.use_cassette 'mirroring_product_with_dedup' do - rmt_source_mirror.mirror(**mirror_params_source) - rmt_dedup_mirror.mirror(**mirror_params_dedup) - end - end - - it_behaves_like 'a deduplicated run', 2, 2, true - end - - context 'tracking downloaded files' do - before do - deduplication_method(:hardlink) - end - - it 'tracks deduplicated files' do - VCR.use_cassette 'mirroring_product_with_dedup' do - rmt_source_mirror.mirror(**mirror_params_source) - rmt_dedup_mirror.mirror(**mirror_params_dedup) - end - rpm_entries = Dir.entries(File.join(source_path)).select { |entry| entry =~ /\.rpm$/ } - count = rpm_entries.inject(0) { |count, entry| count + DownloadedFile.where("local_path like '%#{entry}'").count } - expect(count).to eq(8) - end - - it 'does not track airgap deduplicated files' do - VCR.use_cassette 'mirroring_product_with_dedup' do - rmt_source_mirror.mirror(**mirror_params_source) - rmt_dedup_airgap_mirror.mirror(**mirror_params_dedup) - end - rpm_entries = Dir.entries(File.join(source_path)).select { |entry| entry =~ /\.rpm$/ } - count = rpm_entries.inject(0) { |count, entry| count + DownloadedFile.where("local_path like '%#{entry}'").count } - expect(count).to eq(4) - end - end - - context 'by copy with corruption' do - subject(:deduplicate_mirror) do - VCR.use_cassette 'mirroring_product_with_dedup' do - deduplication_method(:copy) - rmt_dedup_mirror.mirror(**mirror_params_dedup) - end - end - - before do - deduplication_method(:copy) - VCR.use_cassette 'mirroring_product_with_dedup' do - rmt_source_mirror.mirror(**mirror_params_source) - Dir.entries(source_path).select { |entry| entry =~ /(\.drpm|\.rpm)$/ }.each do |filename| - File.write(source_path + filename, 'corruption') - end - end - end - - let(:list_source_rpm_files) do - -> { Dir.glob(File.join(source_path, '**', '*.rpm')) } - end - - let(:list_source_drpm_files) do - -> { Dir.glob(File.join(source_path, '**', '*.drpm')) } - end - - let(:list_dedup_rpm_files) do - -> { Dir.glob(File.join(dedup_path, '**', '*.rpm')) } - end - - let(:list_dedup_drpm_files) do - -> { Dir.glob(File.join(dedup_path, '**', '*.drpm')) } - end - - it 'removes corrupted source rpm files' do - expect { deduplicate_mirror } - .to change { list_source_rpm_files.call.length } - .from(4).to(0) - end - - it 'untracks corrupted source rpm files in the database' do - expect { deduplicate_mirror } - .to change { DownloadedFile.where(local_path: list_source_rpm_files.call).length } - .from(4).to(0) - end - - it 'removes corrupted source drpm files' do - expect { deduplicate_mirror } - .to change { list_source_drpm_files.call.length } - .from(2).to(0) - end - - it 'untracks corrupted source drpm files in the database' do - expect { deduplicate_mirror } - .to change { DownloadedFile.where(local_path: list_source_drpm_files.call).length } - .from(2).to(0) - end - - it 'downloads new rpm files instead of deduplicating from corrupted ones' do - source_files_content = list_source_rpm_files.call - .map { |file| [File.basename(file), File.read(file)] } - - deduplicate_mirror - - aggregate_failures 'compare files content' do - list_dedup_rpm_files.call.each do |target_file| - _, source_content = source_files_content - .find { |name, _| target_file.include?(name) } - - expect(File.read(target_file)).not_to eq(source_content) - expect(File.stat(target_file).nlink).to eq(1) - end - end - end - - it 'tracks new rpm files which would be deduplicated' do - expect { deduplicate_mirror } - .to change { list_dedup_rpm_files.call.length } - .from(0).to(4) - end - - it 'downloads new drpm files instead of deduplicating from corrupted ones' do - source_files_content = list_source_drpm_files.call - .map { |file| [File.basename(file), File.read(file)] } - - deduplicate_mirror - - aggregate_failures 'compare files content' do - list_dedup_drpm_files.call.each do |target_file| - _, source_content = source_files_content - .find { |name, _| target_file.include?(name) } - - expect(File.read(target_file)).not_to eq(source_content) - expect(File.stat(target_file).nlink).to eq(1) - end - end - end - - it 'tracks new drpm files which would be deduplicated' do - expect { deduplicate_mirror } - .to change { list_dedup_drpm_files.call.length } - .from(0).to(2) - end - end - end - - context 'with cached metadata' do - let(:mirroring_dir) do - FileUtils.cp_r(file_fixture('dummy_product'), File.join(@tmp_dir, 'dummy_product')) - @tmp_dir - end - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: mirroring_dir, - logger: logger, - mirror_src: false - ) - end - - let(:mirror_params) do - { - repository_url: 'http://localhost/dummy_product/product/', - local_path: '/dummy_product/product/', - auth_token: 'repo_auth_token' - } - end - - let(:timestamp) { 'Mon, 18 May 2020 09:24:25 GMT' } - - before do - metadata_files = [ - File.join(mirroring_dir, 'dummy_product', 'product.license', '**'), - File.join(mirroring_dir, 'dummy_product', 'product', 'repodata', '**') - ].reduce([]) { |files, path| files + Dir.glob(path) } - metadata_files.each { |file| FileUtils.touch(file, mtime: Time.parse(timestamp).utc) } - - VCR.use_cassette 'mirroring_product_with_cached_metadata' do - rmt_mirror.mirror(**mirror_params) - end - end - - it 'downloads rpm files' do - rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_product/product/')).select { |entry| entry =~ /\.rpm$/ } - expect(rpm_entries.length).to eq(4) - end - - it 'preserves metadata timestamps' do - expect(File.mtime("#{mirroring_dir}/dummy_product/product/repodata/repomd.xml")).to eq(Time.parse(timestamp).utc) - end - end - end - - describe '#replace_directory' do - subject(:replace_directory) { rmt_mirror.send(:replace_directory, source_dir, destination_dir) } - - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: logger, - mirror_src: false - ) - end - - let(:source_dir) { '/tmp/temp-repo-dir' } - let(:destination_dir) { '/var/www/repo/product.license' } - let(:old_dir) { '/var/www/repo/.old_product.license' } - - context 'when the old directory exists' do - before do - expect(Dir).to receive(:exist?).with(old_dir).and_return(true) - expect(Dir).to receive(:exist?).with(destination_dir).and_return(false) - end - - it 'removes it, moves src to dst and sets permissions' do - expect(FileUtils).to receive(:remove_entry).with(old_dir) - expect(FileUtils).to receive(:mv).with(source_dir, destination_dir, { force: true }) - expect(FileUtils).to receive(:chmod).with(0o755, destination_dir) - replace_directory - end - end - - context 'when the destination directory already exists' do - before do - expect(Dir).to receive(:exist?).with(old_dir).and_return(false) - expect(Dir).to receive(:exist?).with(destination_dir).and_return(true) - end - - it 'renames it as .old, moves src to dst and sets permissions' do - expect(FileUtils).to receive(:mv).with(destination_dir, old_dir) - expect(FileUtils).to receive(:mv).with(source_dir, destination_dir, { force: true }) - expect(FileUtils).to receive(:chmod).with(0o755, destination_dir) - replace_directory - end - end - - context 'when an error is encountered' do - it 'raises RMT::Mirror::Exception' do - expect(FileUtils).to receive(:mv).and_raise(Errno::ENOENT) - expect { replace_directory }.to raise_error( - RMT::Mirror::Exception, - "Error while moving directory #{source_dir} to #{destination_dir}: No such file or directory" - ) - end - end - end - - context 'when GPG signature is incomplete', vcr: { cassette_name: 'mirroring_with_auth_token' } do - let(:rmt_mirror) do - described_class.new( - mirroring_base_dir: @tmp_dir, - logger: logger, - mirror_src: false - ) - end - - let(:mirror_params) do - { - repository_url: 'http://localhost/dummy_repo/', - local_path: '/dummy_repo', - auth_token: 'repo_auth_token' - } - end - - around do |example| - @tmp_dir = Dir.mktmpdir('rmt') - example.run - FileUtils.remove_entry(@tmp_dir) - end - - context 'when signatures do not exist' do - let(:response) do - instance_double(Typhoeus::Response, code: 404, body: 'Error', - return_code: :ok, return_message: 'No error') - end - - it 'mirrors as normal' do - expect(logger).to receive(:info).with(/Mirroring repository/).once - expect(logger).to receive(:info).with('Repository metadata signatures are missing').once - expect(logger).to receive(:info).with(/↓/).at_least(:once) - - allow_any_instance_of(RMT::Downloader).to receive(:finalize_download).and_wrap_original do |klass, *args| - if args[1].local_path.include?('repodata/repomd.xml.key') - raise RMT::Downloader::Exception.new('HTTP request failed', response: response) - else - klass.call(*args) - end - end - - rmt_mirror.mirror(**mirror_params) - end - end - - context 'when files fail to download with errors other than 404' do - let(:response) do - instance_double(Typhoeus::Response, code: 502, body: 'Error', - return_code: :ok, return_message: 'No error') - end - - it 'raises RMT::Mirror::Exception' do - expect(logger).to receive(:info).with(/Mirroring repository/).once - expect(logger).to receive(:info).with(/↓/).at_least(:once) - - expect_any_instance_of(described_class).to( - receive(:mirror_metadata).and_call_original - ) - - allow_any_instance_of(RMT::Downloader).to receive(:download_multi).and_wrap_original do |klass, *args| - if args[0][0].local_path.include?('repodata/repomd.xml.asc') - raise RMT::Downloader::Exception.new('HTTP request failed', response: response) - else - klass.call(*args) - end - end - - expect { rmt_mirror.mirror(**mirror_params) }.to raise_error( - RMT::Mirror::Exception, - 'Error while mirroring metadata: Downloading repo signature/key failed with: HTTP request failed, HTTP code 502' - ) - end - end - end -end diff --git a/spec/lib/rmt/mirror_spec.rb b/spec/lib/rmt/mirror_spec.rb index e7de0d856..0348feb5f 100644 --- a/spec/lib/rmt/mirror_spec.rb +++ b/spec/lib/rmt/mirror_spec.rb @@ -1,45 +1,840 @@ require 'rails_helper' RSpec.describe RMT::Mirror do - let(:url) { 'http://some.test.us/path/directory/' } - let(:base_dir) { '/non/existing/' } - let(:repository) { create :repository, external_url: url } - - let(:mirror) do - described_class.new(repository: repository, - base_dir: base_dir, - logger: nil, - mirror_sources: false, - is_airgapped: false) + RSpec::Matchers.define :file_reference_containing_path do |expected| + match do |actual| + actual.local_path.include?(expected) + end + + failure_message do |actual| + "expected that file path #{actual.local_path} would contain #{expected}" + end end - describe '#detect_repository_type' do - let(:repomd_url) { 'http://some.test.us/path/directory/repodata/repomd.xml' } - let(:debian_url) { 'http://some.test.us/path/directory/Release' } + let(:logger) { RMT::Logger.new('/dev/null') } + + describe '#mirror_suma_product_tree' do + subject(:command) { rmt_mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/') } + + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: logger, + mirror_src: false + ) + end + + around do |example| + @tmp_dir = Dir.mktmpdir('rmt') + example.run + FileUtils.remove_entry(@tmp_dir) + end + + context 'all is well', vcr: { cassette_name: 'mirroring_suma_product_tree' } do + before do + expect(logger).to receive(:info).with(/Mirroring SUSE Manager product tree to/).once + expect(logger).to receive(:info).with(/↓ product_tree.json/).once + end + + it 'downloads the suma product tree' do + command + content = File.read(File.join(@tmp_dir, 'suma/product_tree.json')) + expect(Digest::SHA256.hexdigest(content)).to eq('7486026e9c1181affae5b21c9aa64637aa682fcdeacb099e213f0e8c7e86d85d') + end + end + + context 'with download exception' do + before do + expect_any_instance_of(RMT::Downloader).to receive(:download_multi).and_raise(RMT::Downloader::Exception, "418 - I'm a teapot") + end + + it 'raises mirroring exception' do + expect { command }.to raise_error(RMT::Mirror::Exception, "Could not mirror SUSE Manager product tree with error: 418 - I'm a teapot") + end + end + end + + describe '#mirror' do + around do |example| + @tmp_dir = Dir.mktmpdir('rmt') + example.run + FileUtils.remove_entry(@tmp_dir) + end + + before do + allow_any_instance_of(RMT::GPG).to receive(:verify_signature) + end + + after do + Dir.glob(File.join(Dir.tmpdir, 'rmt_mirror_*', '**')) + .each { |tmpdir| FileUtils.remove_entry(tmpdir, true) } + end + + context 'without auth_token', vcr: { cassette_name: 'mirroring' } do + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: logger, + mirror_src: false + ) + end + + let(:mirror_params) do + { + repository_url: 'http://localhost/dummy_repo/', + local_path: '/dummy_repo' + } + end + + before do + rmt_mirror.mirror(**mirror_params) + end + + it 'downloads rpm files' do + rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_repo')).select { |entry| entry =~ /\.rpm$/ } + expect(rpm_entries.length).to eq(4) + end + + it 'downloads drpm files' do + rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_repo')).select { |entry| entry =~ /\.drpm$/ } + expect(rpm_entries.length).to eq(2) + end + end + + context 'importing local repo' do + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: logger, + mirror_src: false + ) + end + + let(:mirror_params) do + { + repository_url: URI.join('file://', File.expand_path(file_fixture('dummy_repo'))).to_s + '/', + local_path: Repository.make_local_path('dummy_repo/'), + repo_name: 'dummy_repo' + } + end + + before do + rmt_mirror.mirror(**mirror_params) + end + + it 'copies rpm files' do + expect(Dir.entries(File.join(@tmp_dir, 'dummy_repo'))).to match_array(Dir.entries(file_fixture('dummy_repo'))) + end + + it 'copies metadata' do + expect(Dir.entries(File.join(@tmp_dir, 'dummy_repo/repodata'))).to match_array(Dir.entries(file_fixture('dummy_repo/repodata'))) + end + end + + context 'without auth_token and with source packages', vcr: { cassette_name: 'mirroring_with_src' } do + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: logger, + mirror_src: mirror_src + ) + end + + let(:mirror_params) do + { + repository_url: 'http://localhost/dummy_repo_with_src/', + local_path: '/dummy_repo' + } + end + + before do + rmt_mirror.mirror(**mirror_params) + end + + context 'when mirror_src is false' do + let(:mirror_src) { false } - context 'repomd repository' do - it 'detects a repomd repository' do - stub_request(:head, repomd_url).to_return(status: 200, body: '', headers: {}) + it 'downloads rpm files' do + rpm_entries = Dir.glob(File.join(@tmp_dir, 'dummy_repo', '**', '*.rpm')) + expect(rpm_entries.length).to eq(2) + end - expect(mirror.detect_repository_type).to eq(:repomd) + it 'downloads drpm files' do + rpm_entries = Dir.glob(File.join(@tmp_dir, 'dummy_repo', '**', '*.drpm')) + expect(rpm_entries.length).to eq(1) + end + end + + context 'when mirror_src is true' do + let(:mirror_src) { true } + + it 'downloads rpm files' do + rpm_entries = Dir.glob(File.join(@tmp_dir, 'dummy_repo', '**', '*.rpm')) + expect(rpm_entries.length).to eq(4) + end + + it 'downloads drpm files' do + rpm_entries = Dir.glob(File.join(@tmp_dir, 'dummy_repo', '**', '*.drpm')) + expect(rpm_entries.length).to eq(1) + end end end - context 'debian flat repository' do - it 'detects a flat debian repository' do - stub_request(:head, repomd_url).to_return(status: 404, body: '', headers: {}) - stub_request(:head, debian_url).to_return(status: 200, body: '', headers: {}) + context 'with auth_token', vcr: { cassette_name: 'mirroring_with_auth_token' } do + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: logger, + mirror_src: false + ) + end + + let(:mirror_params) do + { + repository_url: 'http://localhost/dummy_repo/', + local_path: '/dummy_repo', + auth_token: 'repo_auth_token' + } + end + + before do + expect(logger).to receive(:info).with(/Mirroring repository/).once + expect(logger).to receive(:info).with('Repository metadata signatures are missing').once + expect(logger).to receive(:info).with(/↓/).at_least(1).times + rmt_mirror.mirror(**mirror_params) + end + + it 'downloads rpm files' do + rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_repo')).select { |entry| entry =~ /\.rpm$/ } + expect(rpm_entries.length).to eq(4) + end - expect(mirror.detect_repository_type).to eq(:debian) + it 'downloads drpm files' do + rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_repo')).select { |entry| entry =~ /\.drpm$/ } + expect(rpm_entries.length).to eq(2) end end - context 'unknown repository type' do - it 'raises if a unknown repository type is detected' do - stub_request(:head, repomd_url).to_return(status: 404, body: '', headers: {}) - stub_request(:head, debian_url).to_return(status: 404, body: '', headers: {}) + context 'product with license and signatures', vcr: { cassette_name: 'mirroring_product' } do + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: logger, + mirror_src: false + ) + end + + let(:mirror_params) do + { + repository_url: 'http://localhost/dummy_product/product/', + local_path: '/dummy_product/product/', + auth_token: 'repo_auth_token' + } + end + + before do + expect(logger).to receive(:info).with(/Mirroring repository/).once + expect(logger).to receive(:info).with(/↓/).at_least(1).times + rmt_mirror.mirror(**mirror_params) + end + + it 'downloads rpm files' do + rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_product/product/')).select { |entry| entry =~ /\.rpm$/ } + expect(rpm_entries.length).to eq(4) + end + + it 'downloads drpm files' do + rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_product/product/')).select { |entry| entry =~ /\.drpm$/ } + expect(rpm_entries.length).to eq(2) + end + + it 'downloads repomd.xml signatures' do + ['repomd.xml.key', 'repomd.xml.asc'].each do |file| + expect(File.size(File.join(@tmp_dir, 'dummy_product/product/repodata/', file))).to be > 0 + end + end + + it 'downloads product license' do + ['directory.yast', 'license.txt', 'license.de.txt', 'license.ru.txt'].each do |file| + expect(File.size(File.join(@tmp_dir, 'dummy_product/product.license/', file))).to be > 0 + end + end + end + + context 'when an error occurs' do + let(:mirroring_dir) { @tmp_dir } + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: mirroring_dir, + logger: logger, + mirror_src: false + ) + end + + let(:mirror_params) do + { + repository_url: 'http://localhost/dummy_product/product/', + local_path: '/dummy_product/product/', + auth_token: 'repo_auth_token' + } + end + + context 'when mirroring_base_dir is not writable' do + let(:mirroring_dir) { '/non/existent/path' } + + it 'raises exception', vcr: { cassette_name: 'mirroring_product' } do + expect { rmt_mirror.mirror(**mirror_params) }.to raise_error(RMT::Mirror::Exception) + end + end + + context "when can't create tmp dir", vcr: { cassette_name: 'mirroring_product' } do + before { allow(Dir).to receive(:mktmpdir).and_raise('mktmpdir exception') } + it 'handles the exception' do + expect { rmt_mirror.mirror(**mirror_params) }.to raise_error(RMT::Mirror::Exception) + end + end + + context "when can't download metadata", vcr: { cassette_name: 'mirroring_product' } do + before do + allow_any_instance_of(RMT::Downloader).to receive(:download_multi).and_call_original + expect_any_instance_of(RMT::Downloader) + .to receive(:download_multi) + .with([file_reference_containing_path('repodata/repomd.xml')]) + .and_raise(RMT::Downloader::Exception, "418 - I'm a teapot") + end + it 'handles RMT::Downloader::Exception' do + expect { rmt_mirror.mirror(**mirror_params) } + .to raise_error(RMT::Mirror::Exception, "Error while mirroring metadata: 418 - I'm a teapot") + end + end + + context "when there's no licenses to download", vcr: { cassette_name: 'mirroring' } do + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: logger, + mirror_src: false + ) + end + + let(:mirror_params) do + { + repository_url: 'http://localhost/dummy_repo/', + local_path: '/dummy_product/product/' + } + end + + it 'does not error out' do + expect { rmt_mirror.mirror(**mirror_params) }.not_to raise_error + end + + it 'does not create a product.licenses directory' do + rmt_mirror.mirror(**mirror_params) + expect(Dir).not_to exist(File.join(@tmp_dir, 'dummy_product', 'product.license')) + end + + it 'removes the temporary licenses directory' do + rmt_mirror.mirror(**mirror_params) + tmp_dir_glob = Dir.glob(File.join(Dir.tmpdir, 'rmt_mirror_*', '**')) + expect(tmp_dir_glob.length).to eq(0) + end + end + + context "when can't download some of the license files" do + before do + allow_any_instance_of(RMT::Downloader).to receive(:download_multi).and_wrap_original do |klass, *args| + raise RMT::Downloader::Exception.new('') if args[0][0].local_path =~ /license/ + klass.call(*args) + end + end + it 'handles RMT::Downloader::Exception', vcr: { cassette_name: 'mirroring_product' } do + expect { rmt_mirror.mirror(**mirror_params) }.to raise_error(RMT::Mirror::Exception, /Error while mirroring license files:/) + end + end + + context "when can't parse metadata", vcr: { cassette_name: 'mirroring_product' } do + before { allow_any_instance_of(RepomdParser::RepomdXmlParser).to receive(:parse).and_raise('Parse error') } + it 'removes the temporary metadata directory' do + expect { rmt_mirror.mirror(**mirror_params) } + .to raise_error(RMT::Mirror::Exception, 'Error while mirroring metadata: Parse error') + + tmp_dir_glob = Dir.glob(File.join(Dir.tmpdir, 'rmt_mirror_*', '**')) + expect(tmp_dir_glob.length).to eq(0) + end + end + + context 'when Interrupt is raised', vcr: { cassette_name: 'mirroring_product' } do + before { allow_any_instance_of(RepomdParser::RepomdXmlParser).to receive(:parse).and_raise(Interrupt.new) } + it 'removes the temporary metadata directory' do + expect { rmt_mirror.mirror(**mirror_params) }.to raise_error(Interrupt) + + tmp_dir_glob = Dir.glob(File.join(Dir.tmpdir, 'rmt_mirror_*', '**')) + expect(tmp_dir_glob.length).to eq(0) + end + end + + context "when can't download data", vcr: { cassette_name: 'mirroring_product' } do + it 'handles RMT::Downloader::Exception' do + allow_any_instance_of(RMT::Downloader).to receive(:make_request).and_wrap_original do |klass, *args| + # raise the exception only for the RPMs/DRPMs + raise(RMT::Downloader::Exception, "418 - I'm a teapot") if args[0].local_path =~ /rpm$/ + klass.call(*args) + end + + expect { rmt_mirror.mirror(**mirror_params) }.to raise_error(RMT::Mirror::Exception, 'Error while mirroring packages: Failed to download 6 files') + end + + it 'handles RMT::ChecksumVerifier::Exception' do + allow_any_instance_of(RMT::Downloader).to receive(:make_request).and_wrap_original do |klass, *args| + # raise the exception only for the RPMs/DRPMs + raise(RMT::ChecksumVerifier::Exception, "Checksum doesn't match") if args[0].local_path =~ /rpm$/ + klass.call(*args) + end + + expect { rmt_mirror.mirror(**mirror_params) }.to raise_error(RMT::Mirror::Exception, 'Error while mirroring packages: Failed to download 6 files') + end + end + end + + context 'deduplication' do + let(:rmt_source_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: RMT::Logger.new('/dev/null'), + mirror_src: false + ) + end + + let(:rmt_dedup_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: RMT::Logger.new('/dev/null'), + mirror_src: false + ) + end + + let(:rmt_dedup_airgap_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: RMT::Logger.new('/dev/null'), + mirror_src: false, + airgap_mode: true + ) + end + + let(:mirror_params_source) do + { + repository_url: 'http://localhost/dummy_product/product/', + local_path: '/dummy_product/product/', + auth_token: 'repo_auth_token' + } + end + + let(:mirror_params_dedup) do + { + repository_url: 'http://localhost/dummy_deduped_product/product/', + local_path: '/dummy_deduped_product/product/', + auth_token: 'repo_auth_token' + } + end + + let(:dedup_path) { File.join(@tmp_dir, 'dummy_deduped_product/product/') } + let(:source_path) { File.join(@tmp_dir, 'dummy_product/product/') } + + shared_examples_for 'a deduplicated run' do |source_nlink, dedup_nlink, has_same_content| + it 'downloads source rpm files' do + rpm_entries = Dir.entries(File.join(source_path)).select { |entry| entry =~ /\.rpm$/ } + expect(rpm_entries.length).to eq(4) + end + + it 'deduplicates rpm files' do + rpm_entries = Dir.entries(File.join(dedup_path)).select { |entry| entry =~ /\.rpm$/ } + expect(rpm_entries.length).to eq(4) + end + + it 'has correct content for deduplicated rpm files' do + Dir.entries(File.join(dedup_path)).select { |entry| entry =~ /\.rpm$/ }.each do |file| + if has_same_content + expect(File.read(dedup_path + file)).to eq(File.read(source_path + file)) + else + expect(File.read(dedup_path + file)).not_to eq(File.read(source_path + file)) + end + end + end + + it "source rpms have #{source_nlink} nlink" do + Dir.entries(source_path).select { |entry| entry =~ /\.rpm$/ }.each do |file| + expect(File.stat(source_path + file).nlink).to eq(source_nlink) + end + end + + it "dedup rpms have #{dedup_nlink} nlink" do + Dir.entries(dedup_path).select { |entry| entry =~ /\.rpm$/ }.each do |file| + expect(File.stat(dedup_path + file).nlink).to eq(dedup_nlink) + end + end + + it 'downloads source drpm files' do + rpm_entries = Dir.entries(File.join(source_path)).select { |entry| entry =~ /\.drpm$/ } + expect(rpm_entries.length).to eq(2) + end + + it 'deduplicates drpm files' do + rpm_entries = Dir.entries(File.join(dedup_path)).select { |entry| entry =~ /\.drpm$/ } + expect(rpm_entries.length).to eq(2) + end + + it 'has correct content for deduplicated drpm files' do + Dir.entries(File.join(dedup_path)).select { |entry| entry =~ /\.drpm$/ }.each do |file| + if has_same_content + expect(File.read(dedup_path + file)).to eq(File.read(source_path + file)) + else + expect(File.read(dedup_path + file)).not_to eq(File.read(source_path + file)) + end + end + end + + it "source drpms have #{source_nlink} nlink" do + Dir.entries(source_path).select { |entry| entry =~ /\.drpm$/ }.each do |file| + expect(File.stat(source_path + file).nlink).to eq(source_nlink) + end + end + + it "dedup drpms have #{dedup_nlink} nlink" do + Dir.entries(dedup_path).select { |entry| entry =~ /\.drpm$/ }.each do |file| + expect(File.stat(dedup_path + file).nlink).to eq(dedup_nlink) + end + end + end + + context 'by copy' do + before do + deduplication_method(:copy) + VCR.use_cassette 'mirroring_product_with_dedup' do + rmt_source_mirror.mirror(**mirror_params_source) + rmt_dedup_mirror.mirror(**mirror_params_dedup) + end + end + + it_behaves_like 'a deduplicated run', 1, 1, true + end + + context 'by hardlink' do + before do + deduplication_method(:hardlink) + VCR.use_cassette 'mirroring_product_with_dedup' do + rmt_source_mirror.mirror(**mirror_params_source) + rmt_dedup_mirror.mirror(**mirror_params_dedup) + end + end + + it_behaves_like 'a deduplicated run', 2, 2, true + end + + context 'tracking downloaded files' do + before do + deduplication_method(:hardlink) + end + + it 'tracks deduplicated files' do + VCR.use_cassette 'mirroring_product_with_dedup' do + rmt_source_mirror.mirror(**mirror_params_source) + rmt_dedup_mirror.mirror(**mirror_params_dedup) + end + rpm_entries = Dir.entries(File.join(source_path)).select { |entry| entry =~ /\.rpm$/ } + count = rpm_entries.inject(0) { |count, entry| count + DownloadedFile.where("local_path like '%#{entry}'").count } + expect(count).to eq(8) + end + + it 'does not track airgap deduplicated files' do + VCR.use_cassette 'mirroring_product_with_dedup' do + rmt_source_mirror.mirror(**mirror_params_source) + rmt_dedup_airgap_mirror.mirror(**mirror_params_dedup) + end + rpm_entries = Dir.entries(File.join(source_path)).select { |entry| entry =~ /\.rpm$/ } + count = rpm_entries.inject(0) { |count, entry| count + DownloadedFile.where("local_path like '%#{entry}'").count } + expect(count).to eq(4) + end + end + + context 'by copy with corruption' do + subject(:deduplicate_mirror) do + VCR.use_cassette 'mirroring_product_with_dedup' do + deduplication_method(:copy) + rmt_dedup_mirror.mirror(**mirror_params_dedup) + end + end + + before do + deduplication_method(:copy) + VCR.use_cassette 'mirroring_product_with_dedup' do + rmt_source_mirror.mirror(**mirror_params_source) + Dir.entries(source_path).select { |entry| entry =~ /(\.drpm|\.rpm)$/ }.each do |filename| + File.write(source_path + filename, 'corruption') + end + end + end + + let(:list_source_rpm_files) do + -> { Dir.glob(File.join(source_path, '**', '*.rpm')) } + end + + let(:list_source_drpm_files) do + -> { Dir.glob(File.join(source_path, '**', '*.drpm')) } + end + + let(:list_dedup_rpm_files) do + -> { Dir.glob(File.join(dedup_path, '**', '*.rpm')) } + end + + let(:list_dedup_drpm_files) do + -> { Dir.glob(File.join(dedup_path, '**', '*.drpm')) } + end + + it 'removes corrupted source rpm files' do + expect { deduplicate_mirror } + .to change { list_source_rpm_files.call.length } + .from(4).to(0) + end + + it 'untracks corrupted source rpm files in the database' do + expect { deduplicate_mirror } + .to change { DownloadedFile.where(local_path: list_source_rpm_files.call).length } + .from(4).to(0) + end + + it 'removes corrupted source drpm files' do + expect { deduplicate_mirror } + .to change { list_source_drpm_files.call.length } + .from(2).to(0) + end + + it 'untracks corrupted source drpm files in the database' do + expect { deduplicate_mirror } + .to change { DownloadedFile.where(local_path: list_source_drpm_files.call).length } + .from(2).to(0) + end + + it 'downloads new rpm files instead of deduplicating from corrupted ones' do + source_files_content = list_source_rpm_files.call + .map { |file| [File.basename(file), File.read(file)] } + + deduplicate_mirror + + aggregate_failures 'compare files content' do + list_dedup_rpm_files.call.each do |target_file| + _, source_content = source_files_content + .find { |name, _| target_file.include?(name) } + + expect(File.read(target_file)).not_to eq(source_content) + expect(File.stat(target_file).nlink).to eq(1) + end + end + end + + it 'tracks new rpm files which would be deduplicated' do + expect { deduplicate_mirror } + .to change { list_dedup_rpm_files.call.length } + .from(0).to(4) + end + + it 'downloads new drpm files instead of deduplicating from corrupted ones' do + source_files_content = list_source_drpm_files.call + .map { |file| [File.basename(file), File.read(file)] } + + deduplicate_mirror + + aggregate_failures 'compare files content' do + list_dedup_drpm_files.call.each do |target_file| + _, source_content = source_files_content + .find { |name, _| target_file.include?(name) } + + expect(File.read(target_file)).not_to eq(source_content) + expect(File.stat(target_file).nlink).to eq(1) + end + end + end + + it 'tracks new drpm files which would be deduplicated' do + expect { deduplicate_mirror } + .to change { list_dedup_drpm_files.call.length } + .from(0).to(2) + end + end + end + + context 'with cached metadata' do + let(:mirroring_dir) do + FileUtils.cp_r(file_fixture('dummy_product'), File.join(@tmp_dir, 'dummy_product')) + @tmp_dir + end + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: mirroring_dir, + logger: logger, + mirror_src: false + ) + end + + let(:mirror_params) do + { + repository_url: 'http://localhost/dummy_product/product/', + local_path: '/dummy_product/product/', + auth_token: 'repo_auth_token' + } + end + + let(:timestamp) { 'Mon, 18 May 2020 09:24:25 GMT' } + + before do + metadata_files = [ + File.join(mirroring_dir, 'dummy_product', 'product.license', '**'), + File.join(mirroring_dir, 'dummy_product', 'product', 'repodata', '**') + ].reduce([]) { |files, path| files + Dir.glob(path) } + metadata_files.each { |file| FileUtils.touch(file, mtime: Time.parse(timestamp).utc) } + + VCR.use_cassette 'mirroring_product_with_cached_metadata' do + rmt_mirror.mirror(**mirror_params) + end + end + + it 'downloads rpm files' do + rpm_entries = Dir.entries(File.join(@tmp_dir, 'dummy_product/product/')).select { |entry| entry =~ /\.rpm$/ } + expect(rpm_entries.length).to eq(4) + end + + it 'preserves metadata timestamps' do + expect(File.mtime("#{mirroring_dir}/dummy_product/product/repodata/repomd.xml")).to eq(Time.parse(timestamp).utc) + end + end + end + + describe '#replace_directory' do + subject(:replace_directory) { rmt_mirror.send(:replace_directory, source_dir, destination_dir) } + + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: logger, + mirror_src: false + ) + end + + let(:source_dir) { '/tmp/temp-repo-dir' } + let(:destination_dir) { '/var/www/repo/product.license' } + let(:old_dir) { '/var/www/repo/.old_product.license' } + + context 'when the old directory exists' do + before do + expect(Dir).to receive(:exist?).with(old_dir).and_return(true) + expect(Dir).to receive(:exist?).with(destination_dir).and_return(false) + end + + it 'removes it, moves src to dst and sets permissions' do + expect(FileUtils).to receive(:remove_entry).with(old_dir) + expect(FileUtils).to receive(:mv).with(source_dir, destination_dir, { force: true }) + expect(FileUtils).to receive(:chmod).with(0o755, destination_dir) + replace_directory + end + end + + context 'when the destination directory already exists' do + before do + expect(Dir).to receive(:exist?).with(old_dir).and_return(false) + expect(Dir).to receive(:exist?).with(destination_dir).and_return(true) + end + + it 'renames it as .old, moves src to dst and sets permissions' do + expect(FileUtils).to receive(:mv).with(destination_dir, old_dir) + expect(FileUtils).to receive(:mv).with(source_dir, destination_dir, { force: true }) + expect(FileUtils).to receive(:chmod).with(0o755, destination_dir) + replace_directory + end + end + + context 'when an error is encountered' do + it 'raises RMT::Mirror::Exception' do + expect(FileUtils).to receive(:mv).and_raise(Errno::ENOENT) + expect { replace_directory }.to raise_error( + RMT::Mirror::Exception, + "Error while moving directory #{source_dir} to #{destination_dir}: No such file or directory" + ) + end + end + end + + context 'when GPG signature is incomplete', vcr: { cassette_name: 'mirroring_with_auth_token' } do + let(:rmt_mirror) do + described_class.new( + mirroring_base_dir: @tmp_dir, + logger: logger, + mirror_src: false + ) + end + + let(:mirror_params) do + { + repository_url: 'http://localhost/dummy_repo/', + local_path: '/dummy_repo', + auth_token: 'repo_auth_token' + } + end + + around do |example| + @tmp_dir = Dir.mktmpdir('rmt') + example.run + FileUtils.remove_entry(@tmp_dir) + end + + context 'when signatures do not exist' do + let(:response) do + instance_double(Typhoeus::Response, code: 404, body: 'Error', + return_code: :ok, return_message: 'No error') + end + + it 'mirrors as normal' do + expect(logger).to receive(:info).with(/Mirroring repository/).once + expect(logger).to receive(:info).with('Repository metadata signatures are missing').once + expect(logger).to receive(:info).with(/↓/).at_least(1).times + + allow_any_instance_of(RMT::Downloader).to receive(:finalize_download).and_wrap_original do |klass, *args| + if args[1].local_path.include?('repodata/repomd.xml.key') + raise RMT::Downloader::Exception.new('HTTP request failed', response: response) + else + klass.call(*args) + end + end + + rmt_mirror.mirror(**mirror_params) + end + end + + context 'when files fail to download with errors other than 404' do + let(:response) do + instance_double(Typhoeus::Response, code: 502, body: 'Error', + return_code: :ok, return_message: 'No error') + end + + it 'raises RMT::Mirror::Exception' do + expect(logger).to receive(:info).with(/Mirroring repository/).once + expect(logger).to receive(:info).with(/↓/).at_least(1).times + + expect_any_instance_of(described_class).to( + receive(:mirror_metadata).and_call_original + ) + + allow_any_instance_of(RMT::Downloader).to receive(:download_multi).and_wrap_original do |klass, *args| + if args[0][0].local_path.include?('repodata/repomd.xml.asc') + raise RMT::Downloader::Exception.new('HTTP request failed', response: response) + else + klass.call(*args) + end + end - expect(mirror.detect_repository_type).to be_nil + expect { rmt_mirror.mirror(**mirror_params) }.to raise_error( + RMT::Mirror::Exception, + 'Error while mirroring metadata: Downloading repo signature/key failed with: HTTP request failed, HTTP code 502' + ) end end end