Merge branch 'master' into debian-xz

.env.example

@@ -2,3 +2,5 @@ SECRET_KEY_BASE=8ea53ad3bc6c03923e376c8bdd85059c1885524947a7efe53d5e9c9d4e398611
 EXTERNAL_PORT=8080
 SCC_USERNAME=
 SCC_PASSWORD=
+RMT_METRICS_ENABLED=
+PROMETHEUS_JOB_NAME=

.github/workflows/lint-unit.yml

@@ -81,8 +81,13 @@ jobs:
 
       - name: Run core tests
         run: |
-          bundle exec rspec --format documentation
+          bundle exec rake test:core
 
+      - name: Run core tests with sqlite
+        run: |
+          sed -i 's/adapter: mysql2/adapter: sqlite3/' config/rmt.yml
+          bundle exec rake test:core
+          
       - name: Run PubCloud engines tests
         run: |
           bundle exec rake test:engines
@@ -91,4 +96,4 @@ jobs:
         run: |
           echo "::group::Version verification checks"
           ruby ci/check-version-matches.rb
-          echo "::endgroup::"
+          echo "::endgroup::"

.simplecov

@@ -10,6 +10,10 @@ unless ENV['NO_COVERAGE']
       # omit registration sharing (removing systems using rmt-cli)
       add_filter('engines/registration_sharing/lib/registration_sharing.rb')
 
+      add_filter('lib/rmt/db.rb')
+      add_filter('lib/rmt.rb')
+      add_filter('config')
+
       track_files('app/**/*.rb')
       track_files('lib/**/*.rb')
     end

Gemfile

@@ -32,6 +32,12 @@ gem 'repomd_parser', '~> 1.1.0'
 # Use Rack CORS for handling Cross-Origin Resource Sharing (CORS), making cross-origin AJAX possible
 # gem 'rack-cors'
 
+# Prometheus Exporter:
+gem 'yabeda'
+gem 'yabeda-rails'
+gem 'yabeda-puma-plugin'
+gem 'yabeda-prometheus'
+
 gem 'strong_migrations'
 
 group :development, :test do
@@ -43,7 +49,7 @@ group :development, :test do
   gem 'gettext', require: false # needed for gettext_i18n_rails tasks
   gem 'ruby_parser', '< 3.20', require: false # needed for gettext_i18n_rails tasks, Locked because of Ruby >= 2.6 dependency
   gem 'gettext_test_log'
-  gem 'memory_profiler'
+  gem 'memory_profiler', '~> 1.0.2' # locked because 1.1.0 requires ruby version >= 3.1.0
   gem 'awesome_print'
 end
 

Gemfile.lock

@@ -30,8 +30,10 @@ GEM
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
-    addressable (2.8.6)
-      public_suffix (>= 2.0.2, < 6.0)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
+    anyway_config (2.6.4)
+      ruby-next-core (~> 1.0)
     ast (2.4.2)
     awesome_print (1.9.2)
     base32 (0.3.4)
@@ -136,13 +138,13 @@ GEM
       guard (~> 2.1)
       guard-compat (~> 1.1)
       rspec (>= 2.99.0, < 4.0)
-    hashdiff (1.1.0)
+    hashdiff (1.1.1)
     hpricot (0.8.6)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     json (2.3.1)
     jsonapi-renderer (0.2.2)
-    jwt (2.8.2)
+    jwt (2.9.3)
       base64
     listen (3.6.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
@@ -160,7 +162,7 @@ GEM
     mustache (1.1.1)
     mysql2 (0.5.6)
     nenv (0.3.0)
-    nio4r (2.7.0)
+    nio4r (2.7.3)
     nokogiri (1.12.5)
       mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
@@ -173,11 +175,13 @@ GEM
     prime (0.1.2)
       forwardable
       singleton
+    prometheus-client (4.2.3)
+      base64
     pry (0.14.0)
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (4.0.7)
-    puma (5.6.8)
+    puma (5.6.9)
       nio4r (~> 2.0)
     racc (1.8.0)
     rack (2.2.9)
@@ -209,8 +213,7 @@ GEM
     responders (3.1.1)
       actionpack (>= 5.2)
       railties (>= 5.2)
-    rexml (3.3.6)
-      strscan
+    rexml (3.3.9)
     ronn (0.7.3)
       hpricot (>= 0.8.2)
       mustache (>= 0.7.0)
@@ -267,6 +270,7 @@ GEM
       rubocop (~> 1.19)
     rubocop-thread_safety (0.4.4)
       rubocop (>= 0.53.0)
+    ruby-next-core (1.0.3)
     ruby-progressbar (1.11.0)
     ruby-xz (1.0.3)
     ruby_parser (3.19.2)
@@ -297,7 +301,6 @@ GEM
     sqlite3 (1.4.4)
     strong_migrations (0.7.9)
       activerecord (>= 5)
-    strscan (3.1.0)
     sync (0.5.0)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
@@ -318,10 +321,27 @@ GEM
       activesupport (>= 3)
       railties (>= 3)
       yard (~> 0.9.20)
-    webmock (3.23.1)
+    webmock (3.24.0)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    yabeda (0.13.1)
+      anyway_config (>= 1.0, < 3)
+      concurrent-ruby
+      dry-initializer
+    yabeda-prometheus (0.9.1)
+      prometheus-client (>= 3.0, < 5.0)
+      rack
+      yabeda (~> 0.10)
+    yabeda-puma-plugin (0.7.1)
+      json
+      puma
+      yabeda (~> 0.5)
+    yabeda-rails (0.9.0)
+      activesupport
+      anyway_config (>= 1.3, < 3)
+      railties
+      yabeda (~> 0.8)
     yard (0.9.35)
     zeitwerk (2.6.15)
     zstd-ruby (1.5.6.1)
@@ -352,7 +372,7 @@ DEPENDENCIES
   guard-rspec
   jwt (~> 2.1)
   listen (>= 3.0.5, <= 3.6.0)
-  memory_profiler
+  memory_profiler (~> 1.0.2)
   minitest (~> 5.15.0)
   mysql2 (~> 0.5.3)
   nokogiri (< 1.13)
@@ -384,6 +404,10 @@ DEPENDENCIES
   vcr (~> 6.0)
   versionist
   webmock
+  yabeda
+  yabeda-prometheus
+  yabeda-puma-plugin
+  yabeda-rails
 
 BUNDLED WITH
    1.17.3

PACKAGE.md

@@ -21,7 +21,7 @@ Note: Look below for direction on publishing to registry.
           ```
       * Alternatively, if an OBS working copy is already checked out, update the working copy by running `osc up`
 2. Run `make dist` in your RMT working directory to build a tarball.
-3. Copy the files from the `package/obs` directory to the OBS working directory.
+3. Copy the files from the `package/obs` directory to the OBS working directory `systemsmanagement:SCC:RMT/rmt-server`.
 4. Examine the changes by running `osc status` and `osc diff`.
 5. Stage the changes by running `osc addremove`.
 6. Build the package with osc:
@@ -109,4 +109,4 @@ RMT helm chart is defined [here](https://github.com/SUSE/helm-charts.git) and pu
 
 Edit `rmt-helm/Chart.yaml` to update the chart version (`version`) and rmt-version (`appVersion`). The `BuildTag` version needs to be updated. Look at this example [pull-request](https://github.com/SUSE/helm-charts/pull/5) bumping the version.
 
-Please reach out to the BCI team if you have build related questions.
+Reach out to the BCI team (Dirk Mueller or `#proj-bci` slack channel) to trigger the release of the helm-chart.

app/controllers/api/connect/base_controller.rb

@@ -44,4 +44,14 @@ def authenticate_with_token
     end
   end
 
+  def system_token_header
+    headers[SYSTEM_TOKEN_HEADER] = @system.system_token
+  end
+
+  def refresh_system_token
+    if system_tokens_enabled?
+      @system.update(system_token: SecureRandom.uuid)
+      system_token_header
+    end
+  end
 end

app/controllers/api/connect/v3/systems/products_controller.rb

@@ -5,14 +5,21 @@ class Api::Connect::V3::Systems::ProductsController < Api::Connect::BaseControll
   before_action :check_product_service_and_repositories, only: %i[show activate]
   before_action :load_subscription, only: %i[activate upgrade]
   before_action :check_base_product_dependencies, only: %i[activate upgrade show]
+  after_action :refresh_system_token, only: %i[activate upgrade], if: -> { request.headers.key?(SYSTEM_TOKEN_HEADER) }
 
   def activate
     create_product_activation
     render_service
   end
 
   def show
-    if @system.products.include? @product
+    if @product.identifier.casecmp?('sles')
+      # if system has SLE Micro
+      # it should access to SLES products
+      sle_micro = @system.products.any? { |p| p.identifier.downcase.include?('sle-micro') }
+      sle_micro_same_arch = @system.products.pluck(:arch).include?(@product.arch) if sle_micro
+    end
+    if @system.products.include?(@product) || sle_micro_same_arch
       respond_with(
         @product,
         serializer: ::V3::ProductSerializer,

app/controllers/api/connect/v3/systems/systems_controller.rb

@@ -1,6 +1,7 @@
 class Api::Connect::V3::Systems::SystemsController < Api::Connect::BaseController
 
   before_action :authenticate_system
+  after_action :refresh_system_token, only: [:update], if: -> { request.headers.key?(SYSTEM_TOKEN_HEADER) }
 
   def update
     if params[:online_at].present?
@@ -21,8 +22,8 @@ def update
 
     # Since the payload is handled by rails all values are converted to string
     # e.g. cpus: 16 becomes cpus: "16". We save this as string for now and expect
-    # SCC to handle the convertation correctly
-    @system.system_information = hwinfo_params[:hwinfo].to_json
+    # SCC to handle the conversion correctly
+    @system.system_information = @system.system_information_hash.update(hwinfo_params[:hwinfo]).to_json
 
     if @system.save
       logger.info(N_("Updated system information for host '%s'") % @system.hostname)

app/controllers/api/connect/v4/systems/products_controller.rb

@@ -1,5 +1,6 @@
 class Api::Connect::V4::Systems::ProductsController < Api::Connect::V3::Systems::ProductsController
 
+  after_action :refresh_system_token, only: %i[activate upgrade synchronize destroy], if: -> { request.headers.key?(SYSTEM_TOKEN_HEADER) }
   def destroy
     if @product.base?
       raise ActionController::TranslatedError.new(N_('The product "%s" is a base product and cannot be deactivated'), @product.name)

app/controllers/application_controller.rb

@@ -18,12 +18,13 @@ def authenticate_system(skip_on_duplicated: false)
         return true if skip_on_duplicated && @systems.size > 1
 
         @system = find_system_by_token_header(@systems)
+        update_user_agent
 
         # If SYSTEM_TOKEN_HEADER is present, RMT assumes the client uses a SUSEConnect version
-        # that supports this feature. In this case, refresh the token and include it in the response.
+        # that supports this feature.
         if system_tokens_enabled? && request.headers.key?(SYSTEM_TOKEN_HEADER)
-          @system.update(last_seen_at: Time.zone.now, system_token: SecureRandom.uuid)
-          headers[SYSTEM_TOKEN_HEADER] = @system.system_token
+          @system.update(last_seen_at: Time.zone.now)
+          system_token_header
         # only update last_seen_at each 3 minutes,
         # so that a system that calls SCC every second doesn't write + lock the database row
         elsif !@system.last_seen_at || @system.last_seen_at < 3.minutes.ago
@@ -42,6 +43,15 @@ def authenticate_system(skip_on_duplicated: false)
 
   private
 
+  def zypper_request?
+    user_agent = request.headers['HTTP_USER_AGENT']
+    user_agent&.downcase&.starts_with?('zypp')
+  end
+
+  def update_user_agent
+    @system.set_system_information('user_agent', request.headers['HTTP_USER_AGENT']) unless zypper_request?
+  end
+
   # Token mechanism to detect duplicated systems.
   # 1: system doesn't send a token header (old SUSEConnect version)
   # 2: system sends a token, and it matches an existing system with that token

app/controllers/services_controller.rb

@@ -14,12 +14,10 @@ class ServicesController < ApplicationController
   # authenticate requests on this method for Zypper so we have a better picture
   # which systems are still being active (even if not using SUSEConnect).
   before_action only: %w[show] do
-    ua = request.headers['HTTP_USER_AGENT']
-
     # Zypper will never provide the `system_token` credentials for the system.
     # Hence, if there are duplicates, we will not be able to deterministically
     # tell which system is to be updated. Just skip it altogether on this case.
-    authenticate_system(skip_on_duplicated: true) if ua && ua.downcase.starts_with?('zypp')
+    authenticate_system(skip_on_duplicated: true) if zypper_request?
   end
 
   ZYPPER_SERVICE_TTL = 86400

app/models/repository.rb

@@ -23,14 +23,15 @@ class Repository < ApplicationRecord
   class << self
 
     def remove_suse_repos_without_tokens!
-      where(auth_token: nil).where('external_url LIKE ?', 'https://updates.suse.com%').delete_all
+      where(auth_token: nil).where("external_url LIKE '%.suse.com%'").where(installer_updates: 0).where.not(scc_id: nil).delete_all
     end
 
     # Mangles remote repo URL to make a nicer local path, see specs for examples
     def make_local_path(url)
       uri = URI(url)
       path = uri.path.to_s
-      path.gsub!(%r{^/repo}, '') if (uri.hostname == 'updates.suse.com')
+      # drop '/repo' from SLE11 paths, to avoid double /repo/repo in local storage path.
+      path.gsub!(%r{^/repo/\$RCE/}, '/$RCE/')
       (path == '') ? '/' : path
     end
 

app/models/system.rb

@@ -37,10 +37,17 @@ def self.generate_secure_login
   end
 
   def cloud_provider
+    system_information_hash.fetch(:cloud_provider, nil)
+  end
+
+  def system_information_hash
     # system_information is checked for valid JSON on save. It is safe
     # to assume the structure is valid.
-    info = JSON.parse(system_information).symbolize_keys
-    info.fetch(:cloud_provider, nil)
+    JSON.parse(system_information || '{}').symbolize_keys
+  end
+
+  def set_system_information(key, value)
+    update(system_information: system_information_hash.update(key => value).to_json)
   end
 
   # Generate secure token for System password

app/serializers/v3/product_serializer.rb

@@ -33,8 +33,11 @@ def eula_url
   end
 
   def free
-    # Everything is free on RMT :-)
+    # Everything is free on RMT :-) outside of the Public Cloud (i.e. LTSS)
     # Otherwise Yast and SUSEConnect will request a regcode when activating an extension
+    # FIXME
+    return object.free if defined?(SccProxy::Engine) && object.extension?
+
     true
   end