refactor(config): RHICOMPL-1894 Kafka config read from Settings

README.md

@@ -33,7 +33,7 @@ The Insights Compliance backend comprises of these components/services:
 Before running the project, these services must be running and acessible:
 
 * Kafka — message broker (default port 29092)
-  - set by `KAFKAMQ` environment variable
+  - set by `SETTINGS__KAFKA__BROKERS` environment variable
 * Redis — Job queue and cache
 * PostgreSQL compatible database
   - `DATABASE_SERVICE_NAME=postgres`
@@ -76,7 +76,7 @@ at least three different processes:
 Prerequisites:
 
 * URL to Kafka
-  - environment variable: `KAFKAMQ` (`KAFKAMQ=localhost:29092`)
+  - environment variable: `SETTINGS__KAFKA__BROKERS` (`SETTINGS__KAFKA__BROKERS=localhost:29092`)
 * URL to PostgreSQL database
   - environment variables: `POSTGRESQL_DATABASE`, `POSTGRESQL_SERVICE_HOST`, `POSTGRESQL_USER`, `POSTGRESQL_PASSWORD`, `POSTGRESQL_ADMIN_PASSWORD`, `DATABASE_SERVICE_NAME`
 * URL to Redis

app/producers/application_producer.rb

@@ -3,9 +3,7 @@
 # Common Kafka producer client
 # https://www.rubydoc.info/gems/ruby-kafka/Kafka/Producer
 class ApplicationProducer < Kafka::Client
-  BROKERS = [ENV['KAFKAMQ']].compact.freeze
-  SSL_CA_LOCATION = ENV['RACECAR_SSL_CA_LOCATION']
-  SECURITY_PROTOCOL = ENV['RACECAR_SECURITY_PROTOCOL']
+  BROKERS = Settings.kafka.brokers.split(',').freeze
   CLIENT_ID = 'compliance-backend'
   SERVICE = 'compliance'
   # Define TOPIC in the inherited class.
@@ -29,7 +27,9 @@ def logger
     end
 
     def kafka_ca_cert
-      File.read(self::SSL_CA_LOCATION) if self::SECURITY_PROTOCOL == 'ssl'
+      return unless Settings.kafka.security_protocol == 'ssl'
+
+      File.read(Settings.kafka.ssl_ca_location)
     end
 
     def kafka_config

config/racecar.rb

@@ -1,4 +1,7 @@
 Racecar.configure do |config|
   config.log_level = 'info'
   config.group_id_prefix = 'compliance'
+
+  config.security_protocol = Settings.kafka.security_protocol
+  config.ssl_ca_location = Settings.kafka.ssl_ca_location
 end

config/racecar.yml

@@ -1,20 +1,18 @@
 # These config values will be shared by all environments but can be overridden.
 common: &common
   client_id: "compliance_backend"
+  brokers:
+    <% Settings.kafka.brokers.split(',').each do |broker| %>
+    - <%= broker %>
+    <% end %>
 
 # By default, kafka's container port 9092 is proxied to 2902
 # in insights-upload
 development:
   <<: *common
-  brokers:
-    - <%= ENV["KAFKAMQ"] %>
 
 test:
   <<: *common
-  brokers:
-    - <%= ENV["KAFKAMQ"] %>
 
 production:
   <<: *common
-  brokers:
-    - <%= ENV["KAFKAMQ"] %>

config/settings.yml

@@ -1,6 +1,10 @@
 app_name: 'compliance'
 path_prefix: '/api'
 old_path_prefix: '/r/insights/platform'
+kafka:
+  brokers: ''
+  security_protocol: plaintext
+  ssl_ca_location:
 kafka_consumer_topics:
   inventory_events: 'platform.inventory.events'
 kafka_producer_topics:

docker-compose.yml

@@ -219,7 +219,7 @@ services:
     entrypoint: ''
     command: 'bundle exec racecar -l log/inventory-consumer.log InventoryEventsConsumer'
     environment:
-      - KAFKAMQ=kafka:29092
+      - SETTINGS__KAFKA__BROKERS=kafka:29092
       - DATABASE_SERVICE_NAME=postgres
       - POSTGRES_SERVICE_HOST=db
       - POSTGRESQL_DATABASE=compliance_dev
@@ -272,7 +272,7 @@ services:
       - MALLOC_ARENA_MAX=2
       - SETTINGS__REDIS_URL=redis:6379
       - SETTINGS__PROMETHEUS_EXPORTER_HOST=prometheus
-      - KAFKAMQ=kafka:29092
+      - SETTINGS__KAFKA__BROKERS=kafka:29092
       - DATABASE_SERVICE_NAME=postgres
       - POSTGRES_SERVICE_HOST=db
       - POSTGRESQL_DATABASE=compliance_dev

test/producers/application_producer_test.rb

@@ -3,34 +3,32 @@
 require 'test_helper'
 
 class ApplicationProducerTest < ActiveSupport::TestCase
-  class MockProducer < ApplicationProducer
-    BROKERS = ['broker1'].freeze
+  teardown do
+    Settings.reload!
   end
 
   test 'handles SSL settings' do
-    MockProducer::SECURITY_PROTOCOL = 'ssl'
-    MockProducer::SSL_CA_LOCATION = 'test/fixtures/files/test_ca.crt'
+    Settings.kafka.security_protocol = 'ssl'
+    Settings.kafka.ssl_ca_location = 'test/fixtures/files/test_ca.crt'
+
+    class MockProducer < ApplicationProducer; end
 
     config = {
       client_id: ApplicationProducer::CLIENT_ID,
       ssl_ca_cert: "very secure\n"
     }
-
     assert_equal config, MockProducer.send(:kafka_config)
   end
 
   test 'handles plaintext settings' do
-    MockProducer::SECURITY_PROTOCOL = 'plaintext'
+    Settings.kafka.security_protocol = 'plaintext'
+
+    class MockProducer < ApplicationProducer; end
 
     config = {
       client_id: ApplicationProducer::CLIENT_ID
     }
 
     assert_equal config, MockProducer.send(:kafka_config)
   end
-
-  teardown do
-    # Prevents warnings coming from constant redefinition
-    MockProducer.send(:remove_const, :SECURITY_PROTOCOL)
-  end
 end