wip: add konfig

.gitignore

@@ -31,3 +31,5 @@ Procfile.local
 VERSION
 
 .rubocop-https*
+.env*
+

CONTRIBUTING.md

@@ -24,14 +24,18 @@ bundle install
 
 ## Configuration
 
-At present, configuration is handled using a config file. This lives in `config/postal/postal.yml`. An example configuration file is provided in `config/postal.example.yml`. This example is for development use only and not an example for production use.
+Configuration is handled using a config file. This lives in `config/postal/postal.yml`. An example configuration file is provided in `config/examples/development.yml`. This example is for development use only and not an example for production use.
 
 You'll also need a key for signing. You can generate one of these like this:
 
 ```
 openssl genrsa -out config/postal/signing.key 2048
 ```
 
+If you're running the tests (and you probably should be), you'll find an example file for test configuration in `config/examples/test.yml`. This should be placed in `config/postal/postal.test.yml` with the appropriate values.
+
+If you prefer, you can configure Postal using environment variables. These should be placed in `.env` or `.env.test` as apprpriate.
+
 ## Running
 
 The neatest way to run postal is to ensure that `./bin` is your `$PATH` and then use one of the following commands.

Dockerfile

@@ -59,5 +59,5 @@ FROM base AS ci
 # full target - default if no --target option is given
 FROM base AS full
 
-RUN POSTAL_SKIP_CONFIG_CHECK=1 RAILS_GROUPS=assets bundle exec rake assets:precompile
+RUN RAILS_GROUPS=assets bundle exec rake assets:precompile
 RUN touch /opt/postal/app/public/assets/.prebuilt

Gemfile

@@ -5,7 +5,7 @@ gem "authie"
 gem "autoprefixer-rails"
 gem "bcrypt"
 gem "chronic"
-gem "dotenv-rails"
+gem "dotenv"
 gem "dynamic_form"
 gem "encrypto_signo"
 gem "execjs", "~> 2.7", "< 2.8"
@@ -15,6 +15,7 @@ gem "hashie"
 gem "highline", require: false
 gem "kaminari"
 gem "klogger-logger"
+gem "konfig-config", "~> 2.0"
 gem "mail"
 gem "moonrope"
 gem "mysql2"

Gemfile.lock

@@ -102,9 +102,6 @@ GEM
     deep_merge (1.2.2)
     diff-lcs (1.5.0)
     dotenv (3.0.2)
-    dotenv-rails (3.0.2)
-      dotenv (= 3.0.2)
-      railties (>= 6.1)
     dynamic_form (1.3.1)
       actionview (> 5.2.0)
       activemodel (> 5.2.0)
@@ -151,6 +148,8 @@ GEM
       concurrent-ruby (>= 1.0, < 2.0)
       json
       rouge (>= 3.30, < 5.0)
+    konfig-config (2.1.1)
+      hashie
     loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -335,7 +334,7 @@ DEPENDENCIES
   chronic
   coffee-rails (~> 5.0)
   database_cleaner
-  dotenv-rails
+  dotenv
   dynamic_form
   encrypto_signo
   execjs (~> 2.7, < 2.8)
@@ -347,6 +346,7 @@ DEPENDENCIES
   jquery-rails
   kaminari
   klogger-logger
+  konfig-config (~> 2.0)
   mail
   moonrope
   mysql2

Procfile.dev

@@ -1,3 +1,3 @@
-web: bundle exec puma -C config/puma.rb
+web: unset PORT; bundle exec puma -C config/puma.rb
 worker: bundle exec ruby script/worker.rb
-smtp: bundle exec ruby script/smtp_server.rb
+smtp: unset PORT; bundle exec ruby script/smtp_server.rb

app/lib/dkim_header.rb

@@ -8,9 +8,9 @@ def initialize(domain, message)
       @dkim_key = domain.dkim_key
       @dkim_identifier = domain.dkim_identifier
     else
-      @domain_name = Postal.config.dns.return_path
+      @domain_name = Postal::Config.dns.return_path_domain
       @dkim_key = Postal.signing_key
-      @dkim_identifier = Postal.config.dns.dkim_identifier
+      @dkim_identifier = Postal::Config.dns.dkim_identifier
     end
     @domain = domain
     @message = message

app/lib/message_dequeuer/single_message_processor.rb

@@ -48,7 +48,7 @@ def check_server_suspension
     end
 
     def check_delivery_attempts
-      return if queued_message.attempts < Postal.config.general.maximum_delivery_attempts
+      return if queued_message.attempts < Postal::Config.postal.default_maximum_delivery_attempts
 
       details = "Maximum number of delivery attempts (#{queued_message.attempts}) has been reached."
       if queued_message.message.scope == "incoming"

app/lib/received_header.rb

@@ -3,8 +3,8 @@
 class ReceivedHeader
 
   OUR_HOSTNAMES = {
-    smtp: Postal.config.dns.smtp_server_hostname,
-    http: Postal.config.web.host
+    smtp: Postal::Config.postal.smtp_hostname,
+    http: Postal::Config.postal.web_hostname
   }.freeze
 
   class << self

app/lib/reply_separator.rb

@@ -1,36 +1,34 @@
 # frozen_string_literal: true
 
-module Postal
-  class ReplySeparator
+class ReplySeparator
 
-    RULES = [
-      /^-{2,10} $.*/m,
-      /^>*\s*----- ?Original Message ?-----.*/m,
-      /^>*\s*From:[^\r\n]*[\r\n]+Sent:.*/m,
-      /^>*\s*From:[^\r\n]*[\r\n]+Date:.*/m,
-      /^>*\s*-----Urspr.ngliche Nachricht----- .*/m,
-      /^>*\s*Le[^\r\n]{10,200}a .crit ?:\s*$.*/,
-      /^>*\s*__________________.*/m,
-      /^>*\s*On.{10,200}wrote:\s*$.*/m,
-      /^>*\s*Sent from my.*/m,
-      /^>*\s*=== Please reply above this line ===.*/m,
-      /(^>.*\n?){10,}/
-    ].freeze
+  RULES = [
+    /^-{2,10} $.*/m,
+    /^>*\s*----- ?Original Message ?-----.*/m,
+    /^>*\s*From:[^\r\n]*[\r\n]+Sent:.*/m,
+    /^>*\s*From:[^\r\n]*[\r\n]+Date:.*/m,
+    /^>*\s*-----Urspr.ngliche Nachricht----- .*/m,
+    /^>*\s*Le[^\r\n]{10,200}a .crit ?:\s*$.*/,
+    /^>*\s*__________________.*/m,
+    /^>*\s*On.{10,200}wrote:\s*$.*/m,
+    /^>*\s*Sent from my.*/m,
+    /^>*\s*=== Please reply above this line ===.*/m,
+    /(^>.*\n?){10,}/
+  ].freeze
 
-    def self.separate(text)
-      return "" unless text.is_a?(String)
+  def self.separate(text)
+    return "" unless text.is_a?(String)
 
-      text = text.gsub("\r", "")
-      stripped = String.new
-      RULES.each do |rule|
-        text.gsub!(rule) do
-          stripped = ::Regexp.last_match(0).to_s + "\n" + stripped
-          ""
-        end
+    text = text.gsub("\r", "")
+    stripped = String.new
+    RULES.each do |rule|
+      text.gsub!(rule) do
+        stripped = ::Regexp.last_match(0).to_s + "\n" + stripped
+        ""
       end
-      stripped = stripped.strip
-      [text.strip, stripped.presence]
     end
-
+    stripped = stripped.strip
+    [text.strip, stripped.presence]
   end
+
 end

app/lib/smtp_server/client.rb

@@ -32,7 +32,9 @@ def initialize(ip_address)
     end
 
     def check_ip_address
-      return unless @ip_address && Postal.config.smtp_server.log_exclude_ips && @ip_address =~ Regexp.new(Postal.config.smtp_server.log_exclude_ips)
+      return unless @ip_address &&
+                    Postal::Config.smtp_server.log_ip_address_exclusion_matcher &&
+                    @ip_address =~ Regexp.new(Postal::Config.smtp_server.log_ip_address_exclusion_matcher)
 
       @logging_enabled = false
     end
@@ -109,7 +111,7 @@ def proxy(data)
         @state = :welcome
         log "\e[35m   Client identified as #{@ip_address}\e[0m"
         increment_command_count("PROXY")
-        "220 #{Postal.config.dns.smtp_server_hostname} ESMTP Postal/#{id}"
+        "220 #{Postal::Config.postal.smtp_hostname} ESMTP Postal/#{id}"
       else
         @finished = true
         increment_error_count("proxy-error")
@@ -123,7 +125,7 @@ def quit
     end
 
     def starttls
-      if Postal.config.smtp_server.tls_enabled?
+      if Postal::Config.smtp_server.tls_enabled?
         @start_tls = true
         @tls = true
         increment_command_count("STARTLS")
@@ -141,7 +143,7 @@ def ehlo(data)
       increment_command_count("EHLO")
       [
         "250-My capabilities are",
-        Postal.config.smtp_server.tls_enabled? && !@tls ? "250-STARTTLS" : nil,
+        Postal::Config.smtp_server.tls_enabled? && !@tls ? "250-STARTTLS" : nil,
         "250 AUTH CRAM-MD5 PLAIN LOGIN"
       ].compact
     end
@@ -151,7 +153,7 @@ def helo(data)
       transaction_reset
       @state = :welcomed
       increment_command_count("HELO")
-      "250 #{Postal.config.dns.smtp_server_hostname}"
+      "250 #{Postal::Config.postal.smtp_hostname}"
     end
 
     def rset
@@ -231,7 +233,7 @@ def auth_cram_md5(data)
       increment_command_count("AUTH CRAM-MD5")
 
       challenge = Digest::SHA1.hexdigest(Time.now.to_i.to_s + rand(100_000).to_s)
-      challenge = "<#{challenge[0, 20]}@#{Postal.config.dns.smtp_server_hostname}>"
+      challenge = "<#{challenge[0, 20]}@#{Postal::Config.postal.smtp_hostname}>"
 
       handler = proc do |idata|
         @proc = nil
@@ -309,7 +311,7 @@ def rcpt_to(data)
 
       uname, tag = uname.split("+", 2)
 
-      if domain == Postal.config.dns.return_path || domain =~ /\A#{Regexp.escape(Postal.config.dns.custom_return_path_prefix)}\./
+      if domain == Postal::Config.dns.return_path_domain || domain =~ /\A#{Regexp.escape(Postal::Config.dns.custom_return_path_prefix)}\./
         # This is a return path
         @state = :rcpt_to_received
         if server = ::Server.where(token: uname).first
@@ -326,7 +328,7 @@ def rcpt_to(data)
           "550 Invalid server token"
         end
 
-      elsif domain == Postal.config.dns.route_domain
+      elsif domain == Postal::Config.dns.route_domain
         # This is an email direct to a route. This isn't actually supported yet.
         @state = :rcpt_to_received
         if route = Route.where(token: uname).first
@@ -446,14 +448,14 @@ def data(_data)
     end
 
     def finished
-      if @data.bytesize > Postal.config.smtp_server.max_message_size.megabytes.to_i
+      if @data.bytesize > Postal::Config.smtp_server.max_message_size.megabytes.to_i
         transaction_reset
         @state = :welcomed
         increment_error_count("message-too-large")
-        return format("552 Message too large (maximum size %dMB)", Postal.config.smtp_server.max_message_size)
+        return format("552 Message too large (maximum size %dMB)", Postal::Config.smtp_server.max_message_size)
       end
 
-      if @headers["received"].grep(/by #{Postal.config.dns.smtp_server_hostname}/).count > 4
+      if @headers["received"].grep(/by #{Postal::Config.postal.smtp_hostname}/).count > 4
         transaction_reset
         @state = :welcomed
         increment_error_count("loop-detected")

app/lib/smtp_server/server.rb

@@ -8,6 +8,24 @@ class Server
 
     include HasPrometheusMetrics
 
+    class << self
+
+      def tls_private_key
+        @tls_private_key ||= OpenSSL::PKey.read(File.read(Postal::Config.smtp_server.tls_private_key_path))
+      end
+
+      def tls_certificates
+        @tls_certificates ||= begin
+          data = File.read(Postal::Config.smtp_server.tls_certificate_path)
+          certs = data.scan(/-----BEGIN CERTIFICATE-----.+?-----END CERTIFICATE-----/m)
+          certs.map do |c|
+            OpenSSL::X509::Certificate.new(c)
+          end.freeze
+        end
+      end
+
+    end
+
     def initialize(options = {})
       @options = options
       @options[:debug] ||= false
@@ -43,16 +61,19 @@ def ssl_context
       @ssl_context ||= begin
         ssl_context      = OpenSSL::SSL::SSLContext.new
         ssl_context.cert = Postal.smtp_certificates[0]
-        ssl_context.extra_chain_cert = Postal.smtp_certificates[1..]
-        ssl_context.key = Postal.smtp_private_key
-        ssl_context.ssl_version = Postal.config.smtp_server.ssl_version if Postal.config.smtp_server.ssl_version
-        ssl_context.ciphers = Postal.config.smtp_server.tls_ciphers if Postal.config.smtp_server.tls_ciphers
+        ssl_context.extra_chain_cert = self.class.tls_certificates[1..]
+        ssl_context.key = self.class.tls_private_key
+        ssl_context.ssl_version = Postal::Config.smtp_server.ssl_version if Postal::Config.smtp_server.ssl_version
+        ssl_context.ciphers = Postal::Config.smtp_server.tls_ciphers if Postal::Config.smtp_server.tls_ciphers
         ssl_context
       end
     end
 
     def listen
-      @server = TCPServer.open(Postal.config.smtp_server.bind_address, Postal.config.smtp_server.port)
+      bind_address = ENV.fetch("BIND_ADDRESS", Postal::Config.smtp_server.default_bind_address)
+      port = ENV.fetch("PORT", Postal::Config.smtp_server.default_port)
+
+      @server = TCPServer.open(bind_address, port)
       @server.autoclose = false
       @server.close_on_exec = false
       if defined?(Socket::SOL_SOCKET) && defined?(Socket::SO_KEEPALIVE)
@@ -63,7 +84,8 @@ def listen
         @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPINTVL, 10)
         @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPCNT, 5)
       end
-      logger.info "Listening on #{Postal.config.smtp_server.bind_address}:#{Postal.config.smtp_server.port}"
+
+      logger.info "Listening on #{bind_address}:#{port}"
     end
 
     def unlisten
@@ -90,22 +112,22 @@ def run_event_loop
               # Accept the connection
               new_io = io.accept
               increment_prometheus_counter :postal_smtp_server_connections_total
-              if Postal.config.smtp_server.proxy_protocol
+              if Postal::Config.smtp_server.proxy_protocol?
                 # If we are using the haproxy proxy protocol, we will be sent the
                 # client's IP later. Delay the welcome process.
                 client = Client.new(nil)
-                if Postal.config.smtp_server.log_connect
+                if Postal::Config.smtp_server.log_connections?
                   logger.debug "[#{client.id}] \e[35m   Connection opened from #{new_io.remote_address.ip_address}\e[0m"
                 end
               else
                 # We're not using the proxy protocol so we already know the client's IP
                 client = Client.new(new_io.remote_address.ip_address)
-                if Postal.config.smtp_server.log_connect
+                if Postal::Config.smtp_server.log_connections?
                   logger.debug "[#{client.id}] \e[35m   Connection opened from #{new_io.remote_address.ip_address}\e[0m"
                 end
                 # We know who the client is, welcome them.
                 client.log "\e[35m   Client identified as #{new_io.remote_address.ip_address}\e[0m"
-                new_io.print("220 #{Postal.config.dns.smtp_server_hostname} ESMTP Postal/#{client.id}")
+                new_io.print("220 #{Postal::Config.postal.smtp_hostname} ESMTP Postal/#{client.id}")
               end
               # Register the client and its socket with nio4r
               monitor = @io_selector.register(new_io, :r)

app/mailers/application_mailer.rb

@@ -2,7 +2,7 @@
 
 class ApplicationMailer < ActionMailer::Base
 
-  default from: "#{Postal.smtp_from_name} <#{Postal.smtp_from_address}>"
+  default from: "#{Postal::Config.smtp.from_name} <#{Postal::Config.smtp.from_address}>"
   layout false
 
 end

app/models/bounce_message.rb

@@ -14,7 +14,7 @@ def raw_message
     mail.subject = "Mail Delivery Failed (#{@message.subject})"
     mail.text_part = body
     mail.attachments["Original Message.eml"] = { mime_type: "message/rfc822", encoding: "quoted-printable", content: @message.raw_message }
-    mail.message_id = "<#{SecureRandom.uuid}@#{Postal.config.dns.return_path}>"
+    mail.message_id = "<#{SecureRandom.uuid}@#{Postal::Config.dns.return_path_domain}>"
     mail.to_s
   end
 
@@ -32,7 +32,7 @@ def queue
   end
 
   def postmaster_address
-    @server.postmaster_address || "postmaster@#{@message.domain&.name || Postal.config.web.host}"
+    @server.postmaster_address || "postmaster@#{@message.domain&.name || Postal::Config.postal.web_hostname}"
   end
 
   private