From 2825c9829559b9abbf70d8edad15874542512021 Mon Sep 17 00:00:00 2001 From: Did Date: Fri, 1 Nov 2024 23:05:03 +0000 Subject: [PATCH 1/8] implement the On-Demand TLS functionality --- README.md | 14 +++++++ internal/cmd/deploy.go | 1 + internal/server/service.go | 75 +++++++++++++++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a41c58b..c22ec7a 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,20 @@ applications. To enable this, add the `--tls` flag when deploying an instance: kamal-proxy deploy service1 --target web-1:3000 --host app1.example.com --tls +## On-demand TLS + +In addition of the automatic TLS functionality, Kamal Proxy can also dynamically obtain a TLS certificate +from any host allowed by an external API endpoint of your choice. +This avoids hard-coding hosts in the configuration, especially when you don't know the hosts at the startup. + + kamal-proxy deploy service1 --target web-1:3000 --host "" --tls --tls-on-demand-url=localhost:4567/check + +The On-demand URL endpoint will have to answer a 200 HTTP status code. +Kamal Proxy will call the on-demand URL with a query string of `?host=` containing the host received by Kamal Proxy. + +It also must respond as fast as possible, a couple of milliseconds top. + + ### Custom TLS certificate When you obtained your TLS certificate manually, manage your own certificate authority, diff --git a/internal/cmd/deploy.go b/internal/cmd/deploy.go index 82ae5c4..0563937 100644 --- a/internal/cmd/deploy.go +++ b/internal/cmd/deploy.go @@ -30,6 +30,7 @@ func newDeployCommand() *deployCommand { deployCommand.cmd.Flags().StringSliceVar(&deployCommand.args.Hosts, "host", []string{}, "Host(s) to serve this target on (empty for wildcard)") deployCommand.cmd.Flags().BoolVar(&deployCommand.args.ServiceOptions.TLSEnabled, "tls", false, "Configure TLS for this target (requires a non-empty host)") + deployCommand.cmd.Flags().StringVar(&deployCommand.args.ServiceOptions.TLSOnDemandUrl, "tls-on-demand-url", "", "Will make an HTTP request to the given URL, asking whether a host is allowed to have a certificate issued.") deployCommand.cmd.Flags().BoolVar(&deployCommand.tlsStaging, "tls-staging", false, "Use Let's Encrypt staging environment for certificate provisioning") deployCommand.cmd.Flags().StringVar(&deployCommand.args.ServiceOptions.TLSCertificatePath, "tls-certificate-path", "", "Configure custom TLS certificate path (PEM format)") deployCommand.cmd.Flags().StringVar(&deployCommand.args.ServiceOptions.TLSPrivateKeyPath, "tls-private-key-path", "", "Configure custom TLS private key path (PEM format)") diff --git a/internal/server/service.go b/internal/server/service.go index 5636ae8..11e05ac 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -1,13 +1,16 @@ package server import ( + "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" + "fmt" "log/slog" "net" "net/http" + "net/url" "os" "path" "strings" @@ -66,6 +69,7 @@ type HealthCheckConfig struct { type ServiceOptions struct { TLSEnabled bool `json:"tls_enabled"` + TLSOnDemandUrl string `json:"tls_on_demand_url"` TLSCertificatePath string `json:"tls_certificate_path"` TLSPrivateKeyPath string `json:"tls_private_key_path"` ACMEDirectory string `json:"acme_directory"` @@ -313,19 +317,86 @@ func (s *Service) createCertManager(hosts []string, options ServiceOptions) (Cer // Ensure we're not trying to use Let's Encrypt to fetch a wildcard domain, // as that is not supported with the challenge types that we use. for _, host := range hosts { - if strings.Contains(host, "*") { + if strings.Contains(host, "*") && options.TLSOnDemandUrl == "" { return nil, ErrorAutomaticTLSDoesNotSupportWildcards } } + // // TODO: + // // - create a func instead to return the host policy + // // - create the function calling the on demand url + // var hostPolicy = autocert.HostWhitelist(hosts...) + + // // https://stackoverflow.com/questions/52129908/can-i-have-a-dynamic-host-policty-with-autocert + + // // Wildcard hosts!!! we can handle them now! + // if len(hosts) == 0 && options.TLSOnDemandUrl != "" { + // fmt.Println("🚀🚀🚀 Registering a custom hostPolicy for", hosts) + // hostPolicy = func(ctx context.Context, host string) error { + // slog.Debug("Contacting", options.TLSOnDemandUrl, host) + + // resp, err := http.Get(fmt.Sprintf("%s?domain=%s", options.TLSOnDemandUrl, url.QueryEscape(host))) + + // if err != nil { + // // the TLS on demand URL is not reachable + // slog.Error("Unable to reach the TLS on demand URL", host, err) + // return err + // } + + // if resp.StatusCode != 200 && resp.StatusCode != 201 { + // return fmt.Errorf("%s is not allowed to get a certificate", host) + // } + + // return nil + // } + // } else { + // fmt.Println("😕😕😕", len(hosts), hosts, options.TLSOnDemandUrl) + // } + + hostPolicy, err := s.createAutoCertHostPolicy(hosts, options) + + if err != nil { + return nil, err + } + return &autocert.Manager{ Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(options.ScopedCachePath()), - HostPolicy: autocert.HostWhitelist(hosts...), + HostPolicy: hostPolicy, Client: &acme.Client{DirectoryURL: options.ACMEDirectory}, }, nil } +func (s *Service) createAutoCertHostPolicy(hosts []string, options ServiceOptions) (autocert.HostPolicy, error) { + onDemandTls := len(hosts) == 0 && options.TLSOnDemandUrl != "" + + if !onDemandTls { + return autocert.HostWhitelist(hosts...), nil + } + + _, err := url.ParseRequestURI(options.TLSOnDemandUrl) + + if err != nil { + slog.Error("Unable to parse the tls_on_demand_url URL") + return nil, err + } + + return func(ctx context.Context, host string) error { + resp, err := http.Get(fmt.Sprintf("%s?host=%s", options.TLSOnDemandUrl, url.QueryEscape(host))) + + if err != nil { + slog.Error("Unable to reach the TLS on demand URL", host, err) + return err + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s is not allowed to get a certificate", host) + } + + return nil + }, nil +} + func (s *Service) createMiddleware(options ServiceOptions, certManager CertManager) (http.Handler, error) { var err error var handler http.Handler = http.HandlerFunc(s.serviceRequestWithTarget) From 2a30eb856ee8e28d4b5626d1d7cdb826ef1e2b88 Mon Sep 17 00:00:00 2001 From: Did Date: Fri, 1 Nov 2024 23:12:42 +0000 Subject: [PATCH 2/8] remove debugging statements and old code --- internal/server/service.go | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/internal/server/service.go b/internal/server/service.go index 11e05ac..b4151d8 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -322,37 +322,6 @@ func (s *Service) createCertManager(hosts []string, options ServiceOptions) (Cer } } - // // TODO: - // // - create a func instead to return the host policy - // // - create the function calling the on demand url - // var hostPolicy = autocert.HostWhitelist(hosts...) - - // // https://stackoverflow.com/questions/52129908/can-i-have-a-dynamic-host-policty-with-autocert - - // // Wildcard hosts!!! we can handle them now! - // if len(hosts) == 0 && options.TLSOnDemandUrl != "" { - // fmt.Println("🚀🚀🚀 Registering a custom hostPolicy for", hosts) - // hostPolicy = func(ctx context.Context, host string) error { - // slog.Debug("Contacting", options.TLSOnDemandUrl, host) - - // resp, err := http.Get(fmt.Sprintf("%s?domain=%s", options.TLSOnDemandUrl, url.QueryEscape(host))) - - // if err != nil { - // // the TLS on demand URL is not reachable - // slog.Error("Unable to reach the TLS on demand URL", host, err) - // return err - // } - - // if resp.StatusCode != 200 && resp.StatusCode != 201 { - // return fmt.Errorf("%s is not allowed to get a certificate", host) - // } - - // return nil - // } - // } else { - // fmt.Println("😕😕😕", len(hosts), hosts, options.TLSOnDemandUrl) - // } - hostPolicy, err := s.createAutoCertHostPolicy(hosts, options) if err != nil { From 46d3d38a621a0b080a41da84d6994ee90037d3cd Mon Sep 17 00:00:00 2001 From: Did Date: Fri, 1 Nov 2024 23:19:42 +0000 Subject: [PATCH 3/8] don't use the wildcard anymore the on-demand tls feature --- internal/server/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/service.go b/internal/server/service.go index b4151d8..4ffadc8 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -317,7 +317,7 @@ func (s *Service) createCertManager(hosts []string, options ServiceOptions) (Cer // Ensure we're not trying to use Let's Encrypt to fetch a wildcard domain, // as that is not supported with the challenge types that we use. for _, host := range hosts { - if strings.Contains(host, "*") && options.TLSOnDemandUrl == "" { + if strings.Contains(host, "*") { return nil, ErrorAutomaticTLSDoesNotSupportWildcards } } From ec013eae70ebf9ec44ef08557ddae0c0a8cdd6ee Mon Sep 17 00:00:00 2001 From: Did Date: Fri, 1 Nov 2024 23:23:20 +0000 Subject: [PATCH 4/8] the on-demand URL must contain the http scheme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c22ec7a..8fea2f6 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ In addition of the automatic TLS functionality, Kamal Proxy can also dynamically from any host allowed by an external API endpoint of your choice. This avoids hard-coding hosts in the configuration, especially when you don't know the hosts at the startup. - kamal-proxy deploy service1 --target web-1:3000 --host "" --tls --tls-on-demand-url=localhost:4567/check + kamal-proxy deploy service1 --target web-1:3000 --host "" --tls --tls-on-demand-url="http://localhost:4567/check" The On-demand URL endpoint will have to answer a 200 HTTP status code. Kamal Proxy will call the on-demand URL with a query string of `?host=` containing the host received by Kamal Proxy. From f91ed90604a02de8aaa1e95b5abb594ab1d76c05 Mon Sep 17 00:00:00 2001 From: Didier Lafforgue Date: Sun, 3 Nov 2024 16:02:23 +0100 Subject: [PATCH 5/8] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8fea2f6..82091e4 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ applications. To enable this, add the `--tls` flag when deploying an instance: kamal-proxy deploy service1 --target web-1:3000 --host app1.example.com --tls -## On-demand TLS +### On-demand TLS In addition of the automatic TLS functionality, Kamal Proxy can also dynamically obtain a TLS certificate from any host allowed by an external API endpoint of your choice. From 2e0ef399e4f3dcc1fb7e7cbb8f217e3f522200f0 Mon Sep 17 00:00:00 2001 From: Didier Lafforgue Date: Sat, 30 Nov 2024 15:16:34 +0100 Subject: [PATCH 6/8] don't check for the len of hosts when evaluating the TLSOnDemandUrl --- internal/server/service.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/server/service.go b/internal/server/service.go index 4ffadc8..864d2f6 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -337,9 +337,9 @@ func (s *Service) createCertManager(hosts []string, options ServiceOptions) (Cer } func (s *Service) createAutoCertHostPolicy(hosts []string, options ServiceOptions) (autocert.HostPolicy, error) { - onDemandTls := len(hosts) == 0 && options.TLSOnDemandUrl != "" + slog.Info("createAutoCertHostPolicy called", options.TLSOnDemandUrl, len(hosts), "🚨", "ok") - if !onDemandTls { + if options.TLSOnDemandUrl == "" { return autocert.HostWhitelist(hosts...), nil } @@ -350,7 +350,11 @@ func (s *Service) createAutoCertHostPolicy(hosts []string, options ServiceOption return nil, err } + slog.Info("Will use the tls_on_demand_url URL") + return func(ctx context.Context, host string) error { + slog.Info("Get a certificate for", host, "🤞") + resp, err := http.Get(fmt.Sprintf("%s?host=%s", options.TLSOnDemandUrl, url.QueryEscape(host))) if err != nil { From c258d11b75a455375641de75e60082adf923a39b Mon Sep 17 00:00:00 2001 From: Didier Lafforgue Date: Tue, 3 Dec 2024 00:09:08 +0100 Subject: [PATCH 7/8] chore: lint the code --- internal/server/service.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/server/service.go b/internal/server/service.go index 864d2f6..2d9e6fe 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -323,7 +323,6 @@ func (s *Service) createCertManager(hosts []string, options ServiceOptions) (Cer } hostPolicy, err := s.createAutoCertHostPolicy(hosts, options) - if err != nil { return nil, err } @@ -337,14 +336,13 @@ func (s *Service) createCertManager(hosts []string, options ServiceOptions) (Cer } func (s *Service) createAutoCertHostPolicy(hosts []string, options ServiceOptions) (autocert.HostPolicy, error) { - slog.Info("createAutoCertHostPolicy called", options.TLSOnDemandUrl, len(hosts), "🚨", "ok") + slog.Info("createAutoCertHostPolicy called", "url", options.TLSOnDemandUrl) if options.TLSOnDemandUrl == "" { return autocert.HostWhitelist(hosts...), nil } _, err := url.ParseRequestURI(options.TLSOnDemandUrl) - if err != nil { slog.Error("Unable to parse the tls_on_demand_url URL") return nil, err @@ -353,10 +351,9 @@ func (s *Service) createAutoCertHostPolicy(hosts []string, options ServiceOption slog.Info("Will use the tls_on_demand_url URL") return func(ctx context.Context, host string) error { - slog.Info("Get a certificate for", host, "🤞") + slog.Info("Get a certificate", "host", host) resp, err := http.Get(fmt.Sprintf("%s?host=%s", options.TLSOnDemandUrl, url.QueryEscape(host))) - if err != nil { slog.Error("Unable to reach the TLS on demand URL", host, err) return err From 2e6581c90538baa02d31c1534a321a3681b6fde5 Mon Sep 17 00:00:00 2001 From: Didier Lafforgue Date: Tue, 3 Dec 2024 00:10:55 +0100 Subject: [PATCH 8/8] chore: remove a debug message --- internal/server/service.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/server/service.go b/internal/server/service.go index 2d9e6fe..089abaa 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -336,8 +336,6 @@ func (s *Service) createCertManager(hosts []string, options ServiceOptions) (Cer } func (s *Service) createAutoCertHostPolicy(hosts []string, options ServiceOptions) (autocert.HostPolicy, error) { - slog.Info("createAutoCertHostPolicy called", "url", options.TLSOnDemandUrl) - if options.TLSOnDemandUrl == "" { return autocert.HostWhitelist(hosts...), nil } @@ -348,7 +346,7 @@ func (s *Service) createAutoCertHostPolicy(hosts []string, options ServiceOption return nil, err } - slog.Info("Will use the tls_on_demand_url URL") + slog.Info("Will use the tls_on_demand_url URL", "url", options.TLSOnDemandUrl) return func(ctx context.Context, host string) error { slog.Info("Get a certificate", "host", host)