From ead0b2b4890f4f8c0a5e2733c8335fc6361dc59b Mon Sep 17 00:00:00 2001 From: Joe Shaw Date: Fri, 16 Oct 2015 15:35:52 -0400 Subject: [PATCH 1/2] add a Heroku-like maintenance mode Shuttle will return 503 Service Unavailable errors to clients without visiting backends if this is enabled. If a 503 error page is cached from backends, it will serve it. Shuttle side of litl/galaxy#123 --- admin_test.go | 52 +++++++++++++++++++++ client/config.go | 69 +++++++++++++++------------- http.go | 37 ++++++++------- registry.go | 2 +- service.go | 117 ++++++++++++++++++++++++++++------------------- shuttle_test.go | 3 +- 6 files changed, 182 insertions(+), 98 deletions(-) diff --git a/admin_test.go b/admin_test.go index 8566873..c017697 100644 --- a/admin_test.go +++ b/admin_test.go @@ -714,3 +714,55 @@ func (s *HTTPSuite) TestHTTPSRedirect(c *C) { } c.Assert(resp.StatusCode, Equals, http.StatusOK) } + +func (s *HTTPSuite) TestMaintenanceMode(c *C) { + mainServer := s.backendServers[0] + errServer := s.backendServers[1] + + svcCfg := client.ServiceConfig{ + Name: "VHostTest1", + Addr: "127.0.0.1:9000", + VirtualHosts: []string{"vhost1.test"}, + Backends: []client.BackendConfig{ + {Addr: mainServer.addr}, + }, + MaintenanceMode: true, + } + + if err := Registry.AddService(svcCfg); err != nil { + c.Fatal(err) + } + + // No error page is registered, so we should just get a 503 error with no body + checkHTTP("https://vhost1.test:"+s.httpsPort+"/addr", "vhost1.test", "", 503, c) + + // Use another backend to provide the error page + svcCfg.ErrorPages = map[string][]int{ + "http://" + errServer.addr + "/error?code=503": []int{503}, + } + + if err := Registry.UpdateService(svcCfg); err != nil { + c.Fatal(err) + } + + // Get a 503 error with the cached body + checkHTTP("https://vhost1.test:"+s.httpsPort+"/addr", "vhost1.test", errServer.addr, 503, c) + + // Turn maintenance mode off + svcCfg.MaintenanceMode = false + + if err := Registry.UpdateService(svcCfg); err != nil { + c.Fatal(err) + } + + checkHTTP("https://vhost1.test:"+s.httpsPort+"/addr", "vhost1.test", mainServer.addr, 200, c) + + // Turn it back on + svcCfg.MaintenanceMode = true + + if err := Registry.UpdateService(svcCfg); err != nil { + c.Fatal(err) + } + + checkHTTP("https://vhost1.test:"+s.httpsPort+"/addr", "vhost1.test", errServer.addr, 503, c) +} diff --git a/client/config.go b/client/config.go index f23dd87..b84d06c 100644 --- a/client/config.go +++ b/client/config.go @@ -211,6 +211,10 @@ type ServiceConfig struct { // Backends is a list of all servers handling connections for this service. Backends []BackendConfig `json:"backends,omitempty"` + + // Maintenance mode is a flag to return 503 status codes to clients + // without visiting backends. + MaintenanceMode bool `json:"maintenance_mode"` } // Return a copy of ServiceConfig with any unset fields to their default @@ -285,53 +289,56 @@ func (b *ServiceConfig) String() string { return string(b.Marshal()) } -// Update any unset fields with those from the supplied config. -// FIXME: HTTPSRedirect won't be turned off. Maybe change it to *bool? -func (s *ServiceConfig) Merge(cfg ServiceConfig) { +// Create a new config by merging the values from the current config +// with those set in the new config +func (s ServiceConfig) Merge(cfg ServiceConfig) ServiceConfig { + new := s + // let's try not to change the name - s.Name = cfg.Name + new.Name = cfg.Name - if s.Addr == "" { - s.Addr = cfg.Addr + if cfg.Addr != "" { + new.Addr = cfg.Addr } - if s.Network == "" { - s.Network = cfg.Network + if cfg.Network != "" { + new.Network = cfg.Network } - if s.Balance == "" { - s.Balance = cfg.Balance + if cfg.Balance != "" { + new.Balance = cfg.Balance } - if s.CheckInterval == 0 { - s.CheckInterval = cfg.CheckInterval + if cfg.CheckInterval != 0 { + new.CheckInterval = cfg.CheckInterval } - if s.Fall == 0 { - s.Fall = cfg.Fall + if cfg.Fall != 0 { + new.Fall = cfg.Fall } - if s.Rise == 0 { - s.Rise = cfg.Rise + if cfg.Rise != 0 { + new.Rise = cfg.Rise } - if s.ClientTimeout == 0 { - s.ClientTimeout = cfg.ClientTimeout + if cfg.ClientTimeout != 0 { + new.ClientTimeout = cfg.ClientTimeout } - if s.ServerTimeout == 0 { - s.ServerTimeout = cfg.ServerTimeout + if cfg.ServerTimeout != 0 { + new.ServerTimeout = cfg.ServerTimeout } - if s.DialTimeout == 0 { - s.DialTimeout = cfg.DialTimeout + if cfg.DialTimeout != 0 { + new.DialTimeout = cfg.DialTimeout } - if cfg.HTTPSRedirect { - s.HTTPSRedirect = cfg.HTTPSRedirect + if cfg.VirtualHosts != nil { + new.VirtualHosts = cfg.VirtualHosts } - if s.VirtualHosts == nil { - s.VirtualHosts = cfg.VirtualHosts + if cfg.ErrorPages != nil { + new.ErrorPages = cfg.ErrorPages } - if s.ErrorPages == nil { - s.ErrorPages = cfg.ErrorPages + if cfg.Backends != nil { + new.Backends = cfg.Backends } - if s.Backends == nil { - s.Backends = cfg.Backends - } + new.HTTPSRedirect = cfg.HTTPSRedirect + new.MaintenanceMode = cfg.MaintenanceMode + + return new } diff --git a/http.go b/http.go index 0a84f5f..3bbdaa8 100644 --- a/http.go +++ b/http.go @@ -401,36 +401,35 @@ func (e *ErrorResponse) CheckResponse(pr *ProxyRequest) bool { return true } +func logRequest(req *http.Request, statusCode int, backend string, proxyError error, duration time.Duration) { + id := req.Header.Get("X-Request-Id") + method := req.Method + url := req.Host + req.RequestURI + agent := req.UserAgent() + + clientIP := req.Header.Get("X-Forwarded-For") + if clientIP == "" { + clientIP = req.RemoteAddr + } + + errStr := fmt.Sprintf("%v", proxyError) + fmtStr := "id=%s method=%s client-ip=%s url=%s backend=%s status=%d duration=%s agent=%s, err=%s" + log.Printf(fmtStr, id, method, clientIP, url, backend, statusCode, duration, agent, errStr) +} + func logProxyRequest(pr *ProxyRequest) bool { // TODO: we may to be able to switch this off if pr == nil || pr.Request == nil { return true } - var id, method, clientIP, url, backend, agent string - var status int - duration := pr.FinishTime.Sub(pr.StartTime) - id = pr.Request.Header.Get("X-Request-Id") - method = pr.Request.Method - url = pr.Request.Host + pr.Request.RequestURI - agent = pr.Request.UserAgent() - status = pr.Response.StatusCode - - clientIP = pr.Request.Header.Get("X-Forwarded-For") - if clientIP == "" { - clientIP = pr.Request.RemoteAddr - } - + var backend string if pr.Response != nil && pr.Response.Request != nil && pr.Response.Request.URL != nil { backend = pr.Response.Request.URL.Host } - err := fmt.Sprintf("%v", pr.ProxyError) - - fmtStr := "id=%s method=%s client-ip=%s url=%s backend=%s status=%d duration=%s agent=%s, err=%s" - - log.Printf(fmtStr, id, method, clientIP, url, backend, status, duration, agent, err) + logRequest(pr.Request, pr.Response.StatusCode, backend, pr.ProxyError, duration) return true } diff --git a/registry.go b/registry.go index b8a7100..18d4488 100644 --- a/registry.go +++ b/registry.go @@ -293,7 +293,7 @@ func (s *ServiceRegistry) UpdateService(newCfg client.ServiceConfig) error { } currentCfg := service.Config() - newCfg.Merge(currentCfg) + newCfg = currentCfg.Merge(newCfg) if err := service.UpdateConfig(newCfg); err != nil { return err diff --git a/service.go b/service.go index f673758..c21edfc 100644 --- a/service.go +++ b/service.go @@ -23,25 +23,26 @@ var ( type Service struct { sync.Mutex - Name string - Addr string - HTTPSRedirect bool - VirtualHosts []string - Backends []*Backend - Balance string - CheckInterval int - Fall int - Rise int - ClientTimeout time.Duration - ServerTimeout time.Duration - DialTimeout time.Duration - Sent int64 - Rcvd int64 - Errors int64 - HTTPConns int64 - HTTPErrors int64 - HTTPActive int64 - Network string + Name string + Addr string + HTTPSRedirect bool + VirtualHosts []string + Backends []*Backend + Balance string + CheckInterval int + Fall int + Rise int + ClientTimeout time.Duration + ServerTimeout time.Duration + DialTimeout time.Duration + Sent int64 + Rcvd int64 + Errors int64 + HTTPConns int64 + HTTPErrors int64 + HTTPActive int64 + Network string + MaintenanceMode bool // Next returns the backends in priority order. next func() []*Backend @@ -93,20 +94,21 @@ type ServiceStat struct { // Create a Service from a config struct func NewService(cfg client.ServiceConfig) *Service { s := &Service{ - Name: cfg.Name, - Addr: cfg.Addr, - Balance: cfg.Balance, - CheckInterval: cfg.CheckInterval, - Fall: cfg.Fall, - Rise: cfg.Rise, - HTTPSRedirect: cfg.HTTPSRedirect, - VirtualHosts: cfg.VirtualHosts, - ClientTimeout: time.Duration(cfg.ClientTimeout) * time.Millisecond, - ServerTimeout: time.Duration(cfg.ServerTimeout) * time.Millisecond, - DialTimeout: time.Duration(cfg.DialTimeout) * time.Millisecond, - errorPages: NewErrorResponse(cfg.ErrorPages), - errPagesCfg: cfg.ErrorPages, - Network: cfg.Network, + Name: cfg.Name, + Addr: cfg.Addr, + Balance: cfg.Balance, + CheckInterval: cfg.CheckInterval, + Fall: cfg.Fall, + Rise: cfg.Rise, + HTTPSRedirect: cfg.HTTPSRedirect, + VirtualHosts: cfg.VirtualHosts, + ClientTimeout: time.Duration(cfg.ClientTimeout) * time.Millisecond, + ServerTimeout: time.Duration(cfg.ServerTimeout) * time.Millisecond, + DialTimeout: time.Duration(cfg.DialTimeout) * time.Millisecond, + errorPages: NewErrorResponse(cfg.ErrorPages), + errPagesCfg: cfg.ErrorPages, + Network: cfg.Network, + MaintenanceMode: cfg.MaintenanceMode, } // TODO: insert this into the backends too @@ -180,6 +182,7 @@ func (s *Service) UpdateConfig(cfg client.ServiceConfig) error { s.ServerTimeout = time.Duration(cfg.ServerTimeout) * time.Millisecond s.DialTimeout = time.Duration(cfg.DialTimeout) * time.Millisecond s.HTTPSRedirect = cfg.HTTPSRedirect + s.MaintenanceMode = cfg.MaintenanceMode if s.Balance != cfg.Balance { s.Balance = cfg.Balance @@ -242,19 +245,20 @@ func (s *Service) Config() client.ServiceConfig { func (s *Service) config() client.ServiceConfig { config := client.ServiceConfig{ - Name: s.Name, - Addr: s.Addr, - VirtualHosts: s.VirtualHosts, - HTTPSRedirect: s.HTTPSRedirect, - Balance: s.Balance, - CheckInterval: s.CheckInterval, - Fall: s.Fall, - Rise: s.Rise, - ClientTimeout: int(s.ClientTimeout / time.Millisecond), - ServerTimeout: int(s.ServerTimeout / time.Millisecond), - DialTimeout: int(s.DialTimeout / time.Millisecond), - ErrorPages: s.errPagesCfg, - Network: s.Network, + Name: s.Name, + Addr: s.Addr, + VirtualHosts: s.VirtualHosts, + HTTPSRedirect: s.HTTPSRedirect, + Balance: s.Balance, + CheckInterval: s.CheckInterval, + Fall: s.Fall, + Rise: s.Rise, + ClientTimeout: int(s.ClientTimeout / time.Millisecond), + ServerTimeout: int(s.ServerTimeout / time.Millisecond), + DialTimeout: int(s.DialTimeout / time.Millisecond), + ErrorPages: s.errPagesCfg, + Network: s.Network, + MaintenanceMode: s.MaintenanceMode, } for _, b := range s.Backends { config.Backends = append(config.Backends, b.Config()) @@ -452,6 +456,10 @@ func (s *Service) Available() int { s.Lock() defer s.Unlock() + if s.MaintenanceMode { + return 0 + } + available := 0 for _, b := range s.Backends { if b.Up() { @@ -578,6 +586,23 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } + if s.MaintenanceMode { + // TODO: Should we increment HTTPErrors here as well? + logRequest(r, http.StatusServiceUnavailable, "", nil, 0) + errPage := s.errorPages.Get(http.StatusServiceUnavailable) + if errPage != nil { + headers := w.Header() + for key, val := range errPage.Header() { + headers[key] = val + } + } + w.WriteHeader(http.StatusServiceUnavailable) + if errPage != nil { + w.Write(errPage.Body()) + } + return + } + s.httpProxy.ServeHTTP(w, r, s.NextAddrs()) } diff --git a/shuttle_test.go b/shuttle_test.go index ebb5012..27ad911 100644 --- a/shuttle_test.go +++ b/shuttle_test.go @@ -5,6 +5,7 @@ import ( "io" "io/ioutil" "net" + "os" "sync" "testing" "time" @@ -15,7 +16,7 @@ import ( ) func init() { - debug = false + debug = os.Getenv("SHUTTLE_DEBUG") == "1" if debug { log.DefaultLogger.Level = log.DEBUG From deee3b7d48bb58b58f3d1edda6a8a33550b19de8 Mon Sep 17 00:00:00 2001 From: Joe Shaw Date: Fri, 16 Oct 2015 16:43:02 -0400 Subject: [PATCH 2/2] add /{service}/_config endpoint --- admin.go | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/admin.go b/admin.go index 7f55ea1..457ff33 100644 --- a/admin.go +++ b/admin.go @@ -26,7 +26,7 @@ func getStats(w http.ResponseWriter, r *http.Request) { w.Write(marshal(Registry.Stats())) } -func getService(w http.ResponseWriter, r *http.Request) { +func getServiceStats(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) serviceStats, err := Registry.ServiceStats(vars["service"]) @@ -38,6 +38,18 @@ func getService(w http.ResponseWriter, r *http.Request) { w.Write(marshal(serviceStats)) } +func getServiceConfig(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + serviceStats, err := Registry.ServiceConfig(vars["service"]) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Write(marshal(serviceStats)) +} + // Update the global config func postConfig(w http.ResponseWriter, r *http.Request) { cfg := client.Config{} @@ -120,6 +132,20 @@ func deleteService(w http.ResponseWriter, r *http.Request) { w.Write(marshal(Registry.Config())) } +func getBackendStats(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + serviceName := vars["service"] + backendName := vars["backend"] + + backend, err := Registry.BackendStats(serviceName, backendName) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Write(marshal(backend)) +} + func getBackend(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) serviceName := vars["service"] @@ -187,7 +213,9 @@ func addHandlers() { r.HandleFunc("/_config", getConfig).Methods("GET") r.HandleFunc("/_config", postConfig).Methods("PUT", "POST") r.HandleFunc("/_stats", getStats).Methods("GET") - r.HandleFunc("/{service}", getService).Methods("GET") + r.HandleFunc("/{service}", getServiceStats).Methods("GET") + r.HandleFunc("/{service}/_config", getServiceConfig).Methods("GET") + r.HandleFunc("/{service}/_stats", getServiceStats).Methods("GET") r.HandleFunc("/{service}", postService).Methods("PUT", "POST") r.HandleFunc("/{service}", deleteService).Methods("DELETE") r.HandleFunc("/{service}/{backend}", getBackend).Methods("GET")