From 7c3c7364c203e851c9a1666e2ddc7f177a44594a Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Mon, 25 Nov 2024 09:40:57 +0530 Subject: [PATCH 01/29] initial commit to move data to datamanager --- src/datamanager.go | 363 ++++++++++++++++++++++++++++++++++++++++++ src/drove.go | 232 ++++++++++++++++++++------- src/nginx-header.tmpl | 15 +- src/nixy.go | 110 +++++++------ src/nixy.toml | 28 ++-- 5 files changed, 621 insertions(+), 127 deletions(-) create mode 100644 src/datamanager.go diff --git a/src/datamanager.go b/src/datamanager.go new file mode 100644 index 0000000..12fd552 --- /dev/null +++ b/src/datamanager.go @@ -0,0 +1,363 @@ +package main + +import ( + "fmt" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +type DroveConfig struct { + Name string + Drove []string `json:"-"` + User string `json:"-"` + Pass string `json:"-"` + AccessToken string `json:"-" toml:"access_token"` + Realm string + RealmSuffix string `json:"-" toml:"realm_suffix"` + RoutingTag string `json:"-" toml:"routing_tag"` + LeaderVHost string `json:"-" toml:"leader_vhost"` +} + +// NamespaceData holds the data and metadata for each namespace +type NamespaceData struct { + Drove DroveConfig + Leader LeaderController + Apps map[string]App + KnownVHosts Vhosts + Timestamp time.Time // Timestamp of creation or modification +} + +type StaticConfig struct { + Xproxy string + LeftDelimiter string `json:"-" toml:"left_delimiter"` + RightDelimiter string `json:"-" toml:"right_delimiter"` + MaxFailsUpstream *int `json:"max_fails,omitempty"` + FailTimeoutUpstream string `json:"fail_timeout,omitempty"` + SlowStartUpstream string `json:"slow_start,omitempty"` +} + +// DataManager manages namespaces and data for those namespaces +type DataManager struct { + mu sync.RWMutex // Mutex for concurrency control + namespaces map[string]NamespaceData // Map of namespaces to NamespaceData + StaticData StaticConfig +} + +// NewDataManager creates a new instance of DataManager +func NewDataManager(inXproxy string, inLeftDelimiter string, inRightDelimiter string, + inMaxFailsUpstream *int, inFailTimeoutUpstream string, inSlowStartUpstream string) *DataManager { + return &DataManager{ + namespaces: make(map[string]NamespaceData), + StaticData: StaticConfig{Xproxy: inXproxy, LeftDelimiter: inLeftDelimiter, RightDelimiter: inRightDelimiter, + MaxFailsUpstream: inMaxFailsUpstream, FailTimeoutUpstream: inFailTimeoutUpstream, SlowStartUpstream: inSlowStartUpstream}, + } +} + +// Create inserts data into a namespace +func (dm *DataManager) CreateNamespace(namespace string, inDrove []string, inUser string, inPass string, inAccessToken string, + inRealm string, inRealmSuffix string, inRoutingTag string, inLeaderVhost string) error { + dm.mu.Lock() // Lock to ensure concurrent writes are handled + defer dm.mu.Unlock() // Ensure the lock is always released + + // Start the operation log + logger.WithFields(logrus.Fields{ + "operation": "create", + "namespace": namespace, + }).Info("Attempting to create Namespace") + + // Ensure namespace exists + if _, exists := dm.namespaces[namespace]; !exists { + dm.namespaces[namespace] = NamespaceData{ + Drove: DroveConfig{ + Name: namespace, + Drove: inDrove, + User: inUser, + Pass: inPass, + AccessToken: inAccessToken, + Realm: inRealm, + RealmSuffix: inRealmSuffix, + RoutingTag: inRoutingTag, + LeaderVHost: inLeaderVhost, + }, + Timestamp: time.Now(), + } + } + + // Log success + logger.WithFields(logrus.Fields{ + "operation": "create", + "namespace": namespace, + }).Info("Namespace created successfully") + return nil +} + +func (dm *DataManager) ReadDroveConfig(namespace string) (DroveConfig, error) { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadDroveConfig" + + // Ensure namespace exists + if _, exists := dm.namespaces[namespace]; !exists { + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + }).Error("NamespaceData read failed") + err := fmt.Errorf("namespace '%s' not found", namespace) + return DroveConfig{}, err + } + + ns := dm.namespaces[namespace] + + // Log success + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + }).Debug("ReadDroveConfig successfully") + + return ns.Drove, nil //returning copy +} + +func (dm *DataManager) ReadLeader(namespace string) (LeaderController, error) { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadLeader" + + // Ensure namespace exists + if _, exists := dm.namespaces[namespace]; !exists { + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + }).Error("NamespaceData read failed") + err := fmt.Errorf("namespace '%s' not found", namespace) + return LeaderController{}, err + } + + ns := dm.namespaces[namespace] + + // Log success + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + "leader": ns.Leader, + "ns": ns, + }).Info("ReadLeader successfully") + + return ns.Leader, nil //returning copy +} + +func (dm *DataManager) ReadAllLeaders() []LeaderController { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadAllLeaders" + + // Start the operation log + logger.WithFields(logrus.Fields{ + "operation": operation, + }).Debug("Attempting to read namespace data") + + var allLeaders []LeaderController + + for _, data := range dm.namespaces { + allLeaders = append(allLeaders, data.Leader) + } + // Log success + logger.WithFields(logrus.Fields{ + "operation": operation, + "allLeader": allLeaders, + }).Info("ReadAllLeaders successfully") + + return allLeaders //returning copy +} + +func (dm *DataManager) UpdateLeader(namespace string, leader LeaderController) error { + dm.mu.Lock() // Read lock to allow multiple concurrent reads + defer dm.mu.Unlock() // Ensure the lock is always released + operation := "UpdateLeader" + + // Ensure namespace exists + if _, exists := dm.namespaces[namespace]; !exists { + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + }).Error("NamespaceData read failed") + err := fmt.Errorf("namespace '%s' not found", namespace) + return err + } + + ns := dm.namespaces[namespace] + ns.Leader = leader + ns.Timestamp = time.Now() // Update timestamp on modification + dm.namespaces[namespace] = ns + + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + "leader": ns.Leader, + "time": ns.Timestamp, + }).Info("UpdateLeader data successfully") + return nil +} + +func (dm *DataManager) ReadApp(namespace string) (map[string]App, error) { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadLeader" + + // Ensure namespace exists + if _, exists := dm.namespaces[namespace]; !exists { + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + }).Error("NamespaceData read failed") + err := fmt.Errorf("namespace '%s' not found", namespace) + return map[string]App{}, err + } + + ns := dm.namespaces[namespace] + + // Log success + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + "apps": ns.Apps, + }).Info("ReadApp successfully") + + return ns.Apps, nil //returning copy +} + +func (dm *DataManager) ReadAllApps() map[string]App { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadAllApps" + + allApps := make(map[string]App) + + for _, data := range dm.namespaces { + for appId, appData := range data.Apps { + allApps[appId] = appData + } + } + // Log success + logger.WithFields(logrus.Fields{ + "operation": operation, + "allApps": allApps, + }).Info("ReadAllApps successfully") + + return allApps //returning copy + +} + +func (dm *DataManager) UpdateApps(namespace string, apps map[string]App) error { + dm.mu.Lock() // Read lock to allow multiple concurrent reads + defer dm.mu.Unlock() // Ensure the lock is always released + operation := "UpdateApps" + + // Ensure namespace exists + if _, exists := dm.namespaces[namespace]; !exists { + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + }).Error("NamespaceData read failed") + err := fmt.Errorf("namespace '%s' not found", namespace) + return err + } + + ns := dm.namespaces[namespace] + ns.Apps = apps + ns.Timestamp = time.Now() // Update timestamp on modification + dm.namespaces[namespace] = ns + + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + "apps": dm.namespaces[namespace].Apps, + "time": dm.namespaces[namespace].Timestamp, + }).Info("UpdateApps successfully") + return nil +} + +func (dm *DataManager) ReadKnownVhosts(namespace string) (Vhosts, error) { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadKnownVhosts" + + // Ensure namespace exists + if _, exists := dm.namespaces[namespace]; !exists { + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + }).Error("NamespaceData read failed") + err := fmt.Errorf("namespace '%s' not found", namespace) + return Vhosts{}, err + } + + ns := dm.namespaces[namespace] + + // Log success + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + "knownVHosts": ns.KnownVHosts, + }).Info("ReadKnownVhosts successfully") + + return ns.KnownVHosts, nil //returning copy +} + +func (dm *DataManager) UpdateKnownVhosts(namespace string, KnownVHosts Vhosts) error { + dm.mu.Lock() // Read lock to allow multiple concurrent reads + defer dm.mu.Unlock() // Ensure the lock is always released + operation := "UpdateKnownVhosts" + + // Ensure namespace exists + if _, exists := dm.namespaces[namespace]; !exists { + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + }).Error("NamespaceData read failed") + err := fmt.Errorf("namespace '%s' not found", namespace) + return err + } + + ns := dm.namespaces[namespace] + ns.KnownVHosts = KnownVHosts + ns.Timestamp = time.Now() // Update timestamp on modification + dm.namespaces[namespace] = ns + + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + "knownHosts": dm.namespaces[namespace].KnownVHosts, + "time": dm.namespaces[namespace].Timestamp, + }).Info("UpdateKnownVhosts successfully") + return nil +} + +func (dm *DataManager) ReadAllNamespace() map[string]NamespaceData { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadAllNamespace" + + // Start the operation log + logger.WithFields(logrus.Fields{ + "operation": operation, + }).Info("ReadAllNamespace data successfully") + + return dm.namespaces //returning copy +} + +// Read retrieves data from a specific namespace +func (dm *DataManager) ReadStaticData() StaticConfig { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadStaticData" + + // Start the operation log + logger.WithFields(logrus.Fields{ + "operation": operation, + "staticData": dm.StaticData, + }).Info("ReadStaticData successfully") + + return dm.StaticData //returning copy +} diff --git a/src/drove.go b/src/drove.go index 55281e3..97d323d 100644 --- a/src/drove.go +++ b/src/drove.go @@ -22,6 +22,24 @@ import ( "github.com/sirupsen/logrus" ) +type NamespaceRenderingData struct { + LeaderVHost string `json:"-" toml:"leader_vhost"` + Leader LeaderController + Apps map[string]App + RoutingTag string `json:"-" toml:"routing_tag"` + KnownVHosts Vhosts +} + +type TemplateRenderingData struct { + Xproxy string + LeftDelimiter string `json:"-" toml:"left_delimiter"` + RightDelimiter string `json:"-" toml:"right_delimiter"` + MaxFailsUpstream *int `json:"max_fails,omitempty"` + FailTimeoutUpstream string `json:"fail_timeout,omitempty"` + SlowStartUpstream string `json:"slow_start,omitempty"` + Namespaces map[string]NamespaceRenderingData `json:"namespaces"` +} + // DroveApps struct for our apps nested with tasks. type DroveApps struct { Status string `json:"status"` @@ -83,10 +101,20 @@ func leaderController(endpoint string) *LeaderController { return controllerHost } -func fetchRecentEvents(client *http.Client, syncPoint *CurrSyncPoint) (*DroveEventSummary, error) { +func fetchRecentEvents(client *http.Client, syncPoint *CurrSyncPoint, namespace string) (*DroveEventSummary, error) { + + droveConfig, err := db.ReadDroveConfig(namespace) + if err != nil { + logger.WithFields(logrus.Fields{ + "namespace": namespace, + }).Error("Error loading drove config") + err := errors.New("Error loading Drove Config") + return nil, err + } + var endpoint string for _, es := range health.Endpoints { - if es.Healthy { + if es.Healthy && es.Namespace == namespace { endpoint = es.Endpoint break } @@ -95,20 +123,18 @@ func fetchRecentEvents(client *http.Client, syncPoint *CurrSyncPoint) (*DroveEve err := errors.New("all endpoints are down") return nil, err } - //if config.apiTimeout != 0 { - // timeout = config.apiTimeout - //} + // fetch all apps and tasks with a single request. req, err := http.NewRequest("GET", endpoint+"/apis/v1/cluster/events/summary?lastSyncTime="+fmt.Sprint(syncPoint.LastSyncTime), nil) if err != nil { return nil, err } req.Header.Set("Accept", "application/json") - if config.User != "" { - req.SetBasicAuth(config.User, config.Pass) + if droveConfig.User != "" { + req.SetBasicAuth(droveConfig.User, droveConfig.Pass) } - if config.AccessToken != "" { - req.Header.Add("Authorization", config.AccessToken) + if droveConfig.AccessToken != "" { + req.Header.Add("Authorization", droveConfig.AccessToken) } resp, err := client.Do(req) if err != nil { @@ -123,7 +149,8 @@ func fetchRecentEvents(client *http.Client, syncPoint *CurrSyncPoint) (*DroveEve return nil, err } logger.WithFields(logrus.Fields{ - "data": newEventsApiResponse, + "data": newEventsApiResponse, + "namespace": namespace, }).Debug("events response") if newEventsApiResponse.Status != "SUCCESS" { return nil, errors.New("Events api call failed. Message: " + newEventsApiResponse.Message) @@ -133,10 +160,10 @@ func fetchRecentEvents(client *http.Client, syncPoint *CurrSyncPoint) (*DroveEve return &(newEventsApiResponse.EventSummary), nil } -func refreshLeaderData() { +func refreshLeaderData(namespace string) { var endpoint string for _, es := range health.Endpoints { - if es.Healthy { + if es.Namespace == namespace && es.Healthy { endpoint = es.Endpoint break } @@ -146,25 +173,38 @@ func refreshLeaderData() { go countAllEndpointsDownErrors.Inc() return } - if endpoint != config.Leader.Endpoint { + currentLeader, err := db.ReadLeader(namespace) + if err != nil { + logger.Error("Error while reading current leader for namespace" + namespace) + return + } + if endpoint != currentLeader.Endpoint { logger.WithFields(logrus.Fields{ "new": endpoint, - "old": config.Leader.Endpoint, + "old": currentLeader.Endpoint, }).Info("Looks like master shifted. Will resync app") var newLeader = leaderController(endpoint) if newLeader != nil { - config.Leader = *newLeader + err = db.UpdateLeader(namespace, *newLeader) + if err != nil { + logger.WithFields(logrus.Fields{ + "leader": currentLeader, + "newLeader": newLeader, + "namespace": namespace, + }).Error("Error seting new leader") + } logger.WithFields(logrus.Fields{ - "leader": config.Leader, + "leader": currentLeader, + "newLeader": newLeader, }).Info("New leader being set") - reloadAllApps(true) + reloadAllApps(namespace, true) } else { logrus.Warn("Leade struct generation failed") } } } -func pollEvents() { +func pollEvents(namespace string) { go func() { client := &http.Client{ Timeout: 0 * time.Second, @@ -186,16 +226,18 @@ func pollEvents() { }).Debug("Syncing...") syncData.Lock() defer syncData.Unlock() - refreshLeaderData() - eventSummary, err := fetchRecentEvents(client, &syncData) + refreshLeaderData(namespace) + eventSummary, err := fetchRecentEvents(client, &syncData, namespace) if err != nil { logger.WithFields(logrus.Fields{ - "error": err.Error(), + "error": err.Error(), + "namespace": namespace, }).Error("unable to sync events from drove") } else { if len(eventSummary.EventsCount) > 0 { logger.WithFields(logrus.Fields{ "events": eventSummary.EventsCount, + "namespace": namespace, "localTime": time.Now(), }).Info("Events received") reloadNeeded := false @@ -216,7 +258,8 @@ func pollEvents() { } } else { logger.WithFields(logrus.Fields{ - "events": eventSummary.EventsCount, + "events": eventSummary.EventsCount, + "namespace": namespace, }).Debug("New Events received") } } @@ -266,11 +309,21 @@ func endpointHealth() { health.Endpoints[i].Message = err.Error() continue } - if config.User != "" { - req.SetBasicAuth(config.User, config.Pass) + droveConfig, err := db.ReadDroveConfig(es.Namespace) + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err, + "endpoint": es.Endpoint, + }).Error("an error occurred reading drove config for health request") + health.Endpoints[i].Healthy = false + health.Endpoints[i].Message = err.Error() + continue + } + if droveConfig.User != "" { + req.SetBasicAuth(droveConfig.User, droveConfig.Pass) } - if config.AccessToken != "" { - req.Header.Add("Authorization", config.AccessToken) + if droveConfig.AccessToken != "" { + req.Header.Add("Authorization", droveConfig.AccessToken) } resp, err := client.Do(req) if err != nil { @@ -298,11 +351,11 @@ func endpointHealth() { }() } -func fetchApps(jsonapps *DroveApps) error { +func fetchApps(droveConfig DroveConfig, jsonapps *DroveApps) error { var endpoint string var timeout int = 5 for _, es := range health.Endpoints { - if es.Healthy { + if es.Healthy && droveConfig.Name == es.Namespace { endpoint = es.Endpoint break } @@ -327,11 +380,11 @@ func fetchApps(jsonapps *DroveApps) error { return err } req.Header.Set("Accept", "application/json") - if config.User != "" { - req.SetBasicAuth(config.User, config.Pass) + if droveConfig.User != "" { + req.SetBasicAuth(droveConfig.User, droveConfig.Pass) } - if config.AccessToken != "" { - req.Header.Add("Authorization", config.AccessToken) + if droveConfig.AccessToken != "" { + req.Header.Add("Authorization", droveConfig.AccessToken) } resp, err := client.Do(req) if err != nil { @@ -358,13 +411,13 @@ func matchingVhost(vHost string, realms []string) bool { return false } -func syncApps(jsonapps *DroveApps, vhosts *Vhosts) bool { +func syncApps(droveConfig DroveConfig, jsonapps *DroveApps, vhosts *Vhosts) bool { config.Lock() defer config.Unlock() apps := make(map[string]App) var realms = []string{} - if len(config.Realm) > 0 { - realms = strings.Split(config.Realm, ",") + if len(droveConfig.Realm) > 0 { + realms = strings.Split(droveConfig.Realm, ",") } for _, app := range jsonapps.Apps { var newapp = App{} @@ -377,17 +430,17 @@ func syncApps(jsonapps *DroveApps, vhosts *Vhosts) bool { } // Lets ignore apps if no instances are available if len(newapp.Hosts) > 0 { - var toAppend = matchingVhost(app.Vhost, realms) || (len(config.RealmSuffix) > 0 && strings.HasSuffix(app.Vhost, config.RealmSuffix)) + var toAppend = matchingVhost(app.Vhost, realms) || (len(droveConfig.RealmSuffix) > 0 && strings.HasSuffix(app.Vhost, droveConfig.RealmSuffix)) if toAppend { vhosts.Vhosts[app.Vhost] = true newapp.ID = app.Vhost newapp.Vhost = app.Vhost var groupName = app.Vhost - if len(config.RoutingTag) > 0 { - if tagValue, ok := app.Tags[config.RoutingTag]; ok { + if len(droveConfig.RoutingTag) > 0 { + if tagValue, ok := app.Tags[droveConfig.RoutingTag]; ok { logger.WithFields(logrus.Fields{ - "tag": config.RoutingTag, + "tag": droveConfig.RoutingTag, "vhost": newapp.Vhost, "value": tagValue, }).Info("routing tag found") @@ -395,7 +448,7 @@ func syncApps(jsonapps *DroveApps, vhosts *Vhosts) bool { } else { logger.WithFields(logrus.Fields{ - "tag": config.RoutingTag, + "tag": droveConfig.RoutingTag, "vhost": newapp.Vhost, }).Debug("no routing tag found") } @@ -429,33 +482,73 @@ func syncApps(jsonapps *DroveApps, vhosts *Vhosts) bool { apps[app.Vhost] = newapp } else { logger.WithFields(logrus.Fields{ - "realm": config.Realm, + "realm": droveConfig.Realm, "vhost": app.Vhost, }).Warn("Host ignored due to realm mismath") } } } + + currentApps, er := db.ReadApp(droveConfig.Name) + if er != nil { + logger.Error("Error while reading apps for namespace" + droveConfig.Name) + //Continue to update with latest data + } + // Not all events bring changes, so lets see if anything is new. - eq := reflect.DeepEqual(apps, config.Apps) + eq := reflect.DeepEqual(apps, currentApps) if eq { return true } - config.Apps = apps + + err := db.UpdateApps(droveConfig.Name, apps) + if err != nil { + logger.Error("Error while updating apps for namespace" + droveConfig.Name) + return true + } return false } +func createTemplateData(templateData *TemplateRenderingData) { + namespaceData := db.ReadAllNamespace() + staticData := db.ReadStaticData() + + templateData.Xproxy = staticData.Xproxy + templateData.LeftDelimiter = staticData.LeftDelimiter + templateData.RightDelimiter = staticData.RightDelimiter + templateData.FailTimeoutUpstream = staticData.FailTimeoutUpstream + templateData.MaxFailsUpstream = staticData.MaxFailsUpstream + templateData.SlowStartUpstream = staticData.SlowStartUpstream + + templateData.Namespaces = make(map[string]NamespaceRenderingData) + + for name, data := range namespaceData { + templateData.Namespaces[name] = NamespaceRenderingData{ + LeaderVHost: data.Drove.LeaderVHost, + Leader: data.Leader, + Apps: data.Apps, + KnownVHosts: data.KnownVHosts, + RoutingTag: data.Drove.RoutingTag, + } + } + return +} + func writeConf() error { config.RLock() defer config.RUnlock() + allApps := db.ReadAllApps() + allLeaders := db.ReadAllLeaders() + template, err := getTmpl() if err != nil { return err } - logger.WithFields(logrus.Fields{ - "config: ": config.Apps, - "leader": config.Leader, + "apps: ": allApps, + "leader": allLeaders, }).Info("Config: ") + parent := filepath.Dir(config.NginxConfig) tmpFile, err := ioutil.TempFile(parent, ".nginx.conf.tmp-") if err != nil { @@ -463,14 +556,19 @@ func writeConf() error { } defer tmpFile.Close() lastConfig = tmpFile.Name() - err = template.Execute(tmpFile, &config) + templateData := TemplateRenderingData{} + createTemplateData(&templateData) + logger.WithFields(logrus.Fields{ + "templateData": templateData, + }).Info("Template Data generated") + err = template.Execute(tmpFile, &templateData) if err != nil { return err } config.LastUpdates.LastConfigRendered = time.Now() err = checkConf(tmpFile.Name()) if err != nil { - //os.Remove(tmpFile.Name()) + logger.Error("Error in config generated") return err } err = os.Rename(tmpFile.Name(), config.NginxConfig) @@ -484,8 +582,9 @@ func writeConf() error { func nginxPlus() error { config.RLock() defer config.RUnlock() + allApps := db.ReadAllApps() logger.WithFields(logrus.Fields{}).Info("Updating upstreams for the whitelisted drove tags") - for _, app := range config.Apps { + for _, app := range allApps { var newFormattedServers []string for _, t := range app.Hosts { var hostAndPortMapping string @@ -581,6 +680,9 @@ func checkTmpl() error { } func getTmpl() (*template.Template, error) { + logger.WithFields(logrus.Fields{ + "file": config.NginxTemplate, + }).Info("Reading template") return template.New(filepath.Base(config.NginxTemplate)). Delims(config.LeftDelimiter, config.RightDelimiter). Funcs(template.FuncMap{ @@ -642,10 +744,10 @@ func reloadNginx() error { } func reload() error { - return reloadAllApps(false) + return reloadAllApps(config.DroveNamespaces[0].Name, false) //TODO: for for all namespace } -func updateAndReloadConfig(vhosts *Vhosts) error { +func updateAndReloadConfig(namepsace string, vhosts *Vhosts) error { start := time.Now() config.LastUpdates.LastSync = time.Now() err := writeConf() @@ -675,18 +777,28 @@ func updateAndReloadConfig(vhosts *Vhosts) error { go countSuccessfulReloads.Inc() go observeReloadTimeMetric(elapsed) config.LastUpdates.LastNginxReload = time.Now() - config.KnownVHosts = *vhosts + db.UpdateKnownVhosts(namepsace, *vhosts) } return nil } -func reloadAllApps(leaderShifted bool) error { - logger.Debug("Reloading config") +func reloadAllApps(namespace string, leaderShifted bool) error { + logger.Debug("Reloading config for namespace" + namespace) + + droveConfig, er := db.ReadDroveConfig(namespace) + if er != nil { + logger.WithFields(logrus.Fields{ + "namespace": namespace, + }).Error("Error loading drove config") + er := errors.New("Error loading Drove Config") + return er + } + start := time.Now() jsonapps := DroveApps{} vhosts := Vhosts{} vhosts.Vhosts = map[string]bool{} - err := fetchApps(&jsonapps) + err := fetchApps(droveConfig, &jsonapps) if err != nil || jsonapps.Status != "SUCCESS" { if err != nil { logger.WithFields(logrus.Fields{ @@ -697,16 +809,15 @@ func reloadAllApps(leaderShifted bool) error { go countFailedReloads.Inc() return err } - equal := syncApps(&jsonapps, &vhosts) + equal := syncApps(droveConfig, &jsonapps, &vhosts) if equal && !leaderShifted { logger.Debug("no config changes") return nil } - config.LastUpdates.LastSync = time.Now() if len(config.Nginxplusapiaddr) == 0 || config.Nginxplusapiaddr == "" { //Nginx plus is disabled - err = updateAndReloadConfig(&vhosts) + err = updateAndReloadConfig(namespace, &vhosts) if err != nil { logger.WithFields(logrus.Fields{ "error": err.Error(), @@ -720,9 +831,10 @@ func reloadAllApps(leaderShifted bool) error { if config.NginxReloadDisabled { logger.Warn("Template reload has been disabled") } else { - if !reflect.DeepEqual(vhosts, config.KnownVHosts) { + currentKnownVhosts, _ := db.ReadKnownVhosts(namespace) + if !reflect.DeepEqual(vhosts, currentKnownVhosts) { logger.Info("Need to reload config") - err = updateAndReloadConfig(&vhosts) + err = updateAndReloadConfig(namespace, &vhosts) if err != nil { logger.WithFields(logrus.Fields{ "error": err.Error(), diff --git a/src/nginx-header.tmpl b/src/nginx-header.tmpl index 13ff4a6..36a6031 100644 --- a/src/nginx-header.tmpl +++ b/src/nginx-header.tmpl @@ -39,25 +39,26 @@ http { return 503; } } - {{if and .LeaderVHost .Leader.Endpoint}} - upstream {{.LeaderVHost}} { - server {{.Leader.Host}}:{{.Leader.Port}}; + + {{if and .Namespaces.stage1.LeaderVHost .Namespaces.stage1.Leader.Endpoint}} + upstream {{.Namespaces.stage1.LeaderVHost}} { + server {{.Namespaces.stage1.Leader.Host}}:{{.Namespaces.stage1.Leader.Port}}; } server { listen 7000; - server_name {{.LeaderVHost}}; + server_name {{.Namespaces.stage1.LeaderVHost}}; location / { - proxy_set_header HOST {{.Leader.Host}}; + proxy_set_header HOST {{.Namespaces.stage1.Leader.Host}}; proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; proxy_connect_timeout 30; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; - proxy_pass http://{{.LeaderVHost}}; + proxy_pass http://{{.Namespaces.stage1.LeaderVHost}}; } } {{end}} - {{- range $id, $app := .Apps}} + {{- range $id, $app := .Namespaces.stage1.Apps}} {{- range $gid, $hostgrp := .Groups}} upstream {{$gid}} { {{- range $hostgrp.Hosts}} diff --git a/src/nixy.go b/src/nixy.go index eabb6f3..a886944 100644 --- a/src/nixy.go +++ b/src/nixy.go @@ -51,43 +51,44 @@ type Vhosts struct { Vhosts map[string]bool } +type DroveNamespace struct { + Name string `json:"-" toml:"name"` + Drove []string `json:"-"` + User string `json:"-"` + Pass string `json:"-"` + AccessToken string `json:"-" toml:"access_token"` + Realm string + RealmSuffix string `json:"-" toml:"realm_suffix"` + RoutingTag string `json:"-" toml:"routing_tag"` + LeaderVHost string `json:"-" toml:"leader_vhost"` +} + // Config struct used by the template engine type Config struct { sync.RWMutex Xproxy string - Realm string - RealmSuffix string `json:"-" toml:"realm_suffix"` - Address string `json:"-"` - Port string `json:"-"` - PortWithTLS bool `json:"-" toml:"port_use_tls"` - TLScertFile string `json:"-" toml:"port_tls_certfile"` - TLSkeyFile string `json:"-" toml:"port_tls_keyfile"` - Drove []string `json:"-"` - EventRefreshIntervalSec int `json:"-" toml:"event_refresh_interval_sec"` - User string `json:"-"` - Pass string `json:"-"` - AccessToken string `json:"-" toml:"access_token"` - Nginxplusapiaddr string `json:"-"` - NginxReloadDisabled bool `json:"-" toml:"nginx_reload_disabled"` - NginxConfig string `json:"-" toml:"nginx_config"` - NginxTemplate string `json:"-" toml:"nginx_template"` - NginxCmd string `json:"-" toml:"nginx_cmd"` - NginxIgnoreCheck bool `json:"-" toml:"nginx_ignore_check"` - LeftDelimiter string `json:"-" toml:"left_delimiter"` - RightDelimiter string `json:"-" toml:"right_delimiter"` - MaxFailsUpstream *int `json:"max_fails,omitempty"` - FailTimeoutUpstream string `json:"fail_timeout,omitempty"` - SlowStartUpstream string `json:"slow_start,omitempty"` - LogLevel string `json:"-" toml:"loglevel"` - - apiTimeout int `json:"-" toml:"api_timeout"` - LeaderVHost string `json:"-" toml:"leader_vhost"` - RoutingTag string `json:"-" toml:"routing_tag"` - Leader LeaderController - Statsd StatsdConfig - LastUpdates Updates - Apps map[string]App - KnownVHosts Vhosts + Address string `json:"-"` + Port string `json:"-"` + PortWithTLS bool `json:"-" toml:"port_use_tls"` + TLScertFile string `json:"-" toml:"port_tls_certfile"` + TLSkeyFile string `json:"-" toml:"port_tls_keyfile"` + DroveNamespaces []DroveNamespace `json:"-" toml:"namespaces"` + EventRefreshIntervalSec int `json:"-" toml:"event_refresh_interval_sec"` + Nginxplusapiaddr string `json:"-"` + NginxReloadDisabled bool `json:"-" toml:"nginx_reload_disabled"` + NginxConfig string `json:"-" toml:"nginx_config"` + NginxTemplate string `json:"-" toml:"nginx_template"` + NginxCmd string `json:"-" toml:"nginx_cmd"` + NginxIgnoreCheck bool `json:"-" toml:"nginx_ignore_check"` + LeftDelimiter string `json:"-" toml:"left_delimiter"` + RightDelimiter string `json:"-" toml:"right_delimiter"` + MaxFailsUpstream *int `json:"max_fails,omitempty"` + FailTimeoutUpstream string `json:"fail_timeout,omitempty"` + SlowStartUpstream string `json:"slow_start,omitempty"` + LogLevel string `json:"-" toml:"loglevel"` + apiTimeout int `json:"-" toml:"api_timeout"` + Statsd StatsdConfig + LastUpdates Updates } // Updates timings used for metrics @@ -113,9 +114,10 @@ type Status struct { // EndpointStatus health status struct type EndpointStatus struct { - Endpoint string - Healthy bool - Message string + Namespace string + Endpoint string + Healthy bool + Message string } // Health struct @@ -133,9 +135,10 @@ var config = Config{LeftDelimiter: "{{", RightDelimiter: "}}"} var statsd g2s.Statter var health Health var lastConfig string +var db DataManager var logger = logrus.New() -//set log level +// set log level func setloglevel() { logLevel := logrus.InfoLevel switch config.LogLevel { @@ -157,6 +160,16 @@ func setloglevel() { logger.SetLevel(logLevel) } +// set DataManager +func setupDataManager() { + db = *NewDataManager(config.Xproxy, config.LeftDelimiter, config.RightDelimiter, config.MaxFailsUpstream, + config.FailTimeoutUpstream, config.SlowStartUpstream) + for _, nsConfig := range config.DroveNamespaces { + db.CreateNamespace(nsConfig.Name, nsConfig.Drove, nsConfig.User, nsConfig.Pass, + nsConfig.AccessToken, nsConfig.Realm, nsConfig.RealmSuffix, nsConfig.RoutingTag, nsConfig.LeaderVHost) + } +} + // Eventqueue with buffer of two, because we dont really need more. var eventqueue = make(chan bool, 2) @@ -165,12 +178,15 @@ var tr = &http.Transport{MaxIdleConnsPerHost: 10, TLSClientConfig: &tls.Config{I func newHealth() Health { var h Health - for _, ep := range config.Drove { - var s EndpointStatus - s.Endpoint = ep - s.Healthy = false - s.Message = "OK" - h.Endpoints = append(h.Endpoints, s) + for _, nsConfig := range config.DroveNamespaces { + for _, ep := range nsConfig.Drove { + var s EndpointStatus + s.Endpoint = ep + s.Namespace = nsConfig.Name + s.Healthy = false + s.Message = "OK" + h.Endpoints = append(h.Endpoints, s) + } } return h } @@ -282,6 +298,7 @@ func main() { statsd = g2s.Noop() //fallback to Noop. } setupPrometheusMetrics() + setupDataManager() mux := mux.NewRouter() mux.HandleFunc("/", nixyVersion) mux.HandleFunc("/v1/reload", nixyReload) @@ -313,9 +330,12 @@ func main() { } health = newHealth() endpointHealth() - err = retry.Do(reload) + er := retry.Do(reload) + if er != nil { + logger.Error("Error reloading :" + er.Error()) + } eventWorker() - pollEvents() + pollEvents(config.DroveNamespaces[0].Name) //TODO: add for all namepsaces logger.Info("Address:" + config.Address) if config.PortWithTLS { logger.Info("starting nixy on https://" + config.Address + ":" + config.Port) diff --git a/src/nixy.toml b/src/nixy.toml index d10b9e7..0017d33 100644 --- a/src/nixy.toml +++ b/src/nixy.toml @@ -9,37 +9,35 @@ port = "6000" # X-Proxy header, defaults to hostname xproxy = "" -# Drove API -drove = ["http://localhost:10000"] # add all HA cluster nodes in priority order. -#drove = ["http://localhost:4000", "http://localhost:5000"] # add all HA cluster nodes in priority order. -#drove = ["https://stg-nb6.drove.phonepe.com:443"] # add all HA cluster nodes in priority order. -event_refresh_interval_sec = 5 -user = "" # leave empty if no auth is required. -pass = "" -access_token = "O-Bearer OLYMPUS_TOKEN" -# Nixy realm, set this if you want to be able to filter your apps (e.g. when you have different loadbalancers which should expose different apps) -# Put your subdomain here. It will be used to match the subdomain of the exposed apps -#realm = "api.nixy.stg-drove.phonepe.nb6" # Nginx #nginxplusapiaddr="127.0.0.1" -#nginxplusapiaddr="stg-nginx301.phonepe.nb6" nginx_config = "./nginx-test.conf" nginx_template = "./nginx-header.tmpl" nginx_cmd = "nginx" # optionally "openresty" or "docker exec nginx nginx" nginx_ignore_check = true # optionally disable nginx config test. Health check will always show OK. -traefiklabel = "traefik.backend" -traefikbackend = ["nixy-demo"] #nginxplusapiaddr = "10.57.11.218:2222" maxfailsupstream = 0 failtimeoutupstream = "1s" slowstartupstream = "0s" +event_refresh_interval_sec = 5 # Statsd settings #[statsd] #addr = "10.57.8.171:8125" # optional for statistics #namespace = "nixy.my_mesos_cluster" #sample_rate = 100 -leader_vhost = "drove.ssdev.test" + +# Drove API +[[namespaces]] +name = "stage1" +drove = ["https://localhost:8080"] # add all HA cluster nodes in priority order. +user = "" # leave empty if no auth is required. +pass = "" +access_token = "O-Bearer " +leader_vhost = "drove.ssdev.test" +# Nixy realm, set this if you want to be able to filter your apps (e.g. when you have different loadbalancers which should expose different apps) +# Put your subdomain here. It will be used to match the subdomain of the exposed apps +realm = "drove.xxxx.com" From 278b937707729c6fed757e8cedeb82eab1950996 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Mon, 25 Nov 2024 11:33:42 +0530 Subject: [PATCH 02/29] bug fix in lastKnowHosts --- src/datamanager.go | 61 +++++++++++++++++++++++++++++++++++++++++++--- src/drove.go | 30 +++++++++++++---------- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/src/datamanager.go b/src/datamanager.go index 12fd552..7e6b4c3 100644 --- a/src/datamanager.go +++ b/src/datamanager.go @@ -40,18 +40,22 @@ type StaticConfig struct { // DataManager manages namespaces and data for those namespaces type DataManager struct { - mu sync.RWMutex // Mutex for concurrency control - namespaces map[string]NamespaceData // Map of namespaces to NamespaceData - StaticData StaticConfig + mu sync.RWMutex // Mutex for concurrency control + namespaces map[string]NamespaceData // Map of namespaces to NamespaceData + LastKnownVhosts Vhosts + StaticData StaticConfig } // NewDataManager creates a new instance of DataManager func NewDataManager(inXproxy string, inLeftDelimiter string, inRightDelimiter string, inMaxFailsUpstream *int, inFailTimeoutUpstream string, inSlowStartUpstream string) *DataManager { + empltyLastKnownVhosts := Vhosts{} + empltyLastKnownVhosts.Vhosts = make(map[string]bool) return &DataManager{ namespaces: make(map[string]NamespaceData), StaticData: StaticConfig{Xproxy: inXproxy, LeftDelimiter: inLeftDelimiter, RightDelimiter: inRightDelimiter, MaxFailsUpstream: inMaxFailsUpstream, FailTimeoutUpstream: inFailTimeoutUpstream, SlowStartUpstream: inSlowStartUpstream}, + LastKnownVhosts: empltyLastKnownVhosts, } } @@ -305,6 +309,29 @@ func (dm *DataManager) ReadKnownVhosts(namespace string) (Vhosts, error) { return ns.KnownVHosts, nil //returning copy } +func (dm *DataManager) ReadAllKnownVhosts() Vhosts { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadAllKnownVhosts" + + allKnownVhosts := Vhosts{} + allKnownVhosts.Vhosts = make(map[string]bool) + + for _, data := range dm.namespaces { + for key, value := range data.KnownVHosts.Vhosts { + allKnownVhosts.Vhosts[key] = value + } + } + + // Log success + logger.WithFields(logrus.Fields{ + "operation": operation, + "allApps": allKnownVhosts, + }).Info("ReadAllKnownVhosts successfully") + + return allKnownVhosts //returning copy +} + func (dm *DataManager) UpdateKnownVhosts(namespace string, KnownVHosts Vhosts) error { dm.mu.Lock() // Read lock to allow multiple concurrent reads defer dm.mu.Unlock() // Ensure the lock is always released @@ -334,6 +361,34 @@ func (dm *DataManager) UpdateKnownVhosts(namespace string, KnownVHosts Vhosts) e return nil } +func (dm *DataManager) ReadLastKnownVhosts() Vhosts { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadLastKnownVhosts" + + // Log success + logger.WithFields(logrus.Fields{ + "operation": operation, + "LastKnownVhosts": dm.LastKnownVhosts, + }).Info("LastKnownVhosts successfully") + + return dm.LastKnownVhosts //returning copy +} + +func (dm *DataManager) UpdateLastKnownVhosts(inLastKnownVhosts Vhosts) error { + dm.mu.Lock() // Read lock to allow multiple concurrent reads + defer dm.mu.Unlock() // Ensure the lock is always released + operation := "UpdateLastKnownVhosts" + + dm.LastKnownVhosts = inLastKnownVhosts + + logger.WithFields(logrus.Fields{ + "operation": operation, + "LastKnownVhosts": dm.LastKnownVhosts, + }).Info("UpdateLastKnownVhosts successfully") + return nil +} + func (dm *DataManager) ReadAllNamespace() map[string]NamespaceData { dm.mu.RLock() // Read lock to allow multiple concurrent reads defer dm.mu.RUnlock() // Ensure the lock is always released diff --git a/src/drove.go b/src/drove.go index 97d323d..bfcfcb9 100644 --- a/src/drove.go +++ b/src/drove.go @@ -506,6 +506,12 @@ func syncApps(droveConfig DroveConfig, jsonapps *DroveApps, vhosts *Vhosts) bool logger.Error("Error while updating apps for namespace" + droveConfig.Name) return true } + + er = db.UpdateKnownVhosts(droveConfig.Name, *vhosts) + if er != nil { + logger.Error("Error while updating KnowVhosts for namespace" + droveConfig.Name) + return true + } return false } @@ -747,9 +753,10 @@ func reload() error { return reloadAllApps(config.DroveNamespaces[0].Name, false) //TODO: for for all namespace } -func updateAndReloadConfig(namepsace string, vhosts *Vhosts) error { +func updateAndReloadConfig() error { start := time.Now() config.LastUpdates.LastSync = time.Now() + vhosts := db.ReadAllKnownVhosts() err := writeConf() if err != nil { logger.WithFields(logrus.Fields{ @@ -777,7 +784,7 @@ func updateAndReloadConfig(namepsace string, vhosts *Vhosts) error { go countSuccessfulReloads.Inc() go observeReloadTimeMetric(elapsed) config.LastUpdates.LastNginxReload = time.Now() - db.UpdateKnownVhosts(namepsace, *vhosts) + db.UpdateLastKnownVhosts(vhosts) } return nil } @@ -817,7 +824,7 @@ func reloadAllApps(namespace string, leaderShifted bool) error { config.LastUpdates.LastSync = time.Now() if len(config.Nginxplusapiaddr) == 0 || config.Nginxplusapiaddr == "" { //Nginx plus is disabled - err = updateAndReloadConfig(namespace, &vhosts) + err = updateAndReloadConfig() if err != nil { logger.WithFields(logrus.Fields{ "error": err.Error(), @@ -831,16 +838,13 @@ func reloadAllApps(namespace string, leaderShifted bool) error { if config.NginxReloadDisabled { logger.Warn("Template reload has been disabled") } else { - currentKnownVhosts, _ := db.ReadKnownVhosts(namespace) - if !reflect.DeepEqual(vhosts, currentKnownVhosts) { - logger.Info("Need to reload config") - err = updateAndReloadConfig(namespace, &vhosts) - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err.Error(), - }).Error("unable to update and reload nginx config. NPlus api calls will be skipped.") - return err - } + logger.Info("Need to reload config") + err = updateAndReloadConfig() + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + }).Error("unable to update and reload nginx config. NPlus api calls will be skipped.") + return err } else { logger.Debug("No changes detected in vhosts. No config update is necessary. Upstream updates will happen via nplus apis") } From 15a0d5fe64102bd1893f49fc4889ac16a662d3d7 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Mon, 25 Nov 2024 11:34:15 +0530 Subject: [PATCH 03/29] bug fix in lastKnowHosts --- src/nixy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nixy.go b/src/nixy.go index a886944..9d24947 100644 --- a/src/nixy.go +++ b/src/nixy.go @@ -334,7 +334,7 @@ func main() { if er != nil { logger.Error("Error reloading :" + er.Error()) } - eventWorker() + eventWorker() //Reloader pollEvents(config.DroveNamespaces[0].Name) //TODO: add for all namepsaces logger.Info("Address:" + config.Address) if config.PortWithTLS { From 7842cababfa50e193983649505b1b4ff48098a10 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Mon, 25 Nov 2024 14:05:27 +0530 Subject: [PATCH 04/29] added drove client --- src/datamanager.go | 56 ++++- src/drove.go | 499 +++++++++------------------------------------ src/nixy.go | 67 +++--- src/reload.go | 373 +++++++++++++++++++++++++++++++++ 4 files changed, 557 insertions(+), 438 deletions(-) create mode 100644 src/reload.go diff --git a/src/datamanager.go b/src/datamanager.go index 7e6b4c3..fe5a786 100644 --- a/src/datamanager.go +++ b/src/datamanager.go @@ -40,10 +40,11 @@ type StaticConfig struct { // DataManager manages namespaces and data for those namespaces type DataManager struct { - mu sync.RWMutex // Mutex for concurrency control - namespaces map[string]NamespaceData // Map of namespaces to NamespaceData - LastKnownVhosts Vhosts - StaticData StaticConfig + mu sync.RWMutex // Mutex for concurrency control + namespaces map[string]NamespaceData // Map of namespaces to NamespaceData + LastKnownVhosts Vhosts + LastReloadTimestamp time.Time // Timestamp of creation or modification + StaticData StaticConfig } // NewDataManager creates a new instance of DataManager @@ -55,7 +56,7 @@ func NewDataManager(inXproxy string, inLeftDelimiter string, inRightDelimiter st namespaces: make(map[string]NamespaceData), StaticData: StaticConfig{Xproxy: inXproxy, LeftDelimiter: inLeftDelimiter, RightDelimiter: inRightDelimiter, MaxFailsUpstream: inMaxFailsUpstream, FailTimeoutUpstream: inFailTimeoutUpstream, SlowStartUpstream: inSlowStartUpstream}, - LastKnownVhosts: empltyLastKnownVhosts, + LastKnownVhosts: empltyLastKnownVhosts, LastReloadTimestamp: time.Now(), } } @@ -123,6 +124,33 @@ func (dm *DataManager) ReadDroveConfig(namespace string) (DroveConfig, error) { return ns.Drove, nil //returning copy } +func (dm *DataManager) ReadLastTimestamp(namespace string) (time.Time, error) { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "ReadLastTimestamps" + + // Ensure namespace exists + if _, exists := dm.namespaces[namespace]; !exists { + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + }).Error("NamespaceData read failed") + err := fmt.Errorf("namespace '%s' not found", namespace) + return time.Time{}, err + } + + ns := dm.namespaces[namespace] + + // Log success + logger.WithFields(logrus.Fields{ + "operation": operation, + "namespace": namespace, + "Timestamp": ns.Timestamp, + }).Info("ReadLastTimestamps successfully") + + return ns.Timestamp, nil //returning copy +} + func (dm *DataManager) ReadLeader(namespace string) (LeaderController, error) { dm.mu.RLock() // Read lock to allow multiple concurrent reads defer dm.mu.RUnlock() // Ensure the lock is always released @@ -145,7 +173,6 @@ func (dm *DataManager) ReadLeader(namespace string) (LeaderController, error) { "operation": operation, "namespace": namespace, "leader": ns.Leader, - "ns": ns, }).Info("ReadLeader successfully") return ns.Leader, nil //returning copy @@ -204,7 +231,7 @@ func (dm *DataManager) UpdateLeader(namespace string, leader LeaderController) e return nil } -func (dm *DataManager) ReadApp(namespace string) (map[string]App, error) { +func (dm *DataManager) ReadApps(namespace string) (map[string]App, error) { dm.mu.RLock() // Read lock to allow multiple concurrent reads defer dm.mu.RUnlock() // Ensure the lock is always released operation := "ReadLeader" @@ -361,6 +388,20 @@ func (dm *DataManager) UpdateKnownVhosts(namespace string, KnownVHosts Vhosts) e return nil } +func (dm *DataManager) ReadLastReloadTimestamp() time.Time { + dm.mu.RLock() // Read lock to allow multiple concurrent reads + defer dm.mu.RUnlock() // Ensure the lock is always released + operation := "LastReloadTimestamp" + + // Log success + logger.WithFields(logrus.Fields{ + "operation": operation, + "LastReloadTimestamp": dm.LastReloadTimestamp, + }).Info("ReadLoadedTime successfully") + + return dm.LastReloadTimestamp //returning copy +} + func (dm *DataManager) ReadLastKnownVhosts() Vhosts { dm.mu.RLock() // Read lock to allow multiple concurrent reads defer dm.mu.RUnlock() // Ensure the lock is always released @@ -381,6 +422,7 @@ func (dm *DataManager) UpdateLastKnownVhosts(inLastKnownVhosts Vhosts) error { operation := "UpdateLastKnownVhosts" dm.LastKnownVhosts = inLastKnownVhosts + dm.LastReloadTimestamp = time.Now() logger.WithFields(logrus.Fields{ "operation": operation, diff --git a/src/drove.go b/src/drove.go index bfcfcb9..5d0d92a 100644 --- a/src/drove.go +++ b/src/drove.go @@ -1,45 +1,21 @@ package main import ( - "bytes" "encoding/json" "errors" "fmt" - "io/ioutil" "net" "net/http" "net/url" - "os" - "os/exec" - "path/filepath" "reflect" "strconv" "strings" "sync" - "text/template" "time" "github.com/sirupsen/logrus" ) -type NamespaceRenderingData struct { - LeaderVHost string `json:"-" toml:"leader_vhost"` - Leader LeaderController - Apps map[string]App - RoutingTag string `json:"-" toml:"routing_tag"` - KnownVHosts Vhosts -} - -type TemplateRenderingData struct { - Xproxy string - LeftDelimiter string `json:"-" toml:"left_delimiter"` - RightDelimiter string `json:"-" toml:"right_delimiter"` - MaxFailsUpstream *int `json:"max_fails,omitempty"` - FailTimeoutUpstream string `json:"fail_timeout,omitempty"` - SlowStartUpstream string `json:"slow_start,omitempty"` - Namespaces map[string]NamespaceRenderingData `json:"namespaces"` -} - // DroveApps struct for our apps nested with tasks. type DroveApps struct { Status string `json:"status"` @@ -68,6 +44,13 @@ type DroveEventsApiResponse struct { Message string `json:"message"` } +type DroveClient struct { + namespace string + syncPoint CurrSyncPoint + httpClient *http.Client + eventRefreshInterval int +} + type CurrSyncPoint struct { sync.RWMutex LastSyncTime int64 @@ -113,8 +96,8 @@ func fetchRecentEvents(client *http.Client, syncPoint *CurrSyncPoint, namespace } var endpoint string - for _, es := range health.Endpoints { - if es.Healthy && es.Namespace == namespace { + for _, es := range health.NamesapceEndpoints[namespace] { + if es.Healthy { endpoint = es.Endpoint break } @@ -160,9 +143,9 @@ func fetchRecentEvents(client *http.Client, syncPoint *CurrSyncPoint, namespace return &(newEventsApiResponse.EventSummary), nil } -func refreshLeaderData(namespace string) { +func refreshLeaderData(namespace string) bool { var endpoint string - for _, es := range health.Endpoints { + for _, es := range health.NamesapceEndpoints[namespace] { if es.Namespace == namespace && es.Healthy { endpoint = es.Endpoint break @@ -171,12 +154,12 @@ func refreshLeaderData(namespace string) { if endpoint == "" { logger.Error("all endpoints are down") go countAllEndpointsDownErrors.Inc() - return + return false } currentLeader, err := db.ReadLeader(namespace) if err != nil { logger.Error("Error while reading current leader for namespace" + namespace) - return + return false } if endpoint != currentLeader.Endpoint { logger.WithFields(logrus.Fields{ @@ -197,37 +180,45 @@ func refreshLeaderData(namespace string) { "leader": currentLeader, "newLeader": newLeader, }).Info("New leader being set") - reloadAllApps(namespace, true) + return true } else { logrus.Warn("Leade struct generation failed") } } + return false } -func pollEvents(namespace string) { - go func() { - client := &http.Client{ +func newDroveClient(name string) *DroveClient { + return &DroveClient{namespace: name, + syncPoint: CurrSyncPoint{}, + httpClient: &http.Client{ Timeout: 0 * time.Second, Transport: tr, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, - } - syncData := CurrSyncPoint{} - refreshInterval := 2 - if config.EventRefreshIntervalSec > 0 { - refreshInterval = config.EventRefreshIntervalSec - } - ticker := time.NewTicker(time.Duration(refreshInterval) * time.Second) + }, + eventRefreshInterval: config.EventRefreshIntervalSec, + } +} + +func pollDroveEvents(name string) { + go func() { + droveClient := newDroveClient(name) + ticker := time.NewTicker(time.Duration(droveClient.eventRefreshInterval) * time.Second) for range ticker.C { func() { + namespace := droveClient.namespace logger.WithFields(logrus.Fields{ - "at": time.Now(), + "at": time.Now(), + "namespace": namespace, }).Debug("Syncing...") - syncData.Lock() - defer syncData.Unlock() - refreshLeaderData(namespace) - eventSummary, err := fetchRecentEvents(client, &syncData, namespace) + + droveClient.syncPoint.Lock() + defer droveClient.syncPoint.Unlock() + + leaderShifted := refreshLeaderData(namespace) + eventSummary, err := fetchRecentEvents(droveClient.httpClient, &droveClient.syncPoint, namespace) if err != nil { logger.WithFields(logrus.Fields{ "error": err.Error(), @@ -247,12 +238,8 @@ func pollEvents(namespace string) { if _, ok := eventSummary.EventsCount["INSTANCE_STATE_CHANGE"]; ok { reloadNeeded = true } - if reloadNeeded { - select { - case eventqueue <- true: // add reload to our queue channel, unless it is full of course. - default: - logger.Warn("queue is full") - } + if reloadNeeded || leaderShifted { + refreshApps(namespace, leaderShifted) } else { logger.Debug("Irrelevant events ignored") } @@ -268,30 +255,13 @@ func pollEvents(namespace string) { }() } -func eventWorker() { - go func() { - // a ticker channel to limit reloads to drove, 1s is enough for now. - ticker := time.NewTicker(1 * time.Second) - for { - select { - case <-ticker.C: - <-eventqueue - reload() - } - } - }() -} - -func endpointHealth() { +func namepaceEndpointHealth(namespace string) { go func() { ticker := time.NewTicker(2 * time.Second) for { select { case <-ticker.C: - //logger.WithFields(logrus.Fields{ - // "health": health, - //}).Info("Reloading endpoint health") - for i, es := range health.Endpoints { + for i, es := range health.NamesapceEndpoints[namespace] { client := &http.Client{ Timeout: 5 * time.Second, Transport: tr, @@ -305,8 +275,8 @@ func endpointHealth() { "error": err.Error(), "endpoint": es.Endpoint, }).Error("an error occurred creating endpoint health request") - health.Endpoints[i].Healthy = false - health.Endpoints[i].Message = err.Error() + health.NamesapceEndpoints[namespace][i].Healthy = false + health.NamesapceEndpoints[namespace][i].Message = err.Error() continue } droveConfig, err := db.ReadDroveConfig(es.Namespace) @@ -315,8 +285,8 @@ func endpointHealth() { "error": err, "endpoint": es.Endpoint, }).Error("an error occurred reading drove config for health request") - health.Endpoints[i].Healthy = false - health.Endpoints[i].Message = err.Error() + health.NamesapceEndpoints[namespace][i].Healthy = false + health.NamesapceEndpoints[namespace][i].Message = err.Error() continue } if droveConfig.User != "" { @@ -328,23 +298,27 @@ func endpointHealth() { resp, err := client.Do(req) if err != nil { logger.WithFields(logrus.Fields{ - "error": err.Error(), - "endpoint": es.Endpoint, + "error": err.Error(), + "endpoint": es.Endpoint, + "namespace": namespace, }).Error("endpoint is down") go countEndpointDownErrors.Inc() - health.Endpoints[i].Healthy = false - health.Endpoints[i].Message = err.Error() + health.NamesapceEndpoints[namespace][i].Healthy = false + health.NamesapceEndpoints[namespace][i].Message = err.Error() continue } resp.Body.Close() if resp.StatusCode != 200 { - health.Endpoints[i].Healthy = false - health.Endpoints[i].Message = resp.Status + health.NamesapceEndpoints[namespace][i].Healthy = false + health.NamesapceEndpoints[namespace][i].Message = resp.Status continue } - health.Endpoints[i].Healthy = true - health.Endpoints[i].Message = "OK" - logger.WithFields(logrus.Fields{"host": es.Endpoint}).Debug(" Endpoint is healthy") + health.NamesapceEndpoints[namespace][i].Healthy = true + health.NamesapceEndpoints[namespace][i].Message = "OK" + logger.WithFields(logrus.Fields{ + "host": es.Endpoint, + "namespace": namespace, + }).Debug(" Endpoint is healthy") } } } @@ -354,8 +328,8 @@ func endpointHealth() { func fetchApps(droveConfig DroveConfig, jsonapps *DroveApps) error { var endpoint string var timeout int = 5 - for _, es := range health.Endpoints { - if es.Healthy && droveConfig.Name == es.Namespace { + for _, es := range health.NamesapceEndpoints[droveConfig.Name] { + if es.Healthy { endpoint = es.Endpoint break } @@ -411,7 +385,7 @@ func matchingVhost(vHost string, realms []string) bool { return false } -func syncApps(droveConfig DroveConfig, jsonapps *DroveApps, vhosts *Vhosts) bool { +func syncAppsAndVhosts(droveConfig DroveConfig, jsonapps *DroveApps, vhosts *Vhosts) bool { config.Lock() defer config.Unlock() apps := make(map[string]App) @@ -489,7 +463,7 @@ func syncApps(droveConfig DroveConfig, jsonapps *DroveApps, vhosts *Vhosts) bool } } - currentApps, er := db.ReadApp(droveConfig.Name) + currentApps, er := db.ReadApps(droveConfig.Name) if er != nil { logger.Error("Error while reading apps for namespace" + droveConfig.Name) //Continue to update with latest data @@ -515,281 +489,7 @@ func syncApps(droveConfig DroveConfig, jsonapps *DroveApps, vhosts *Vhosts) bool return false } -func createTemplateData(templateData *TemplateRenderingData) { - namespaceData := db.ReadAllNamespace() - staticData := db.ReadStaticData() - - templateData.Xproxy = staticData.Xproxy - templateData.LeftDelimiter = staticData.LeftDelimiter - templateData.RightDelimiter = staticData.RightDelimiter - templateData.FailTimeoutUpstream = staticData.FailTimeoutUpstream - templateData.MaxFailsUpstream = staticData.MaxFailsUpstream - templateData.SlowStartUpstream = staticData.SlowStartUpstream - - templateData.Namespaces = make(map[string]NamespaceRenderingData) - - for name, data := range namespaceData { - templateData.Namespaces[name] = NamespaceRenderingData{ - LeaderVHost: data.Drove.LeaderVHost, - Leader: data.Leader, - Apps: data.Apps, - KnownVHosts: data.KnownVHosts, - RoutingTag: data.Drove.RoutingTag, - } - } - return -} - -func writeConf() error { - config.RLock() - defer config.RUnlock() - allApps := db.ReadAllApps() - allLeaders := db.ReadAllLeaders() - - template, err := getTmpl() - if err != nil { - return err - } - logger.WithFields(logrus.Fields{ - "apps: ": allApps, - "leader": allLeaders, - }).Info("Config: ") - - parent := filepath.Dir(config.NginxConfig) - tmpFile, err := ioutil.TempFile(parent, ".nginx.conf.tmp-") - if err != nil { - return err - } - defer tmpFile.Close() - lastConfig = tmpFile.Name() - templateData := TemplateRenderingData{} - createTemplateData(&templateData) - logger.WithFields(logrus.Fields{ - "templateData": templateData, - }).Info("Template Data generated") - err = template.Execute(tmpFile, &templateData) - if err != nil { - return err - } - config.LastUpdates.LastConfigRendered = time.Now() - err = checkConf(tmpFile.Name()) - if err != nil { - logger.Error("Error in config generated") - return err - } - err = os.Rename(tmpFile.Name(), config.NginxConfig) - if err != nil { - return err - } - lastConfig = config.NginxConfig - return nil -} - -func nginxPlus() error { - config.RLock() - defer config.RUnlock() - allApps := db.ReadAllApps() - logger.WithFields(logrus.Fields{}).Info("Updating upstreams for the whitelisted drove tags") - for _, app := range allApps { - var newFormattedServers []string - for _, t := range app.Hosts { - var hostAndPortMapping string - ipRecords, error := net.LookupHost(string(t.Host)) - if error != nil { - logger.WithFields(logrus.Fields{ - "error": error, - "hostname": t.Host, - }).Error("dns lookup failed !! skipping the hostname") - continue - } - ipRecord := ipRecords[0] - hostAndPortMapping = ipRecord + ":" + fmt.Sprint(t.Port) - newFormattedServers = append(newFormattedServers, hostAndPortMapping) - - } - - logger.WithFields(logrus.Fields{ - "vhost": app.Vhost, - }).Info("app.vhost") - - logger.WithFields(logrus.Fields{ - "upstreams": newFormattedServers, - }).Info("nginx upstreams") - - logger.WithFields(logrus.Fields{ - "nginx": config.Nginxplusapiaddr, - }).Info("endpoint") - - endpoint := "http://" + config.Nginxplusapiaddr + "/api" - - tr := &http.Transport{ - MaxIdleConns: 30, - DisableCompression: true, - } - - client := &http.Client{Transport: tr} - c := NginxClient{endpoint, client} - nginxClient, error := NewNginxClient(c.httpClient, c.apiEndpoint) - if error != nil { - logger.WithFields(logrus.Fields{ - "error": error, - }).Error("unable to make call to nginx plus") - return error - } - upstreamtocheck := app.Vhost - var finalformattedServers []UpstreamServer - - for _, server := range newFormattedServers { - formattedServer := UpstreamServer{Server: server, MaxFails: config.MaxFailsUpstream, FailTimeout: config.FailTimeoutUpstream, SlowStart: config.SlowStartUpstream} - finalformattedServers = append(finalformattedServers, formattedServer) - } - - added, deleted, updated, error := nginxClient.UpdateHTTPServers(upstreamtocheck, finalformattedServers) - - if added != nil { - logger.WithFields(logrus.Fields{ - "nginx upstreams added": added, - }).Info("nginx upstreams added") - } - if deleted != nil { - logger.WithFields(logrus.Fields{ - "nginx upstreams deleted": deleted, - }).Info("nginx upstreams deleted") - } - if updated != nil { - logger.WithFields(logrus.Fields{ - "nginx upsteams updated": updated, - }).Info("nginx upstreams updated") - } - if error != nil { - logger.WithFields(logrus.Fields{ - "error": error, - }).Error("unable to update nginx upstreams") - return error - } - } - return nil -} - -func checkTmpl() error { - config.RLock() - defer config.RUnlock() - t, err := getTmpl() - if err != nil { - return err - } - err = t.Execute(ioutil.Discard, &config) - if err != nil { - return err - } - return nil -} - -func getTmpl() (*template.Template, error) { - logger.WithFields(logrus.Fields{ - "file": config.NginxTemplate, - }).Info("Reading template") - return template.New(filepath.Base(config.NginxTemplate)). - Delims(config.LeftDelimiter, config.RightDelimiter). - Funcs(template.FuncMap{ - "hasPrefix": strings.HasPrefix, - "hasSuffix": strings.HasPrefix, - "contains": strings.Contains, - "split": strings.Split, - "join": strings.Join, - "trim": strings.Trim, - "replace": strings.Replace, - "tolower": strings.ToLower, - "getenv": os.Getenv, - "datetime": time.Now}). - ParseFiles(config.NginxTemplate) -} - -func checkConf(path string) error { - // Always return OK if disabled in config. - if config.NginxIgnoreCheck { - return nil - } - // This is to allow arguments as well. Example "docker exec nginx..." - args := strings.Fields(config.NginxCmd) - head := args[0] - args = args[1:] - args = append(args, "-c") - args = append(args, path) - args = append(args, "-t") - cmd := exec.Command(head, args...) - //cmd := exec.Command(parts..., "-c", path, "-t") - var stderr bytes.Buffer - cmd.Stderr = &stderr - err := cmd.Run() // will wait for command to return - if err != nil { - msg := fmt.Sprint(err) + ": " + stderr.String() - errstd := errors.New(msg) - return errstd - } - return nil -} - -func reloadNginx() error { - // This is to allow arguments as well. Example "docker exec nginx..." - args := strings.Fields(config.NginxCmd) - head := args[0] - args = args[1:] - args = append(args, "-s") - args = append(args, "reload") - cmd := exec.Command(head, args...) - var stderr bytes.Buffer - cmd.Stderr = &stderr - err := cmd.Run() // will wait for command to return - if err != nil { - msg := fmt.Sprint(err) + ": " + stderr.String() - errstd := errors.New(msg) - return errstd - } - return nil -} - -func reload() error { - return reloadAllApps(config.DroveNamespaces[0].Name, false) //TODO: for for all namespace -} - -func updateAndReloadConfig() error { - start := time.Now() - config.LastUpdates.LastSync = time.Now() - vhosts := db.ReadAllKnownVhosts() - err := writeConf() - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err.Error(), - }).Error("unable to generate nginx config") - go statsCount("reload.failed", 1) - go countFailedReloads.Inc() - return err - } - config.LastUpdates.LastConfigValid = time.Now() - err = reloadNginx() - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err.Error(), - }).Error("unable to reload nginx") - go statsCount("reload.failed", 1) - go countFailedReloads.Inc() - } else { - elapsed := time.Since(start) - logger.WithFields(logrus.Fields{ - "took": elapsed, - }).Info("config updated") - go statsCount("reload.success", 1) - go statsTiming("reload.time", elapsed) - go countSuccessfulReloads.Inc() - go observeReloadTimeMetric(elapsed) - config.LastUpdates.LastNginxReload = time.Now() - db.UpdateLastKnownVhosts(vhosts) - } - return nil -} - -func reloadAllApps(namespace string, leaderShifted bool) error { +func refreshApps(namespace string, leaderShifted bool) error { logger.Debug("Reloading config for namespace" + namespace) droveConfig, er := db.ReadDroveConfig(namespace) @@ -801,7 +501,6 @@ func reloadAllApps(namespace string, leaderShifted bool) error { return er } - start := time.Now() jsonapps := DroveApps{} vhosts := Vhosts{} vhosts.Vhosts = map[string]bool{} @@ -816,52 +515,36 @@ func reloadAllApps(namespace string, leaderShifted bool) error { go countFailedReloads.Inc() return err } - equal := syncApps(droveConfig, &jsonapps, &vhosts) - if equal && !leaderShifted { - logger.Debug("no config changes") - return nil - } - config.LastUpdates.LastSync = time.Now() - if len(config.Nginxplusapiaddr) == 0 || config.Nginxplusapiaddr == "" { - //Nginx plus is disabled - err = updateAndReloadConfig() - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err.Error(), - }).Error("unable to reload nginx config") - go statsCount("reload.failed", 1) - go countFailedReloads.Inc() - return err - } - } else { - //Nginx plus is enabled - if config.NginxReloadDisabled { - logger.Warn("Template reload has been disabled") - } else { - logger.Info("Need to reload config") - err = updateAndReloadConfig() - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err.Error(), - }).Error("unable to update and reload nginx config. NPlus api calls will be skipped.") - return err - } else { - logger.Debug("No changes detected in vhosts. No config update is necessary. Upstream updates will happen via nplus apis") - } + equal := syncAppsAndVhosts(droveConfig, &jsonapps, &vhosts) + lastConfigUpdated := true + namespaceLastUpdated, err := db.ReadLastTimestamp(namespace) + if err == nil { + if db.ReadLastReloadTimestamp().Before(namespaceLastUpdated) { + logger.Info("Last reload still not applied") + lastConfigUpdated = false } - err = nginxPlus() } - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err.Error(), - }).Error("unable to generate nginx config") - go statsCount("reload.failed", 1) - go countFailedReloads.Inc() - return err + //Ideally only lastConfigUpdated can dictate if reload is required + if equal && !leaderShifted && lastConfigUpdated { + logger.Debug("no config changes") + return nil } - elapsed := time.Since(start) - logger.WithFields(logrus.Fields{ - "took": elapsed, - }).Info("config updated") + + triggerReload(namespace, !equal, leaderShifted, lastConfigUpdated) return nil } + +func triggerReload(namespace string, appsChanged, leaderShifted bool, lastConfigUpdated bool) { + logger.WithFields(logrus.Fields{ + "namespace": namespace, + "lastConfigUpdated": lastConfigUpdated, + "leaderShifted": leaderShifted, + "appsChanged": appsChanged, + }).Info("Trigging refresh") //logging exact reason of reload + + select { + case reloadSignalQueue <- true: // add reload to channel, unless it is full of course. + default: + logger.Warn("reloadSignalQueue is full") + } +} diff --git a/src/nixy.go b/src/nixy.go index 9d24947..166c8cc 100644 --- a/src/nixy.go +++ b/src/nixy.go @@ -13,7 +13,6 @@ import ( "time" "github.com/BurntSushi/toml" - "github.com/avast/retry-go" "github.com/gorilla/mux" "github.com/peterbourgon/g2s" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -122,9 +121,9 @@ type EndpointStatus struct { // Health struct type Health struct { - Config Status - Template Status - Endpoints []EndpointStatus + Config Status + Template Status + NamesapceEndpoints map[string][]EndpointStatus } // Global variables @@ -170,33 +169,48 @@ func setupDataManager() { } } -// Eventqueue with buffer of two, because we dont really need more. -var eventqueue = make(chan bool, 2) +// Reload signal with buffer of two, because we dont really need more. +var reloadSignalQueue = make(chan bool, 2) // Global http transport for connection reuse var tr = &http.Transport{MaxIdleConnsPerHost: 10, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} func newHealth() Health { var h Health + h.NamesapceEndpoints = make(map[string][]EndpointStatus) for _, nsConfig := range config.DroveNamespaces { + e := []EndpointStatus{} for _, ep := range nsConfig.Drove { var s EndpointStatus s.Endpoint = ep s.Namespace = nsConfig.Name s.Healthy = false s.Message = "OK" - h.Endpoints = append(h.Endpoints, s) + e = append(e, s) } + h.NamesapceEndpoints[nsConfig.Name] = e } return h } +func setupPollEvents() { + for _, nsConfig := range config.DroveNamespaces { + pollDroveEvents(nsConfig.Name) + } +} + +func setupEndpointHealth() { + for _, nsConfig := range config.DroveNamespaces { + namepaceEndpointHealth(nsConfig.Name) + } +} + func nixyReload(w http.ResponseWriter, r *http.Request) { logger.WithFields(logrus.Fields{ "client": r.RemoteAddr, - }).Info("drove reload triggered") + }).Info("Reload triggered") select { - case eventqueue <- true: // Add reload to our queue channel, unless it is full of course. + case reloadSignalQueue <- true: // Add reload to our queue channel, unless it is full of course. w.WriteHeader(202) fmt.Fprintln(w, "queued") return @@ -233,16 +247,21 @@ func nixyHealth(w http.ResponseWriter, r *http.Request) { health.Config.Healthy = true } } - allBackendsDown := true - for _, endpoint := range health.Endpoints { - if endpoint.Healthy { - allBackendsDown = false - break + anyNamesapceDown := false + for _, nsEnpoint := range health.NamesapceEndpoints { + allBackendsDownForGivenNS := true + for _, endpoint := range nsEnpoint { + if endpoint.Healthy { + allBackendsDownForGivenNS = false + break + } } + anyNamesapceDown = anyNamesapceDown || allBackendsDownForGivenNS } - if allBackendsDown { + if anyNamesapceDown { w.WriteHeader(http.StatusInternalServerError) } + w.Header().Add("Content-Type", "application/json; charset=utf-8") b, _ := json.MarshalIndent(health, "", " ") w.Write(b) @@ -251,7 +270,7 @@ func nixyHealth(w http.ResponseWriter, r *http.Request) { func nixyConfig(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", "application/json; charset=utf-8") - b, _ := json.MarshalIndent(&config, "", " ") + b, _ := json.MarshalIndent(&db, "", " ") w.Write(b) return } @@ -297,6 +316,12 @@ func main() { }).Error("unable to Dial statsd") statsd = g2s.Noop() //fallback to Noop. } + + //default refresh interval + if config.EventRefreshIntervalSec <= 0 { + config.EventRefreshIntervalSec = 2 + } + setupPrometheusMetrics() setupDataManager() mux := mux.NewRouter() @@ -329,13 +354,9 @@ func main() { } } health = newHealth() - endpointHealth() - er := retry.Do(reload) - if er != nil { - logger.Error("Error reloading :" + er.Error()) - } - eventWorker() //Reloader - pollEvents(config.DroveNamespaces[0].Name) //TODO: add for all namepsaces + setupEndpointHealth() + setupPollEvents() + reloadWorker() //Reloader logger.Info("Address:" + config.Address) if config.PortWithTLS { logger.Info("starting nixy on https://" + config.Address + ":" + config.Port) diff --git a/src/reload.go b/src/reload.go new file mode 100644 index 0000000..832a9e0 --- /dev/null +++ b/src/reload.go @@ -0,0 +1,373 @@ +package main + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + "time" + + "github.com/sirupsen/logrus" +) + +type NamespaceRenderingData struct { + LeaderVHost string `json:"-" toml:"leader_vhost"` + Leader LeaderController + Apps map[string]App + RoutingTag string `json:"-" toml:"routing_tag"` + KnownVHosts Vhosts +} + +type TemplateRenderingData struct { + Xproxy string + LeftDelimiter string `json:"-" toml:"left_delimiter"` + RightDelimiter string `json:"-" toml:"right_delimiter"` + MaxFailsUpstream *int `json:"max_fails,omitempty"` + FailTimeoutUpstream string `json:"fail_timeout,omitempty"` + SlowStartUpstream string `json:"slow_start,omitempty"` + Namespaces map[string]NamespaceRenderingData `json:"namespaces"` +} + +func reload() error { + start := time.Now() + var err error + config.LastUpdates.LastSync = time.Now() + if len(config.Nginxplusapiaddr) == 0 || config.Nginxplusapiaddr == "" { + //Nginx plus is disabled + err = updateAndReloadConfig() + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + }).Error("unable to reload nginx config") + go statsCount("reload.failed", 1) + go countFailedReloads.Inc() + return err + } + } else { + //Nginx plus is enabled + if config.NginxReloadDisabled { + logger.Warn("Template reload has been disabled") + } else { + logger.Info("Need to reload config") + err = updateAndReloadConfig() + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + }).Error("unable to update and reload nginx config. NPlus api calls will be skipped.") + return err + } else { + logger.Debug("No changes detected in vhosts. No config update is necessary. Upstream updates will happen via nplus apis") + } + } + err = nginxPlus() + } + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + }).Error("unable to generate nginx config") + go statsCount("reload.failed", 1) + go countFailedReloads.Inc() + return err + } + elapsed := time.Since(start) + logger.WithFields(logrus.Fields{ + "took": elapsed, + }).Info("config updated") + return nil + +} + +func updateAndReloadConfig() error { + start := time.Now() + config.LastUpdates.LastSync = time.Now() + vhosts := db.ReadAllKnownVhosts() + err := writeConf() + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + }).Error("unable to generate nginx config") + go statsCount("reload.failed", 1) + go countFailedReloads.Inc() + return err + } + config.LastUpdates.LastConfigValid = time.Now() + err = reloadNginx() + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + }).Error("unable to reload nginx") + go statsCount("reload.failed", 1) + go countFailedReloads.Inc() + } else { + elapsed := time.Since(start) + logger.WithFields(logrus.Fields{ + "took": elapsed, + }).Info("config updated") + go statsCount("reload.success", 1) + go statsTiming("reload.time", elapsed) + go countSuccessfulReloads.Inc() + go observeReloadTimeMetric(elapsed) + config.LastUpdates.LastNginxReload = time.Now() + db.UpdateLastKnownVhosts(vhosts) + } + return nil +} + +func createTemplateData(templateData *TemplateRenderingData) { + namespaceData := db.ReadAllNamespace() + staticData := db.ReadStaticData() + + templateData.Xproxy = staticData.Xproxy + templateData.LeftDelimiter = staticData.LeftDelimiter + templateData.RightDelimiter = staticData.RightDelimiter + templateData.FailTimeoutUpstream = staticData.FailTimeoutUpstream + templateData.MaxFailsUpstream = staticData.MaxFailsUpstream + templateData.SlowStartUpstream = staticData.SlowStartUpstream + + templateData.Namespaces = make(map[string]NamespaceRenderingData) + + for name, data := range namespaceData { + templateData.Namespaces[name] = NamespaceRenderingData{ + LeaderVHost: data.Drove.LeaderVHost, + Leader: data.Leader, + Apps: data.Apps, + KnownVHosts: data.KnownVHosts, + RoutingTag: data.Drove.RoutingTag, + } + } + return +} + +func writeConf() error { + config.RLock() + defer config.RUnlock() + allApps := db.ReadAllApps() + allLeaders := db.ReadAllLeaders() + + template, err := getTmpl() + if err != nil { + return err + } + logger.WithFields(logrus.Fields{ + "apps: ": allApps, + "leader": allLeaders, + }).Info("Config: ") + + parent := filepath.Dir(config.NginxConfig) + tmpFile, err := ioutil.TempFile(parent, ".nginx.conf.tmp-") + if err != nil { + return err + } + defer tmpFile.Close() + lastConfig = tmpFile.Name() + templateData := TemplateRenderingData{} + createTemplateData(&templateData) + logger.WithFields(logrus.Fields{ + "templateData": templateData, + }).Info("Template Data generated") + err = template.Execute(tmpFile, &templateData) + if err != nil { + return err + } + config.LastUpdates.LastConfigRendered = time.Now() + err = checkConf(tmpFile.Name()) + if err != nil { + logger.Error("Error in config generated") + return err + } + err = os.Rename(tmpFile.Name(), config.NginxConfig) + if err != nil { + return err + } + lastConfig = config.NginxConfig + return nil +} + +func nginxPlus() error { + config.RLock() + defer config.RUnlock() + allApps := db.ReadAllApps() + logger.WithFields(logrus.Fields{}).Info("Updating upstreams for the whitelisted drove tags") + for _, app := range allApps { + var newFormattedServers []string + for _, t := range app.Hosts { + var hostAndPortMapping string + ipRecords, error := net.LookupHost(string(t.Host)) + if error != nil { + logger.WithFields(logrus.Fields{ + "error": error, + "hostname": t.Host, + }).Error("dns lookup failed !! skipping the hostname") + continue + } + ipRecord := ipRecords[0] + hostAndPortMapping = ipRecord + ":" + fmt.Sprint(t.Port) + newFormattedServers = append(newFormattedServers, hostAndPortMapping) + + } + + logger.WithFields(logrus.Fields{ + "vhost": app.Vhost, + }).Info("app.vhost") + + logger.WithFields(logrus.Fields{ + "upstreams": newFormattedServers, + }).Info("nginx upstreams") + + logger.WithFields(logrus.Fields{ + "nginx": config.Nginxplusapiaddr, + }).Info("endpoint") + + endpoint := "http://" + config.Nginxplusapiaddr + "/api" + + tr := &http.Transport{ + MaxIdleConns: 30, + DisableCompression: true, + } + + client := &http.Client{Transport: tr} + c := NginxClient{endpoint, client} + nginxClient, error := NewNginxClient(c.httpClient, c.apiEndpoint) + if error != nil { + logger.WithFields(logrus.Fields{ + "error": error, + }).Error("unable to make call to nginx plus") + return error + } + upstreamtocheck := app.Vhost + var finalformattedServers []UpstreamServer + + for _, server := range newFormattedServers { + formattedServer := UpstreamServer{Server: server, MaxFails: config.MaxFailsUpstream, FailTimeout: config.FailTimeoutUpstream, SlowStart: config.SlowStartUpstream} + finalformattedServers = append(finalformattedServers, formattedServer) + } + + added, deleted, updated, error := nginxClient.UpdateHTTPServers(upstreamtocheck, finalformattedServers) + + if added != nil { + logger.WithFields(logrus.Fields{ + "nginx upstreams added": added, + }).Info("nginx upstreams added") + } + if deleted != nil { + logger.WithFields(logrus.Fields{ + "nginx upstreams deleted": deleted, + }).Info("nginx upstreams deleted") + } + if updated != nil { + logger.WithFields(logrus.Fields{ + "nginx upsteams updated": updated, + }).Info("nginx upstreams updated") + } + if error != nil { + logger.WithFields(logrus.Fields{ + "error": error, + }).Error("unable to update nginx upstreams") + return error + } + } + return nil +} + +func checkTmpl() error { + config.RLock() + defer config.RUnlock() + t, err := getTmpl() + if err != nil { + return err + } + err = t.Execute(ioutil.Discard, &config) + if err != nil { + return err + } + return nil +} + +func getTmpl() (*template.Template, error) { + logger.WithFields(logrus.Fields{ + "file": config.NginxTemplate, + }).Info("Reading template") + return template.New(filepath.Base(config.NginxTemplate)). + Delims(config.LeftDelimiter, config.RightDelimiter). + Funcs(template.FuncMap{ + "hasPrefix": strings.HasPrefix, + "hasSuffix": strings.HasPrefix, + "contains": strings.Contains, + "split": strings.Split, + "join": strings.Join, + "trim": strings.Trim, + "replace": strings.Replace, + "tolower": strings.ToLower, + "getenv": os.Getenv, + "datetime": time.Now}). + ParseFiles(config.NginxTemplate) +} + +func checkConf(path string) error { + // Always return OK if disabled in config. + if config.NginxIgnoreCheck { + return nil + } + // This is to allow arguments as well. Example "docker exec nginx..." + args := strings.Fields(config.NginxCmd) + head := args[0] + args = args[1:] + args = append(args, "-c") + args = append(args, path) + args = append(args, "-t") + cmd := exec.Command(head, args...) + //cmd := exec.Command(parts..., "-c", path, "-t") + var stderr bytes.Buffer + cmd.Stderr = &stderr + err := cmd.Run() // will wait for command to return + if err != nil { + msg := fmt.Sprint(err) + ": " + stderr.String() + errstd := errors.New(msg) + return errstd + } + return nil +} + +func reloadNginx() error { + // This is to allow arguments as well. Example "docker exec nginx..." + args := strings.Fields(config.NginxCmd) + head := args[0] + args = args[1:] + args = append(args, "-s") + args = append(args, "reload") + cmd := exec.Command(head, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + err := cmd.Run() // will wait for command to return + if err != nil { + msg := fmt.Sprint(err) + ": " + stderr.String() + errstd := errors.New(msg) + return errstd + } + return nil +} + +func reloadWorker() { + go func() { + // a ticker channel to limit reloads to drove, 1s is enough for now. + ticker := time.NewTicker(1 * time.Second) + for { + select { + case <-ticker.C: + select { + case <-reloadSignalQueue: + reload() // Trigger reload on event + default: + logger.Info("No signal to reload config") + } + } + } + }() +} From 627044f4b52386101768b1f16eb8afde1d86d88d Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Mon, 25 Nov 2024 14:25:07 +0530 Subject: [PATCH 05/29] added refersh elaspe time --- src/drove.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/drove.go b/src/drove.go index 5d0d92a..e886a79 100644 --- a/src/drove.go +++ b/src/drove.go @@ -491,7 +491,7 @@ func syncAppsAndVhosts(droveConfig DroveConfig, jsonapps *DroveApps, vhosts *Vho func refreshApps(namespace string, leaderShifted bool) error { logger.Debug("Reloading config for namespace" + namespace) - + start := time.Now() droveConfig, er := db.ReadDroveConfig(namespace) if er != nil { logger.WithFields(logrus.Fields{ @@ -531,6 +531,10 @@ func refreshApps(namespace string, leaderShifted bool) error { } triggerReload(namespace, !equal, leaderShifted, lastConfigUpdated) + elapsed := time.Since(start) + logger.WithFields(logrus.Fields{ + "took": elapsed, + }).Info("Apps updated") return nil } From ef179791580425bf2c480759a6731febb61957e3 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Mon, 25 Nov 2024 14:34:34 +0530 Subject: [PATCH 06/29] fixed template for 2 drove controller read --- src/nginx-header.tmpl | 19 ++++++++++++------- src/nixy.toml | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/nginx-header.tmpl b/src/nginx-header.tmpl index 36a6031..445a9f0 100644 --- a/src/nginx-header.tmpl +++ b/src/nginx-header.tmpl @@ -40,25 +40,29 @@ http { } } - {{if and .Namespaces.stage1.LeaderVHost .Namespaces.stage1.Leader.Endpoint}} - upstream {{.Namespaces.stage1.LeaderVHost}} { - server {{.Namespaces.stage1.Leader.Host}}:{{.Namespaces.stage1.Leader.Port}}; + {{- range $nid, $namespace := .Namespaces}} + {{if and $namespace.LeaderVHost $namespace.Leader.Endpoint}} + upstream {{$namespace.LeaderVHost}} { + server {{$namespace.Leader.Host}}:{{$namespace.Leader.Port}}; } server { listen 7000; - server_name {{.Namespaces.stage1.LeaderVHost}}; + server_name {{$namespace.LeaderVHost}}; location / { - proxy_set_header HOST {{.Namespaces.stage1.Leader.Host}}; + proxy_set_header HOST {{$namespace.Leader.Host}}; proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; proxy_connect_timeout 30; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; - proxy_pass http://{{.Namespaces.stage1.LeaderVHost}}; + proxy_pass http://{{$namespace.LeaderVHost}}; } } {{end}} - {{- range $id, $app := .Namespaces.stage1.Apps}} + {{end}} + + {{- range $nid, $namespace := .Namespaces}} + {{- range $id, $app := $namespace.Apps}} {{- range $gid, $hostgrp := .Groups}} upstream {{$gid}} { {{- range $hostgrp.Hosts}} @@ -103,4 +107,5 @@ http { } } {{- end}} + {{- end}} } diff --git a/src/nixy.toml b/src/nixy.toml index 0017d33..aa74e77 100644 --- a/src/nixy.toml +++ b/src/nixy.toml @@ -37,7 +37,18 @@ drove = ["https://localhost:8080"] # add all HA cluster nodes in priority order. user = "" # leave empty if no auth is required. pass = "" access_token = "O-Bearer " -leader_vhost = "drove.ssdev.test" +leader_vhost = "drove.ssdev.stage1" # Nixy realm, set this if you want to be able to filter your apps (e.g. when you have different loadbalancers which should expose different apps) # Put your subdomain here. It will be used to match the subdomain of the exposed apps -realm = "drove.xxxx.com" +realm = "blah1.xxx.stage1" + +[[namespaces]] +name = "stage2" +drove = ["https://localhost:8080"] # add all HA cluster nodes in priority order. +user = "" # leave empty if no auth is required. +pass = "" +access_token = "O-Bearer " +leader_vhost = "drove.ssdev.stage2" +# Nixy realm, set this if you want to be able to filter your apps (e.g. when you have different loadbalancers which should expose different apps) +# Put your subdomain here. It will be used to match the subdomain of the exposed apps +realm = "blah2.xxx.stage2" From 36b374d54d9ea6fb42c45b732a596e623d177a49 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Mon, 25 Nov 2024 15:07:33 +0530 Subject: [PATCH 07/29] added stats per namespace --- src/drove.go | 7 ++++--- src/prometheus.go | 45 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/src/drove.go b/src/drove.go index e886a79..bd8bfee 100644 --- a/src/drove.go +++ b/src/drove.go @@ -153,7 +153,7 @@ func refreshLeaderData(namespace string) bool { } if endpoint == "" { logger.Error("all endpoints are down") - go countAllEndpointsDownErrors.Inc() + go countAllEndpointsDownErrors.WithLabelValues(namespace).Inc() return false } currentLeader, err := db.ReadLeader(namespace) @@ -302,7 +302,7 @@ func namepaceEndpointHealth(namespace string) { "endpoint": es.Endpoint, "namespace": namespace, }).Error("endpoint is down") - go countEndpointDownErrors.Inc() + go countEndpointDownErrors.WithLabelValues(namespace).Inc() health.NamesapceEndpoints[namespace][i].Healthy = false health.NamesapceEndpoints[namespace][i].Message = err.Error() continue @@ -512,7 +512,7 @@ func refreshApps(namespace string, leaderShifted bool) error { }).Error("unable to sync from drove") } go statsCount("reload.failed", 1) - go countFailedReloads.Inc() + go countDroveAppSyncErrors.WithLabelValues(namespace).Inc() return err } equal := syncAppsAndVhosts(droveConfig, &jsonapps, &vhosts) @@ -532,6 +532,7 @@ func refreshApps(namespace string, leaderShifted bool) error { triggerReload(namespace, !equal, leaderShifted, lastConfigUpdated) elapsed := time.Since(start) + go observeAppRefreshTimeMetric(namespace, elapsed) logger.WithFields(logrus.Fields{ "took": elapsed, }).Info("Apps updated") diff --git a/src/prometheus.go b/src/prometheus.go index bbc4445..3323b23 100644 --- a/src/prometheus.go +++ b/src/prometheus.go @@ -38,54 +38,78 @@ var ( Help: "Total number of warnings about invalid subdomain label", }, ) - countDuplicateSubdomainLabelWarnings = prometheus.NewCounter( + countDuplicateSubdomainLabelWarnings = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: ns, Name: "duplicate_subdomain_label_warnings", Help: "Total number of warnings about duplicate subdomain label", }, + []string{"namespace"}, ) - countEndpointCheckFails = prometheus.NewCounter( + countEndpointCheckFails = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: ns, Name: "endpoint_check_fails", Help: "Total number of endpoint check failure errors", }, + []string{"namespace"}, ) - countEndpointDownErrors = prometheus.NewCounter( + countEndpointDownErrors = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: ns, Name: "endpoint_down_errors", Help: "Total number of endpoint down errors", }, + []string{"namespace"}, ) - countAllEndpointsDownErrors = prometheus.NewCounter( + countAllEndpointsDownErrors = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: ns, Name: "all_endpoints_down_errors", Help: "Total number of all endpoints down errors", }, + []string{"namespace"}, ) - countDroveStreamErrors = prometheus.NewCounter( + countDroveAppSyncErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: ns, + Name: "drove_app_sync_failed", + Help: "Total number of failed Nginx reloads", + }, + []string{"namespace"}, + ) + countDroveStreamErrors = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: ns, Name: "drove_stream_errors", Help: "Total number of Drove stream errors", }, + []string{"namespace"}, ) - countDroveStreamNoDataWarnings = prometheus.NewCounter( + countDroveStreamNoDataWarnings = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: ns, Name: "drove_stream_no_data_warnings", Help: "Total number of warnings about no data in Drove stream", }, + []string{"namespace"}, ) - countDroveEventsReceived = prometheus.NewCounter( + countDroveEventsReceived = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: ns, Name: "drove_events_received", Help: "Total number of received Drove events", }, + []string{"namespace"}, + ) + histogramAppRefereshDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: ns, + Name: "drove_app_referesh_duration", + Help: "Drove app referesh duration", + Buckets: prometheus.ExponentialBuckets(0.05, 2, 10), + }, + []string{"namespace"}, ) ) @@ -98,11 +122,18 @@ func setupPrometheusMetrics() { prometheus.MustRegister(countEndpointCheckFails) prometheus.MustRegister(countEndpointDownErrors) prometheus.MustRegister(countAllEndpointsDownErrors) + prometheus.MustRegister(countDroveAppSyncErrors) prometheus.MustRegister(countDroveStreamErrors) prometheus.MustRegister(countDroveStreamNoDataWarnings) prometheus.MustRegister(countDroveEventsReceived) + prometheus.MustRegister(histogramAppRefereshDuration) + } func observeReloadTimeMetric(e time.Duration) { histogramReloadDuration.Observe(float64(e) / float64(time.Second)) } + +func observeAppRefreshTimeMetric(namesapece string, e time.Duration) { + histogramAppRefereshDuration.WithLabelValues(namesapece).Observe(float64(e) / float64(time.Second)) +} From c42435120ec9d0239048cc44ca7c6fcbc1d5dfff Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Mon, 25 Nov 2024 17:06:10 +0530 Subject: [PATCH 08/29] fixed http client --- src/drove.go | 43 ++++++++++++++++--------------------------- src/nixy.go | 5 +++++ 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/drove.go b/src/drove.go index bd8bfee..c6204c2 100644 --- a/src/drove.go +++ b/src/drove.go @@ -84,7 +84,7 @@ func leaderController(endpoint string) *LeaderController { return controllerHost } -func fetchRecentEvents(client *http.Client, syncPoint *CurrSyncPoint, namespace string) (*DroveEventSummary, error) { +func fetchRecentEvents(httpClient *http.Client, syncPoint *CurrSyncPoint, namespace string) (*DroveEventSummary, error) { droveConfig, err := db.ReadDroveConfig(namespace) if err != nil { @@ -119,7 +119,7 @@ func fetchRecentEvents(client *http.Client, syncPoint *CurrSyncPoint, namespace if droveConfig.AccessToken != "" { req.Header.Add("Authorization", droveConfig.AccessToken) } - resp, err := client.Do(req) + resp, err := httpClient.Do(req) if err != nil { return nil, err } @@ -192,7 +192,7 @@ func newDroveClient(name string) *DroveClient { return &DroveClient{namespace: name, syncPoint: CurrSyncPoint{}, httpClient: &http.Client{ - Timeout: 0 * time.Second, + Timeout: time.Duration(config.apiTimeout) * time.Second, Transport: tr, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse @@ -239,7 +239,7 @@ func pollDroveEvents(name string) { reloadNeeded = true } if reloadNeeded || leaderShifted { - refreshApps(namespace, leaderShifted) + refreshApps(droveClient.httpClient, namespace, leaderShifted) } else { logger.Debug("Irrelevant events ignored") } @@ -257,18 +257,18 @@ func pollDroveEvents(name string) { func namepaceEndpointHealth(namespace string) { go func() { + healthCheckClient := &http.Client{ + Timeout: 5 * time.Second, + Transport: tr, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } ticker := time.NewTicker(2 * time.Second) for { select { case <-ticker.C: for i, es := range health.NamesapceEndpoints[namespace] { - client := &http.Client{ - Timeout: 5 * time.Second, - Transport: tr, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - } req, err := http.NewRequest("GET", es.Endpoint+"/apis/v1/ping", nil) if err != nil { logger.WithFields(logrus.Fields{ @@ -295,7 +295,7 @@ func namepaceEndpointHealth(namespace string) { if droveConfig.AccessToken != "" { req.Header.Add("Authorization", droveConfig.AccessToken) } - resp, err := client.Do(req) + resp, err := healthCheckClient.Do(req) if err != nil { logger.WithFields(logrus.Fields{ "error": err.Error(), @@ -325,9 +325,8 @@ func namepaceEndpointHealth(namespace string) { }() } -func fetchApps(droveConfig DroveConfig, jsonapps *DroveApps) error { +func fetchApps(httpClient *http.Client, droveConfig DroveConfig, jsonapps *DroveApps) error { var endpoint string - var timeout int = 5 for _, es := range health.NamesapceEndpoints[droveConfig.Name] { if es.Healthy { endpoint = es.Endpoint @@ -338,16 +337,6 @@ func fetchApps(droveConfig DroveConfig, jsonapps *DroveApps) error { err := errors.New("all endpoints are down") return err } - if config.apiTimeout != 0 { - timeout = config.apiTimeout - } - client := &http.Client{ - Timeout: time.Duration(timeout) * time.Second, - Transport: tr, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - } // fetch all apps and tasks with a single request. req, err := http.NewRequest("GET", endpoint+"/apis/v1/endpoints", nil) if err != nil { @@ -360,7 +349,7 @@ func fetchApps(droveConfig DroveConfig, jsonapps *DroveApps) error { if droveConfig.AccessToken != "" { req.Header.Add("Authorization", droveConfig.AccessToken) } - resp, err := client.Do(req) + resp, err := httpClient.Do(req) if err != nil { return err } @@ -489,7 +478,7 @@ func syncAppsAndVhosts(droveConfig DroveConfig, jsonapps *DroveApps, vhosts *Vho return false } -func refreshApps(namespace string, leaderShifted bool) error { +func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) error { logger.Debug("Reloading config for namespace" + namespace) start := time.Now() droveConfig, er := db.ReadDroveConfig(namespace) @@ -504,7 +493,7 @@ func refreshApps(namespace string, leaderShifted bool) error { jsonapps := DroveApps{} vhosts := Vhosts{} vhosts.Vhosts = map[string]bool{} - err := fetchApps(droveConfig, &jsonapps) + err := fetchApps(httpClient, droveConfig, &jsonapps) if err != nil || jsonapps.Status != "SUCCESS" { if err != nil { logger.WithFields(logrus.Fields{ diff --git a/src/nixy.go b/src/nixy.go index 166c8cc..51c38fa 100644 --- a/src/nixy.go +++ b/src/nixy.go @@ -322,6 +322,11 @@ func main() { config.EventRefreshIntervalSec = 2 } + //default apiTimeout + if config.apiTimeout <= 0 { + config.apiTimeout = 10 + } + setupPrometheusMetrics() setupDataManager() mux := mux.NewRouter() From 4453f92a651144d5556396f77ba35e6929db63a7 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Mon, 25 Nov 2024 18:20:08 +0530 Subject: [PATCH 09/29] code review fix: log fix --- src/datamanager.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/datamanager.go b/src/datamanager.go index fe5a786..9ec80ac 100644 --- a/src/datamanager.go +++ b/src/datamanager.go @@ -227,7 +227,7 @@ func (dm *DataManager) UpdateLeader(namespace string, leader LeaderController) e "namespace": namespace, "leader": ns.Leader, "time": ns.Timestamp, - }).Info("UpdateLeader data successfully") + }).Info("UpdateLeader data finished successfully") return nil } @@ -305,7 +305,7 @@ func (dm *DataManager) UpdateApps(namespace string, apps map[string]App) error { "namespace": namespace, "apps": dm.namespaces[namespace].Apps, "time": dm.namespaces[namespace].Timestamp, - }).Info("UpdateApps successfully") + }).Info("UpdateApps finished successfully") return nil } @@ -384,7 +384,7 @@ func (dm *DataManager) UpdateKnownVhosts(namespace string, KnownVHosts Vhosts) e "namespace": namespace, "knownHosts": dm.namespaces[namespace].KnownVHosts, "time": dm.namespaces[namespace].Timestamp, - }).Info("UpdateKnownVhosts successfully") + }).Info("UpdateKnownVhosts finished successfully") return nil } @@ -427,7 +427,7 @@ func (dm *DataManager) UpdateLastKnownVhosts(inLastKnownVhosts Vhosts) error { logger.WithFields(logrus.Fields{ "operation": operation, "LastKnownVhosts": dm.LastKnownVhosts, - }).Info("UpdateLastKnownVhosts successfully") + }).Info("UpdateLastKnownVhosts finished successfully") return nil } From f648fd21102f8705fdbde40ffa31b88304d87afe Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Tue, 26 Nov 2024 09:48:49 +0530 Subject: [PATCH 10/29] fixed reload logic --- src/drove.go | 211 +++++++++++++++++++++++++++------------------------ src/nixy.go | 59 +++++++++----- 2 files changed, 153 insertions(+), 117 deletions(-) diff --git a/src/drove.go b/src/drove.go index c6204c2..5d8e1fc 100644 --- a/src/drove.go +++ b/src/drove.go @@ -202,60 +202,124 @@ func newDroveClient(name string) *DroveClient { } } -func pollDroveEvents(name string) { +func pollingHandler(droveClient *DroveClient) { + namespace := droveClient.namespace + logger.WithFields(logrus.Fields{ + "at": time.Now(), + "namespace": namespace, + }).Debug("Syncing...") + + droveClient.syncPoint.Lock() + defer droveClient.syncPoint.Unlock() + + leaderShifted := refreshLeaderData(namespace) + eventSummary, err := fetchRecentEvents(droveClient.httpClient, &droveClient.syncPoint, namespace) + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + "namespace": namespace, + }).Error("unable to sync events from drove") + } else { + if len(eventSummary.EventsCount) > 0 { + logger.WithFields(logrus.Fields{ + "events": eventSummary.EventsCount, + "namespace": namespace, + "localTime": time.Now(), + }).Info("Events received") + reloadNeeded := false + if _, ok := eventSummary.EventsCount["APP_STATE_CHANGE"]; ok { + reloadNeeded = true + } + if _, ok := eventSummary.EventsCount["INSTANCE_STATE_CHANGE"]; ok { + reloadNeeded = true + } + if reloadNeeded || leaderShifted { + refreshApps(droveClient.httpClient, namespace, leaderShifted) + } else { + logger.Debug("Irrelevant events ignored") + } + } else { + logger.WithFields(logrus.Fields{ + "events": eventSummary.EventsCount, + "namespace": namespace, + }).Debug("New Events received") + } + } +} + +func pollDroveEvents(name string, forceRefreshSignal chan bool) { go func() { droveClient := newDroveClient(name) ticker := time.NewTicker(time.Duration(droveClient.eventRefreshInterval) * time.Second) - for range ticker.C { - func() { - namespace := droveClient.namespace - logger.WithFields(logrus.Fields{ - "at": time.Now(), - "namespace": namespace, - }).Debug("Syncing...") - - droveClient.syncPoint.Lock() - defer droveClient.syncPoint.Unlock() - - leaderShifted := refreshLeaderData(namespace) - eventSummary, err := fetchRecentEvents(droveClient.httpClient, &droveClient.syncPoint, namespace) - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err.Error(), - "namespace": namespace, - }).Error("unable to sync events from drove") - } else { - if len(eventSummary.EventsCount) > 0 { - logger.WithFields(logrus.Fields{ - "events": eventSummary.EventsCount, - "namespace": namespace, - "localTime": time.Now(), - }).Info("Events received") - reloadNeeded := false - if _, ok := eventSummary.EventsCount["APP_STATE_CHANGE"]; ok { - reloadNeeded = true - } - if _, ok := eventSummary.EventsCount["INSTANCE_STATE_CHANGE"]; ok { - reloadNeeded = true - } - if reloadNeeded || leaderShifted { - refreshApps(droveClient.httpClient, namespace, leaderShifted) - } else { - logger.Debug("Irrelevant events ignored") - } - } else { - logger.WithFields(logrus.Fields{ - "events": eventSummary.EventsCount, - "namespace": namespace, - }).Debug("New Events received") - } - } - }() + for { + select { + case <-ticker.C: + logger.Info("Refreshing drove events as per schedule for namesapce:" + droveClient.namespace) + pollingHandler(droveClient) + case <-forceRefreshSignal: + logger.Info("Refreshing drove events due to force referesh namesapce:" + droveClient.namespace) + pollingHandler(droveClient) + } } }() } -func namepaceEndpointHealth(namespace string) { +func endpointHealthHandler(healthCheckClient *http.Client, namespace string) { + for i, es := range health.NamesapceEndpoints[namespace] { + req, err := http.NewRequest("GET", es.Endpoint+"/apis/v1/ping", nil) + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + "endpoint": es.Endpoint, + }).Error("an error occurred creating endpoint health request") + health.NamesapceEndpoints[namespace][i].Healthy = false + health.NamesapceEndpoints[namespace][i].Message = err.Error() + continue + } + droveConfig, err := db.ReadDroveConfig(es.Namespace) + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err, + "endpoint": es.Endpoint, + }).Error("an error occurred reading drove config for health request") + health.NamesapceEndpoints[namespace][i].Healthy = false + health.NamesapceEndpoints[namespace][i].Message = err.Error() + continue + } + if droveConfig.User != "" { + req.SetBasicAuth(droveConfig.User, droveConfig.Pass) + } + if droveConfig.AccessToken != "" { + req.Header.Add("Authorization", droveConfig.AccessToken) + } + resp, err := healthCheckClient.Do(req) + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + "endpoint": es.Endpoint, + "namespace": namespace, + }).Error("endpoint is down") + go countEndpointDownErrors.WithLabelValues(namespace).Inc() + health.NamesapceEndpoints[namespace][i].Healthy = false + health.NamesapceEndpoints[namespace][i].Message = err.Error() + continue + } + resp.Body.Close() + if resp.StatusCode != 200 { + health.NamesapceEndpoints[namespace][i].Healthy = false + health.NamesapceEndpoints[namespace][i].Message = resp.Status + continue + } + health.NamesapceEndpoints[namespace][i].Healthy = true + health.NamesapceEndpoints[namespace][i].Message = "OK" + logger.WithFields(logrus.Fields{ + "host": es.Endpoint, + "namespace": namespace, + }).Debug(" Endpoint is healthy") + } +} + +func endpointHealth(namespace string) { go func() { healthCheckClient := &http.Client{ Timeout: 5 * time.Second, @@ -268,58 +332,7 @@ func namepaceEndpointHealth(namespace string) { for { select { case <-ticker.C: - for i, es := range health.NamesapceEndpoints[namespace] { - req, err := http.NewRequest("GET", es.Endpoint+"/apis/v1/ping", nil) - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err.Error(), - "endpoint": es.Endpoint, - }).Error("an error occurred creating endpoint health request") - health.NamesapceEndpoints[namespace][i].Healthy = false - health.NamesapceEndpoints[namespace][i].Message = err.Error() - continue - } - droveConfig, err := db.ReadDroveConfig(es.Namespace) - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err, - "endpoint": es.Endpoint, - }).Error("an error occurred reading drove config for health request") - health.NamesapceEndpoints[namespace][i].Healthy = false - health.NamesapceEndpoints[namespace][i].Message = err.Error() - continue - } - if droveConfig.User != "" { - req.SetBasicAuth(droveConfig.User, droveConfig.Pass) - } - if droveConfig.AccessToken != "" { - req.Header.Add("Authorization", droveConfig.AccessToken) - } - resp, err := healthCheckClient.Do(req) - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err.Error(), - "endpoint": es.Endpoint, - "namespace": namespace, - }).Error("endpoint is down") - go countEndpointDownErrors.WithLabelValues(namespace).Inc() - health.NamesapceEndpoints[namespace][i].Healthy = false - health.NamesapceEndpoints[namespace][i].Message = err.Error() - continue - } - resp.Body.Close() - if resp.StatusCode != 200 { - health.NamesapceEndpoints[namespace][i].Healthy = false - health.NamesapceEndpoints[namespace][i].Message = resp.Status - continue - } - health.NamesapceEndpoints[namespace][i].Healthy = true - health.NamesapceEndpoints[namespace][i].Message = "OK" - logger.WithFields(logrus.Fields{ - "host": es.Endpoint, - "namespace": namespace, - }).Debug(" Endpoint is healthy") - } + endpointHealthHandler(healthCheckClient, namespace) } } }() diff --git a/src/nixy.go b/src/nixy.go index 51c38fa..3542eee 100644 --- a/src/nixy.go +++ b/src/nixy.go @@ -171,6 +171,7 @@ func setupDataManager() { // Reload signal with buffer of two, because we dont really need more. var reloadSignalQueue = make(chan bool, 2) +var refreshSignalQueue = make(map[string]chan bool) // Global http transport for connection reuse var tr = &http.Transport{MaxIdleConnsPerHost: 10, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} @@ -193,32 +194,61 @@ func newHealth() Health { return h } +func setupDefaultConfig() { + //default refresh interval + if config.EventRefreshIntervalSec <= 0 { + config.EventRefreshIntervalSec = 2 + } + + //default apiTimeout + if config.apiTimeout <= 0 { + config.apiTimeout = 10 + } +} + +func setupRefreshChannel() { + for _, nsConfig := range config.DroveNamespaces { + refreshSignalQueue[nsConfig.Name] = make(chan bool, 2) + } +} + func setupPollEvents() { for _, nsConfig := range config.DroveNamespaces { - pollDroveEvents(nsConfig.Name) + pollDroveEvents(nsConfig.Name, refreshSignalQueue[nsConfig.Name]) } } func setupEndpointHealth() { for _, nsConfig := range config.DroveNamespaces { - namepaceEndpointHealth(nsConfig.Name) + endpointHealth(nsConfig.Name) + } +} + +func forceReload() bool { + queued := true + for _, nsConfig := range config.DroveNamespaces { + select { + case refreshSignalQueue[nsConfig.Name] <- true: // Add referesh to our signal channel, unless it is full of course. + default: + queued = false + } } + return queued } func nixyReload(w http.ResponseWriter, r *http.Request) { logger.WithFields(logrus.Fields{ "client": r.RemoteAddr, }).Info("Reload triggered") - select { - case reloadSignalQueue <- true: // Add reload to our queue channel, unless it is full of course. + queued := forceReload() + if queued { w.WriteHeader(202) fmt.Fprintln(w, "queued") return - default: - w.WriteHeader(202) - fmt.Fprintln(w, "queue is full") - return } + w.WriteHeader(202) + fmt.Fprintln(w, "queue is full") + return } func nixyHealth(w http.ResponseWriter, r *http.Request) { @@ -317,16 +347,8 @@ func main() { statsd = g2s.Noop() //fallback to Noop. } - //default refresh interval - if config.EventRefreshIntervalSec <= 0 { - config.EventRefreshIntervalSec = 2 - } - - //default apiTimeout - if config.apiTimeout <= 0 { - config.apiTimeout = 10 - } - + setupDefaultConfig() + setupRefreshChannel() setupPrometheusMetrics() setupDataManager() mux := mux.NewRouter() @@ -362,6 +384,7 @@ func main() { setupEndpointHealth() setupPollEvents() reloadWorker() //Reloader + // forceReload() logger.Info("Address:" + config.Address) if config.PortWithTLS { logger.Info("starting nixy on https://" + config.Address + ":" + config.Port) From 0d86623d62caa4455ab9b4c60b562d7b43d1602f Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Tue, 26 Nov 2024 14:18:01 +0530 Subject: [PATCH 11/29] change templates for docker --- docker/docker-nginx.tmpl.subst | 22 +++++++++++++--------- docker/docker-nixy.toml.subst | 13 ++++++------- docker/nginx.tmpl | 20 ++++++++++++-------- examples/nginx.tmpl | 22 +++++++++++++--------- examples/nixy.toml | 28 ++++++++++++++-------------- 5 files changed, 58 insertions(+), 47 deletions(-) diff --git a/docker/docker-nginx.tmpl.subst b/docker/docker-nginx.tmpl.subst index e7e9da7..01e31cc 100644 --- a/docker/docker-nginx.tmpl.subst +++ b/docker/docker-nginx.tmpl.subst @@ -39,29 +39,32 @@ http { return 503; } } - {{if and .LeaderVHost .Leader.Endpoint}} - upstream {{.LeaderVHost}} { - server {{.Leader.Host}}:{{.Leader.Port}} + {{- range $nid, $namespace := .Namespaces}} + {{if and $namespace.LeaderVHost $namespace.Leader.Endpoint}} + upstream {{$namespace.LeaderVHost}} { + server {{$namespace.Leader.Host}}:{{$namespace.Leader.Port}}; } server { listen 7000; - server_name {{.LeaderVHost}}; + server_name {{$namespace.LeaderVHost}}; location / { - proxy_set_header HOST {{.Leader.Host}}; + proxy_set_header HOST {{$namespace.Leader.Host}}; proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; proxy_connect_timeout 30; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; - proxy_pass http://{{.LeaderVHost}} + proxy_pass http://{{$namespace.LeaderVHost}}; } } {{end}} - {{- range $id, $app := .Apps}} + {{end}} + {{- range $nid, $namespace := .Namespaces}} + {{- range $id, $app := $namespace.Apps}} upstream {{$app.Vhost}} { {{- range $app.Hosts}} server {{ .Host }}:{{ .Port }}; - {{- end}} + {{end}} } server { listen 7000; @@ -76,5 +79,6 @@ http { proxy_pass http://{{$app.Vhost}}; } } - {{- end}} + {{end}} + {{end}} } diff --git a/docker/docker-nixy.toml.subst b/docker/docker-nixy.toml.subst index d616f09..93463e7 100644 --- a/docker/docker-nixy.toml.subst +++ b/docker/docker-nixy.toml.subst @@ -6,12 +6,6 @@ port = "6000" # X-Proxy header, defaults to hostname xproxy = "" -# Drove API -drove = [${DROVE_CONTROLLER_LIST}] # add all HA cluster nodes in priority order. -event_refresh_interval_sec = 5 -user = "${DROVE_USERNAME}" # leave empty if no auth is required. -pass = "${DROVE_PASSWORD}" - # Nginx nginx_config = "/etc/nginx/nginx.conf" nginx_template = "${TEMPLATE_PATH}" @@ -22,5 +16,10 @@ maxfailsupstream = 0 failtimeoutupstream = "1s" slowstartupstream = "0s" +# Drove API +[[namespaces]] +drove = [${DROVE_CONTROLLER_LIST}] # add all HA cluster nodes in priority order. +user = "${DROVE_USERNAME}" # leave empty if no auth is required. +pass = "${DROVE_PASSWORD}" leader_vhost = "${NGINX_DROVE_VHOST}" - +event_refresh_interval_sec = 5 diff --git a/docker/nginx.tmpl b/docker/nginx.tmpl index c597a4f..dbb5c40 100644 --- a/docker/nginx.tmpl +++ b/docker/nginx.tmpl @@ -38,24 +38,27 @@ http { return 503; } } - {{if and .LeaderVHost .Leader.Endpoint}} - upstream {{.LeaderVHost}} { - server {{.Leader.Host}}:{{.Leader.Port}}; + {{- range $nid, $namespace := .Namespaces}} + {{if and $namespace.LeaderVHost $namespace.Leader.Endpoint}} + upstream {{$namespace.LeaderVHost}} { + server {{$namespace.Leader.Host}}:{{$namespace.Leader.Port}}; } server { listen 7000; - server_name {{.LeaderVHost}}; + server_name {{$namespace.LeaderVHost}}; location / { - proxy_set_header HOST {{.Leader.Host}}; + proxy_set_header HOST {{$namespace.Leader.Host}}; proxy_connect_timeout 30; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; - proxy_pass http://{{.LeaderVHost}}; + proxy_pass http://{{$namespace.LeaderVHost}}; } } {{end}} - {{- range $id, $app := .Apps}} + {{end}} + {{- range $nid, $namespace := .Namespaces}} + {{- range $id, $app := $namespace.Apps}} upstream {{$app.Vhost}} { {{- range $app.Hosts}} server {{ .Host }}:{{ .Port }}; @@ -73,5 +76,6 @@ http { proxy_pass http://{{$app.Vhost}}; } } - {{- end}} + {{end}} + {{end}} } diff --git a/examples/nginx.tmpl b/examples/nginx.tmpl index c597a4f..63672a4 100644 --- a/examples/nginx.tmpl +++ b/examples/nginx.tmpl @@ -38,28 +38,31 @@ http { return 503; } } - {{if and .LeaderVHost .Leader.Endpoint}} - upstream {{.LeaderVHost}} { - server {{.Leader.Host}}:{{.Leader.Port}}; + {{- range $nid, $namespace := .Namespaces}} + {{if and $namespace.LeaderVHost $namespace.Leader.Endpoint}} + upstream {{$namespace.LeaderVHost}} { + server {{$namespace.Leader.Host}}:{{$namespace.Leader.Port}}; } server { listen 7000; - server_name {{.LeaderVHost}}; + server_name {{$namespace.LeaderVHost}}; location / { - proxy_set_header HOST {{.Leader.Host}}; + proxy_set_header HOST {{$namespace.Leader.Host}}; proxy_connect_timeout 30; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; - proxy_pass http://{{.LeaderVHost}}; + proxy_pass http://{{$namespace.LeaderVHost}}; } } {{end}} - {{- range $id, $app := .Apps}} + {{end}} + {{- range $nid, $namespace := .Namespaces}} + {{- range $id, $app := $namespace.Apps}} upstream {{$app.Vhost}} { {{- range $app.Hosts}} server {{ .Host }}:{{ .Port }}; - {{- end}} + {{end}} } server { listen 7000; @@ -73,5 +76,6 @@ http { proxy_pass http://{{$app.Vhost}}; } } - {{- end}} + {{end}} + {{end}} } diff --git a/examples/nixy.toml b/examples/nixy.toml index d10b9e7..bd0246a 100644 --- a/examples/nixy.toml +++ b/examples/nixy.toml @@ -9,17 +9,6 @@ port = "6000" # X-Proxy header, defaults to hostname xproxy = "" -# Drove API -drove = ["http://localhost:10000"] # add all HA cluster nodes in priority order. -#drove = ["http://localhost:4000", "http://localhost:5000"] # add all HA cluster nodes in priority order. -#drove = ["https://stg-nb6.drove.phonepe.com:443"] # add all HA cluster nodes in priority order. -event_refresh_interval_sec = 5 -user = "" # leave empty if no auth is required. -pass = "" -access_token = "O-Bearer OLYMPUS_TOKEN" -# Nixy realm, set this if you want to be able to filter your apps (e.g. when you have different loadbalancers which should expose different apps) -# Put your subdomain here. It will be used to match the subdomain of the exposed apps -#realm = "api.nixy.stg-drove.phonepe.nb6" # Nginx #nginxplusapiaddr="127.0.0.1" @@ -29,8 +18,6 @@ nginx_template = "./nginx-header.tmpl" nginx_cmd = "nginx" # optionally "openresty" or "docker exec nginx nginx" nginx_ignore_check = true # optionally disable nginx config test. Health check will always show OK. -traefiklabel = "traefik.backend" -traefikbackend = ["nixy-demo"] #nginxplusapiaddr = "10.57.11.218:2222" maxfailsupstream = 0 failtimeoutupstream = "1s" @@ -41,5 +28,18 @@ slowstartupstream = "0s" #addr = "10.57.8.171:8125" # optional for statistics #namespace = "nixy.my_mesos_cluster" #sample_rate = 100 -leader_vhost = "drove.ssdev.test" + +# Drove API +[[namespaces]] +drove = ["http://localhost:10000"] # add all HA cluster nodes in priority order. +#drove = ["http://localhost:4000", "http://localhost:5000"] # add all HA cluster nodes in priority order. +#drove = ["https://stg-nb6.drove.phonepe.com:443"] # add all HA cluster nodes in priority order. +event_refresh_interval_sec = 5 +user = "" # leave empty if no auth is required. +pass = "" +access_token = "O-Bearer OLYMPUS_TOKEN" +# Nixy realm, set this if you want to be able to filter your apps (e.g. when you have different loadbalancers which should expose different apps) +# Put your subdomain here. It will be used to match the subdomain of the exposed apps +#realm = "api.nixy.stg-drove.phonepe.nb6" +leader_vhost = "drove.ssdev.test" From 0dfeca875974d5dc6f984fa27abb79501d5d63c5 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Wed, 27 Nov 2024 17:15:08 +0530 Subject: [PATCH 12/29] added name in enviornment variable for docekr --- README.md | 3 +++ docker/docker-nixy.toml.subst | 1 + docker/entrypoint.sh | 1 + 3 files changed, 5 insertions(+) diff --git a/README.md b/README.md index 75a0ec0..f843ebd 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ The following environment variables can be used to tune the behaviour of the con | NGINX_DROVE_VHOST | **Optional** The vhost for drove endpoint to be configured. | If this is set, drove-gateway will expose the leader controller over the provided vhost. | | DROVE_USERNAME | **Optional.** Set to `guest` by default. | Username to login to drove. Read-only user is sufficient. | | DROVE_PASSWORD | **Optional.** Set to `guest` by default. | Password to drove cluster for the above username.| +| DROVE_CLUSTER_NAME | **Optional.** Set to `default` by default. | Name of drove cluster.| + You can run the container using following command for example: @@ -39,6 +41,7 @@ docker run --name dgw --rm \ -e TZ=Asia/Calcutta \ -e DROVE_USERNAME=guest \ -e DROVE_PASSWORD=guest \ + -e DROVE_CLUSTER_NAME=stage \ -e NGINX_DROVE_VHOST=drove.local \ --network host \ ghcr.io/phonepe/drove-gateway diff --git a/docker/docker-nixy.toml.subst b/docker/docker-nixy.toml.subst index 93463e7..47e98c3 100644 --- a/docker/docker-nixy.toml.subst +++ b/docker/docker-nixy.toml.subst @@ -21,5 +21,6 @@ slowstartupstream = "0s" drove = [${DROVE_CONTROLLER_LIST}] # add all HA cluster nodes in priority order. user = "${DROVE_USERNAME}" # leave empty if no auth is required. pass = "${DROVE_PASSWORD}" +name = "${DROVE_CLUSTER_NAME}" leader_vhost = "${NGINX_DROVE_VHOST}" event_refresh_interval_sec = 5 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 2936583..10ef703 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -31,6 +31,7 @@ else export DROVE_CONTROLLER_LIST=$(for host in "${hosts[@]}"; do echo "\"$host\""; done|paste -sd ',' -) export DROVE_USERNAME="${DROVE_USERNAME:-guest}" export DROVE_PASSWORD="${DROVE_PASSWORD:-guest}" + export DROVE_CLUSTER_NAME"${DROVE_PASSWORD:-default}" export NGINX_NUM_WORKERS=${NGINX_NUM_WORKERS:-2} From 5ed4a16d02914cf1557044617f67156449c0d220 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Wed, 27 Nov 2024 17:24:51 +0530 Subject: [PATCH 13/29] fixed logging --- examples/nixy.toml | 1 + src/reload.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/nixy.toml b/examples/nixy.toml index bd0246a..1676f8d 100644 --- a/examples/nixy.toml +++ b/examples/nixy.toml @@ -38,6 +38,7 @@ drove = ["http://localhost:10000"] # add all HA cluster nodes in priority order. event_refresh_interval_sec = 5 user = "" # leave empty if no auth is required. pass = "" +name = "default" access_token = "O-Bearer OLYMPUS_TOKEN" # Nixy realm, set this if you want to be able to filter your apps (e.g. when you have different loadbalancers which should expose different apps) # Put your subdomain here. It will be used to match the subdomain of the exposed apps diff --git a/src/reload.go b/src/reload.go index 832a9e0..fe93925 100644 --- a/src/reload.go +++ b/src/reload.go @@ -79,7 +79,7 @@ func reload() error { elapsed := time.Since(start) logger.WithFields(logrus.Fields{ "took": elapsed, - }).Info("config updated") + }).Info("config reloaded successfully") return nil } @@ -109,7 +109,7 @@ func updateAndReloadConfig() error { elapsed := time.Since(start) logger.WithFields(logrus.Fields{ "took": elapsed, - }).Info("config updated") + }).Info("config updated successfully") go statsCount("reload.success", 1) go statsTiming("reload.time", elapsed) go countSuccessfulReloads.Inc() From dbabd3ab44fe4fcbf38b6b9e5a6892267aff3e01 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Thu, 5 Dec 2024 15:54:20 +0530 Subject: [PATCH 14/29] code review fixes --- docker/docker-nginx.tmpl.subst | 4 +- docker/nginx.tmpl | 4 +- examples/nginx.tmpl | 4 +- examples/nixy.toml | 1 - src/datamanager.go | 46 -------------- src/drove.go | 109 ++++++++++++++++++++++----------- src/nginx-header.tmpl | 4 +- src/nixy.go | 57 ++++++++--------- src/reload.go | 86 ++++++++++++-------------- 9 files changed, 138 insertions(+), 177 deletions(-) diff --git a/docker/docker-nginx.tmpl.subst b/docker/docker-nginx.tmpl.subst index 01e31cc..fb3c9b4 100644 --- a/docker/docker-nginx.tmpl.subst +++ b/docker/docker-nginx.tmpl.subst @@ -59,8 +59,7 @@ http { } {{end}} {{end}} - {{- range $nid, $namespace := .Namespaces}} - {{- range $id, $app := $namespace.Apps}} + {{- range $id, $app := .Apps}} upstream {{$app.Vhost}} { {{- range $app.Hosts}} server {{ .Host }}:{{ .Port }}; @@ -80,5 +79,4 @@ http { } } {{end}} - {{end}} } diff --git a/docker/nginx.tmpl b/docker/nginx.tmpl index dbb5c40..ad47d75 100644 --- a/docker/nginx.tmpl +++ b/docker/nginx.tmpl @@ -57,8 +57,7 @@ http { } {{end}} {{end}} - {{- range $nid, $namespace := .Namespaces}} - {{- range $id, $app := $namespace.Apps}} + {{- range $id, $app := .Apps}} upstream {{$app.Vhost}} { {{- range $app.Hosts}} server {{ .Host }}:{{ .Port }}; @@ -77,5 +76,4 @@ http { } } {{end}} - {{end}} } diff --git a/examples/nginx.tmpl b/examples/nginx.tmpl index 63672a4..5ba5320 100644 --- a/examples/nginx.tmpl +++ b/examples/nginx.tmpl @@ -57,8 +57,7 @@ http { } {{end}} {{end}} - {{- range $nid, $namespace := .Namespaces}} - {{- range $id, $app := $namespace.Apps}} + {{- range $id, $app := .Apps}} upstream {{$app.Vhost}} { {{- range $app.Hosts}} server {{ .Host }}:{{ .Port }}; @@ -77,5 +76,4 @@ http { } } {{end}} - {{end}} } diff --git a/examples/nixy.toml b/examples/nixy.toml index 1676f8d..3e49683 100644 --- a/examples/nixy.toml +++ b/examples/nixy.toml @@ -39,7 +39,6 @@ event_refresh_interval_sec = 5 user = "" # leave empty if no auth is required. pass = "" name = "default" -access_token = "O-Bearer OLYMPUS_TOKEN" # Nixy realm, set this if you want to be able to filter your apps (e.g. when you have different loadbalancers which should expose different apps) # Put your subdomain here. It will be used to match the subdomain of the exposed apps #realm = "api.nixy.stg-drove.phonepe.nb6" diff --git a/src/datamanager.go b/src/datamanager.go index 9ec80ac..9ab89f9 100644 --- a/src/datamanager.go +++ b/src/datamanager.go @@ -178,30 +178,6 @@ func (dm *DataManager) ReadLeader(namespace string) (LeaderController, error) { return ns.Leader, nil //returning copy } -func (dm *DataManager) ReadAllLeaders() []LeaderController { - dm.mu.RLock() // Read lock to allow multiple concurrent reads - defer dm.mu.RUnlock() // Ensure the lock is always released - operation := "ReadAllLeaders" - - // Start the operation log - logger.WithFields(logrus.Fields{ - "operation": operation, - }).Debug("Attempting to read namespace data") - - var allLeaders []LeaderController - - for _, data := range dm.namespaces { - allLeaders = append(allLeaders, data.Leader) - } - // Log success - logger.WithFields(logrus.Fields{ - "operation": operation, - "allLeader": allLeaders, - }).Info("ReadAllLeaders successfully") - - return allLeaders //returning copy -} - func (dm *DataManager) UpdateLeader(namespace string, leader LeaderController) error { dm.mu.Lock() // Read lock to allow multiple concurrent reads defer dm.mu.Unlock() // Ensure the lock is always released @@ -258,28 +234,6 @@ func (dm *DataManager) ReadApps(namespace string) (map[string]App, error) { return ns.Apps, nil //returning copy } -func (dm *DataManager) ReadAllApps() map[string]App { - dm.mu.RLock() // Read lock to allow multiple concurrent reads - defer dm.mu.RUnlock() // Ensure the lock is always released - operation := "ReadAllApps" - - allApps := make(map[string]App) - - for _, data := range dm.namespaces { - for appId, appData := range data.Apps { - allApps[appId] = appData - } - } - // Log success - logger.WithFields(logrus.Fields{ - "operation": operation, - "allApps": allApps, - }).Info("ReadAllApps successfully") - - return allApps //returning copy - -} - func (dm *DataManager) UpdateApps(namespace string, apps map[string]App) error { dm.mu.Lock() // Read lock to allow multiple concurrent reads defer dm.mu.Unlock() // Ensure the lock is always released diff --git a/src/drove.go b/src/drove.go index 5d8e1fc..8e4243c 100644 --- a/src/drove.go +++ b/src/drove.go @@ -45,10 +45,9 @@ type DroveEventsApiResponse struct { } type DroveClient struct { - namespace string - syncPoint CurrSyncPoint - httpClient *http.Client - eventRefreshInterval int + namespace string + syncPoint CurrSyncPoint + httpClient *http.Client } type CurrSyncPoint struct { @@ -56,6 +55,8 @@ type CurrSyncPoint struct { LastSyncTime int64 } +var droveClients map[string]*DroveClient + func leaderController(endpoint string) *LeaderController { if endpoint == "" { return nil @@ -198,11 +199,12 @@ func newDroveClient(name string) *DroveClient { return http.ErrUseLastResponse }, }, - eventRefreshInterval: config.EventRefreshIntervalSec, } } -func pollingHandler(droveClient *DroveClient) { +func pollingHandler(droveClient *DroveClient, reloadChannel chan<- bool, waitGroup *sync.WaitGroup) { + defer waitGroup.Done() + appsRefreshed := false namespace := droveClient.namespace logger.WithFields(logrus.Fields{ "at": time.Now(), @@ -234,7 +236,7 @@ func pollingHandler(droveClient *DroveClient) { reloadNeeded = true } if reloadNeeded || leaderShifted { - refreshApps(droveClient.httpClient, namespace, leaderShifted) + appsRefreshed = refreshApps(droveClient.httpClient, namespace, leaderShifted) } else { logger.Debug("Irrelevant events ignored") } @@ -245,25 +247,62 @@ func pollingHandler(droveClient *DroveClient) { }).Debug("New Events received") } } + reloadChannel <- appsRefreshed +} + +func pollingEvents() { + var waitGroup sync.WaitGroup + reloadChannel := make(chan bool, len(droveClients)) + + for _, droveClient := range droveClients { + waitGroup.Add(1) + go pollingHandler(droveClient, reloadChannel, &waitGroup) + } + + waitGroup.Wait() + close(reloadChannel) + + reloadConfig := false + for result := range reloadChannel { + if result { + reloadConfig = true + break + } + } + logger.WithFields(logrus.Fields{ + "reloadConfig": reloadConfig, + }).Info("Drove poll event result") + + if reloadConfig { + reloadSignalQueue <- true + } + } -func pollDroveEvents(name string, forceRefreshSignal chan bool) { +func schedulePollDroveEvents() { go func() { - droveClient := newDroveClient(name) - ticker := time.NewTicker(time.Duration(droveClient.eventRefreshInterval) * time.Second) + ticker := time.NewTicker(time.Duration(config.EventRefreshIntervalSec) * time.Second) for { select { case <-ticker.C: - logger.Info("Refreshing drove events as per schedule for namesapce:" + droveClient.namespace) - pollingHandler(droveClient) - case <-forceRefreshSignal: - logger.Info("Refreshing drove events due to force referesh namesapce:" + droveClient.namespace) - pollingHandler(droveClient) + logger.Info("Refreshing drove events as per schedule") + pollingEvents() + case <-refreshSignalQueue: + logger.Info("Refreshing drove events due to force referesh") + pollingEvents() } } }() } +func setupPollEvents() { + droveClients = make(map[string]*DroveClient) + for _, nsConfig := range config.DroveNamespaces { + droveClients[nsConfig.Name] = newDroveClient(nsConfig.Name) + } + schedulePollDroveEvents() +} + func endpointHealthHandler(healthCheckClient *http.Client, namespace string) { for i, es := range health.NamesapceEndpoints[namespace] { req, err := http.NewRequest("GET", es.Endpoint+"/apis/v1/ping", nil) @@ -338,6 +377,12 @@ func endpointHealth(namespace string) { }() } +func setupEndpointHealth() { + for _, nsConfig := range config.DroveNamespaces { + endpointHealth(nsConfig.Name) + } +} + func fetchApps(httpClient *http.Client, droveConfig DroveConfig, jsonapps *DroveApps) error { var endpoint string for _, es := range health.NamesapceEndpoints[droveConfig.Name] { @@ -491,7 +536,7 @@ func syncAppsAndVhosts(droveConfig DroveConfig, jsonapps *DroveApps, vhosts *Vho return false } -func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) error { +func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) bool { logger.Debug("Reloading config for namespace" + namespace) start := time.Now() droveConfig, er := db.ReadDroveConfig(namespace) @@ -499,8 +544,7 @@ func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) logger.WithFields(logrus.Fields{ "namespace": namespace, }).Error("Error loading drove config") - er := errors.New("Error loading Drove Config") - return er + return false } jsonapps := DroveApps{} @@ -515,7 +559,7 @@ func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) } go statsCount("reload.failed", 1) go countDroveAppSyncErrors.WithLabelValues(namespace).Inc() - return err + return false } equal := syncAppsAndVhosts(droveConfig, &jsonapps, &vhosts) lastConfigUpdated := true @@ -529,29 +573,20 @@ func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) //Ideally only lastConfigUpdated can dictate if reload is required if equal && !leaderShifted && lastConfigUpdated { logger.Debug("no config changes") - return nil + return false } - triggerReload(namespace, !equal, leaderShifted, lastConfigUpdated) - elapsed := time.Since(start) - go observeAppRefreshTimeMetric(namespace, elapsed) - logger.WithFields(logrus.Fields{ - "took": elapsed, - }).Info("Apps updated") - return nil -} - -func triggerReload(namespace string, appsChanged, leaderShifted bool, lastConfigUpdated bool) { logger.WithFields(logrus.Fields{ "namespace": namespace, "lastConfigUpdated": lastConfigUpdated, "leaderShifted": leaderShifted, - "appsChanged": appsChanged, - }).Info("Trigging refresh") //logging exact reason of reload + "appsChanged": !equal, + }).Info("Config reload required") //logging exact reason of reload - select { - case reloadSignalQueue <- true: // add reload to channel, unless it is full of course. - default: - logger.Warn("reloadSignalQueue is full") - } + elapsed := time.Since(start) + go observeAppRefreshTimeMetric(namespace, elapsed) + logger.WithFields(logrus.Fields{ + "took": elapsed, + }).Info("Apps updated") + return true } diff --git a/src/nginx-header.tmpl b/src/nginx-header.tmpl index 445a9f0..fc9b4ed 100644 --- a/src/nginx-header.tmpl +++ b/src/nginx-header.tmpl @@ -61,8 +61,7 @@ http { {{end}} {{end}} - {{- range $nid, $namespace := .Namespaces}} - {{- range $id, $app := $namespace.Apps}} + {{- range $id, $app := .Apps}} {{- range $gid, $hostgrp := .Groups}} upstream {{$gid}} { {{- range $hostgrp.Hosts}} @@ -107,5 +106,4 @@ http { } } {{- end}} - {{- end}} } diff --git a/src/nixy.go b/src/nixy.go index 3542eee..91424dc 100644 --- a/src/nixy.go +++ b/src/nixy.go @@ -3,6 +3,7 @@ package main import ( "crypto/tls" "encoding/json" + "errors" "flag" "fmt" "io/ioutil" @@ -171,7 +172,7 @@ func setupDataManager() { // Reload signal with buffer of two, because we dont really need more. var reloadSignalQueue = make(chan bool, 2) -var refreshSignalQueue = make(map[string]chan bool) +var refreshSignalQueue = make(chan bool, 2) // Global http transport for connection reuse var tr = &http.Transport{MaxIdleConnsPerHost: 10, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} @@ -195,6 +196,11 @@ func newHealth() Health { } func setupDefaultConfig() { + // Lets default empty Xproxy to hostname. + if config.Xproxy == "" { + config.Xproxy, _ = os.Hostname() + } + //default refresh interval if config.EventRefreshIntervalSec <= 0 { config.EventRefreshIntervalSec = 2 @@ -206,41 +212,25 @@ func setupDefaultConfig() { } } -func setupRefreshChannel() { - for _, nsConfig := range config.DroveNamespaces { - refreshSignalQueue[nsConfig.Name] = make(chan bool, 2) - } -} - -func setupPollEvents() { - for _, nsConfig := range config.DroveNamespaces { - pollDroveEvents(nsConfig.Name, refreshSignalQueue[nsConfig.Name]) - } -} - -func setupEndpointHealth() { +func validateConfig() error { for _, nsConfig := range config.DroveNamespaces { - endpointHealth(nsConfig.Name) - } -} - -func forceReload() bool { - queued := true - for _, nsConfig := range config.DroveNamespaces { - select { - case refreshSignalQueue[nsConfig.Name] <- true: // Add referesh to our signal channel, unless it is full of course. - default: - queued = false + if nsConfig.Name == "" { + return errors.New("Drove namespace name is mandatory") } } - return queued + return nil } func nixyReload(w http.ResponseWriter, r *http.Request) { logger.WithFields(logrus.Fields{ "client": r.RemoteAddr, }).Info("Reload triggered") - queued := forceReload() + queued := true + select { + case refreshSignalQueue <- true: // Add referesh to our signal channel, unless it is full of course. + default: + queued = false + } if queued { w.WriteHeader(202) fmt.Fprintln(w, "queued") @@ -334,11 +324,14 @@ func main() { "error": err.Error(), }).Fatal("problem parsing config") } - // Lets default empty Xproxy to hostname. - if config.Xproxy == "" { - config.Xproxy, _ = os.Hostname() - } setloglevel() + err = validateConfig() + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + }).Fatal("problem in config") + } + statsd, err = setupStatsd() if err != nil { logger.WithFields(logrus.Fields{ @@ -346,9 +339,7 @@ func main() { }).Error("unable to Dial statsd") statsd = g2s.Noop() //fallback to Noop. } - setupDefaultConfig() - setupRefreshChannel() setupPrometheusMetrics() setupDataManager() mux := mux.NewRouter() diff --git a/src/reload.go b/src/reload.go index fe93925..b0dce6b 100644 --- a/src/reload.go +++ b/src/reload.go @@ -18,14 +18,11 @@ import ( ) type NamespaceRenderingData struct { - LeaderVHost string `json:"-" toml:"leader_vhost"` + LeaderVHost string Leader LeaderController - Apps map[string]App - RoutingTag string `json:"-" toml:"routing_tag"` - KnownVHosts Vhosts } -type TemplateRenderingData struct { +type RenderingData struct { Xproxy string LeftDelimiter string `json:"-" toml:"left_delimiter"` RightDelimiter string `json:"-" toml:"right_delimiter"` @@ -33,15 +30,18 @@ type TemplateRenderingData struct { FailTimeoutUpstream string `json:"fail_timeout,omitempty"` SlowStartUpstream string `json:"slow_start,omitempty"` Namespaces map[string]NamespaceRenderingData `json:"namespaces"` + Apps map[string]App } func reload() error { start := time.Now() var err error + data := RenderingData{} + createRenderingData(&data) config.LastUpdates.LastSync = time.Now() if len(config.Nginxplusapiaddr) == 0 || config.Nginxplusapiaddr == "" { //Nginx plus is disabled - err = updateAndReloadConfig() + err = updateAndReloadConfig(&data) if err != nil { logger.WithFields(logrus.Fields{ "error": err.Error(), @@ -56,7 +56,7 @@ func reload() error { logger.Warn("Template reload has been disabled") } else { logger.Info("Need to reload config") - err = updateAndReloadConfig() + err = updateAndReloadConfig(&data) if err != nil { logger.WithFields(logrus.Fields{ "error": err.Error(), @@ -66,7 +66,7 @@ func reload() error { logger.Debug("No changes detected in vhosts. No config update is necessary. Upstream updates will happen via nplus apis") } } - err = nginxPlus() + err = nginxPlus(&data) } if err != nil { logger.WithFields(logrus.Fields{ @@ -84,11 +84,11 @@ func reload() error { } -func updateAndReloadConfig() error { +func updateAndReloadConfig(data *RenderingData) error { start := time.Now() config.LastUpdates.LastSync = time.Now() vhosts := db.ReadAllKnownVhosts() - err := writeConf() + err := writeConf(data) if err != nil { logger.WithFields(logrus.Fields{ "error": err.Error(), @@ -120,45 +120,44 @@ func updateAndReloadConfig() error { return nil } -func createTemplateData(templateData *TemplateRenderingData) { +func createRenderingData(data *RenderingData) { namespaceData := db.ReadAllNamespace() staticData := db.ReadStaticData() - templateData.Xproxy = staticData.Xproxy - templateData.LeftDelimiter = staticData.LeftDelimiter - templateData.RightDelimiter = staticData.RightDelimiter - templateData.FailTimeoutUpstream = staticData.FailTimeoutUpstream - templateData.MaxFailsUpstream = staticData.MaxFailsUpstream - templateData.SlowStartUpstream = staticData.SlowStartUpstream + data.Xproxy = staticData.Xproxy + data.LeftDelimiter = staticData.LeftDelimiter + data.RightDelimiter = staticData.RightDelimiter + data.FailTimeoutUpstream = staticData.FailTimeoutUpstream + data.MaxFailsUpstream = staticData.MaxFailsUpstream + data.SlowStartUpstream = staticData.SlowStartUpstream + data.Namespaces = make(map[string]NamespaceRenderingData) - templateData.Namespaces = make(map[string]NamespaceRenderingData) + allApps := make(map[string]App) - for name, data := range namespaceData { - templateData.Namespaces[name] = NamespaceRenderingData{ - LeaderVHost: data.Drove.LeaderVHost, - Leader: data.Leader, - Apps: data.Apps, - KnownVHosts: data.KnownVHosts, - RoutingTag: data.Drove.RoutingTag, + for name, nmData := range namespaceData { + data.Namespaces[name] = NamespaceRenderingData{ + LeaderVHost: nmData.Drove.LeaderVHost, + Leader: nmData.Leader, + } + for appId, appData := range nmData.Apps { + allApps[appId] = appData //TODO:fix this } } + data.Apps = allApps + logger.WithFields(logrus.Fields{ + "data": data, + }).Info("Rendering data generated") return } -func writeConf() error { +func writeConf(data *RenderingData) error { config.RLock() defer config.RUnlock() - allApps := db.ReadAllApps() - allLeaders := db.ReadAllLeaders() template, err := getTmpl() if err != nil { return err } - logger.WithFields(logrus.Fields{ - "apps: ": allApps, - "leader": allLeaders, - }).Info("Config: ") parent := filepath.Dir(config.NginxConfig) tmpFile, err := ioutil.TempFile(parent, ".nginx.conf.tmp-") @@ -167,12 +166,7 @@ func writeConf() error { } defer tmpFile.Close() lastConfig = tmpFile.Name() - templateData := TemplateRenderingData{} - createTemplateData(&templateData) - logger.WithFields(logrus.Fields{ - "templateData": templateData, - }).Info("Template Data generated") - err = template.Execute(tmpFile, &templateData) + err = template.Execute(tmpFile, &data) if err != nil { return err } @@ -190,12 +184,12 @@ func writeConf() error { return nil } -func nginxPlus() error { +func nginxPlus(data *RenderingData) error { + //Current implementation only updates AppVhosts, does not suppport routing tag & LeaderVhost config.RLock() defer config.RUnlock() - allApps := db.ReadAllApps() - logger.WithFields(logrus.Fields{}).Info("Updating upstreams for the whitelisted drove tags") - for _, app := range allApps { + logger.WithFields(logrus.Fields{"apps": data.Apps}).Info("Updating upstreams for the whitelisted drove tags") + for _, app := range data.Apps { var newFormattedServers []string for _, t := range app.Hosts { var hostAndPortMapping string @@ -361,12 +355,8 @@ func reloadWorker() { for { select { case <-ticker.C: - select { - case <-reloadSignalQueue: - reload() // Trigger reload on event - default: - logger.Info("No signal to reload config") - } + <-reloadSignalQueue + reload() } } }() From 8e43c8fda3d81058ef38a5fb1420999eb4e38eb6 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Thu, 5 Dec 2024 16:50:25 +0530 Subject: [PATCH 15/29] merging app if vhostname matches across differetn drove cluster --- src/reload.go | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/reload.go b/src/reload.go index b0dce6b..f299ee0 100644 --- a/src/reload.go +++ b/src/reload.go @@ -139,8 +139,42 @@ func createRenderingData(data *RenderingData) { LeaderVHost: nmData.Drove.LeaderVHost, Leader: nmData.Leader, } + //Merging App if already exists for appId, appData := range nmData.Apps { - allApps[appId] = appData //TODO:fix this + if existingAppData, ok := allApps[appId]; ok { + //Appending hosts + existingAppData.Hosts = append(existingAppData.Hosts, appData.Hosts...) + + //adding same as that of app refresh logic + if existingAppData.Tags == nil { + existingAppData.Tags = make(map[string]string) + } + + for tagK, tagV := range appData.Tags { + existingAppData.Tags[tagK] = tagV + } + + for groupName, groupData := range appData.Groups { + if existingGroup, ok := existingAppData.Groups[groupName]; ok { + existingGroup.Hosts = append(existingGroup.Hosts, groupData.Hosts...) + + if existingGroup.Tags == nil { + existingGroup.Tags = make(map[string]string) + } + + for tn, tv := range groupData.Tags { + existingGroup.Tags[tn] = tv + } + + existingAppData.Groups[groupName] = existingGroup + } else { + existingAppData.Groups[groupName] = groupData + } + } + allApps[appId] = existingAppData + } else { + allApps[appId] = appData + } } } data.Apps = allApps From 2679c76cac03f1574df33f1a7d058330b921860e Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Fri, 6 Dec 2024 10:17:57 +0530 Subject: [PATCH 16/29] nil check in merging apps --- src/reload.go | 44 +++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/src/reload.go b/src/reload.go index f299ee0..529126b 100644 --- a/src/reload.go +++ b/src/reload.go @@ -145,30 +145,40 @@ func createRenderingData(data *RenderingData) { //Appending hosts existingAppData.Hosts = append(existingAppData.Hosts, appData.Hosts...) - //adding same as that of app refresh logic + //adding same as that of app refresh logic in drove client if existingAppData.Tags == nil { existingAppData.Tags = make(map[string]string) } - - for tagK, tagV := range appData.Tags { - existingAppData.Tags[tagK] = tagV + if appData.Tags != nil { + for tagK, tagV := range appData.Tags { + existingAppData.Tags[tagK] = tagV + } } - for groupName, groupData := range appData.Groups { - if existingGroup, ok := existingAppData.Groups[groupName]; ok { - existingGroup.Hosts = append(existingGroup.Hosts, groupData.Hosts...) - - if existingGroup.Tags == nil { - existingGroup.Tags = make(map[string]string) - } + if existingAppData.Groups == nil { + existingAppData.Groups = make(map[string]HostGroup) + } - for tn, tv := range groupData.Tags { - existingGroup.Tags[tn] = tv + if appData.Groups != nil { + for groupName, groupData := range appData.Groups { + if existingGroup, ok := existingAppData.Groups[groupName]; ok { + + //Appending hosts + existingGroup.Hosts = append(existingGroup.Hosts, groupData.Hosts...) + + if existingGroup.Tags == nil { + existingGroup.Tags = make(map[string]string) + } + + if groupData.Tags != nil { + for tn, tv := range groupData.Tags { + existingGroup.Tags[tn] = tv + } + } + existingAppData.Groups[groupName] = existingGroup + } else { + existingAppData.Groups[groupName] = groupData } - - existingAppData.Groups[groupName] = existingGroup - } else { - existingAppData.Groups[groupName] = groupData } } allApps[appId] = existingAppData From 2894a4e231e5db4dc6070145005a453b78e8e195 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Fri, 6 Dec 2024 10:19:12 +0530 Subject: [PATCH 17/29] code review fix --- src/nixy.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/nixy.toml b/src/nixy.toml index aa74e77..8650424 100644 --- a/src/nixy.toml +++ b/src/nixy.toml @@ -36,7 +36,6 @@ name = "stage1" drove = ["https://localhost:8080"] # add all HA cluster nodes in priority order. user = "" # leave empty if no auth is required. pass = "" -access_token = "O-Bearer " leader_vhost = "drove.ssdev.stage1" # Nixy realm, set this if you want to be able to filter your apps (e.g. when you have different loadbalancers which should expose different apps) # Put your subdomain here. It will be used to match the subdomain of the exposed apps @@ -47,7 +46,6 @@ name = "stage2" drove = ["https://localhost:8080"] # add all HA cluster nodes in priority order. user = "" # leave empty if no auth is required. pass = "" -access_token = "O-Bearer " leader_vhost = "drove.ssdev.stage2" # Nixy realm, set this if you want to be able to filter your apps (e.g. when you have different loadbalancers which should expose different apps) # Put your subdomain here. It will be used to match the subdomain of the exposed apps From 7d58fb169a151348cdbcaced52c0b07fabac0a6b Mon Sep 17 00:00:00 2001 From: Vishnu Naini Date: Wed, 25 Dec 2024 01:24:01 +0530 Subject: [PATCH 18/29] nginx-plus: use upstream client as-is and add version check --- src/go.mod | 1 + src/nginx.go | 1385 ------------------------------------------------- src/reload.go | 10 +- 3 files changed, 6 insertions(+), 1390 deletions(-) delete mode 100644 src/nginx.go diff --git a/src/go.mod b/src/go.mod index dea66c2..1b533e3 100644 --- a/src/go.mod +++ b/src/go.mod @@ -18,6 +18,7 @@ require ( github.com/sirupsen/logrus v1.9.3 golang.org/x/crypto v0.21.0 golang.org/x/sys v0.18.0 + github.com/nginxinc/nginx-plus-go-client v1.3.0 ) require ( diff --git a/src/nginx.go b/src/nginx.go deleted file mode 100644 index 729109c..0000000 --- a/src/nginx.go +++ /dev/null @@ -1,1385 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "reflect" - "strings" -) - -const ( - // APIVersion is a version of NGINX Plus API. - APIVersion = 6 - - pathNotFoundCode = "PathNotFound" - streamContext = true - httpContext = false - defaultServerPort = "80" -) - -// Default values for servers in Upstreams. -var ( - defaultMaxConns = 0 - defaultMaxFails = 1 - defaultFailTimeout = "10s" - defaultSlowStart = "0s" - defaultBackup = false - defaultDown = false - defaultWeight = 1 -) - -// NginxClient lets you access NGINX Plus API. -type NginxClient struct { - apiEndpoint string - httpClient *http.Client -} - -type versions []int - -// UpstreamServer lets you configure HTTP upstreams. -type UpstreamServer struct { - ID int `json:"id,omitempty"` - Server string `json:"server"` - MaxConns *int `json:"max_conns,omitempty"` - MaxFails *int `json:"max_fails,omitempty"` - FailTimeout string `json:"fail_timeout,omitempty"` - SlowStart string `json:"slow_start,omitempty"` - Route string `json:"route,omitempty"` - Backup *bool `json:"backup,omitempty"` - Down *bool `json:"down,omitempty"` - Drain bool `json:"drain,omitempty"` - Weight *int `json:"weight,omitempty"` - Service string `json:"service,omitempty"` -} - -// StreamUpstreamServer lets you configure Stream upstreams. -type StreamUpstreamServer struct { - ID int `json:"id,omitempty"` - Server string `json:"server"` - MaxConns *int `json:"max_conns,omitempty"` - MaxFails *int `json:"max_fails,omitempty"` - FailTimeout string `json:"fail_timeout,omitempty"` - SlowStart string `json:"slow_start,omitempty"` - Backup *bool `json:"backup,omitempty"` - Down *bool `json:"down,omitempty"` - Weight *int `json:"weight,omitempty"` - Service string `json:"service,omitempty"` -} - -type apiErrorResponse struct { - Error apiError - RequestID string `json:"request_id"` - Href string -} - -func (resp *apiErrorResponse) toString() string { - return fmt.Sprintf("error.status=%v; error.text=%v; error.code=%v; request_id=%v; href=%v", - resp.Error.Status, resp.Error.Text, resp.Error.Code, resp.RequestID, resp.Href) -} - -type apiError struct { - Status int - Text string - Code string -} - -type internalError struct { - apiError - err string -} - -// Error allows internalError to match the Error interface. -func (internalError *internalError) Error() string { - return internalError.err -} - -// Wrap is a way of including current context while preserving previous error information, -// similar to `return fmt.Errorf("error doing foo, err: %v", err)` but for our internalError type. -func (internalError *internalError) Wrap(err string) *internalError { - internalError.err = fmt.Sprintf("%v. %v", err, internalError.err) - return internalError -} - -// Stats represents NGINX Plus stats fetched from the NGINX Plus API. -// https://nginx.org/en/docs/http/ngx_http_api_module.html -type Stats struct { - NginxInfo NginxInfo - Processes Processes - Connections Connections - Slabs Slabs - HTTPRequests HTTPRequests - SSL SSL - ServerZones ServerZones - Upstreams Upstreams - StreamServerZones StreamServerZones - StreamUpstreams StreamUpstreams - StreamZoneSync *StreamZoneSync - LocationZones LocationZones - Resolvers Resolvers -} - -// NginxInfo contains general information about NGINX Plus. -type NginxInfo struct { - Version string - Build string - Address string - Generation uint64 - LoadTimestamp string `json:"load_timestamp"` - Timestamp string - ProcessID uint64 `json:"pid"` - ParentProcessID uint64 `json:"ppid"` -} - -// Connections represents connection related stats. -type Connections struct { - Accepted uint64 - Dropped uint64 - Active uint64 - Idle uint64 -} - -// Slabs is map of slab stats by zone name. -type Slabs map[string]Slab - -// Slab represents slab related stats. -type Slab struct { - Pages Pages - Slots Slots -} - -// Pages represents the slab memory usage stats. -type Pages struct { - Used uint64 - Free uint64 -} - -// Slots is a map of slots by slot size -type Slots map[string]Slot - -// Slot represents slot related stats. -type Slot struct { - Used uint64 - Free uint64 - Reqs uint64 - Fails uint64 -} - -// HTTPRequests represents HTTP request related stats. -type HTTPRequests struct { - Total uint64 - Current uint64 -} - -// SSL represents SSL related stats. -type SSL struct { - Handshakes uint64 - HandshakesFailed uint64 `json:"handshakes_failed"` - SessionReuses uint64 `json:"session_reuses"` -} - -// ServerZones is map of server zone stats by zone name -type ServerZones map[string]ServerZone - -// ServerZone represents server zone related stats. -type ServerZone struct { - Processing uint64 - Requests uint64 - Responses Responses - Discarded uint64 - Received uint64 - Sent uint64 -} - -// StreamServerZones is map of stream server zone stats by zone name. -type StreamServerZones map[string]StreamServerZone - -// StreamServerZone represents stream server zone related stats. -type StreamServerZone struct { - Processing uint64 - Connections uint64 - Sessions Sessions - Discarded uint64 - Received uint64 - Sent uint64 -} - -// StreamZoneSync represents the sync information per each shared memory zone and the sync information per node in a cluster -type StreamZoneSync struct { - Zones map[string]SyncZone - Status StreamZoneSyncStatus -} - -// SyncZone represents the synchronization status of a shared memory zone -type SyncZone struct { - RecordsPending uint64 `json:"records_pending"` - RecordsTotal uint64 `json:"records_total"` -} - -// StreamZoneSyncStatus represents the status of a shared memory zone -type StreamZoneSyncStatus struct { - BytesIn uint64 `json:"bytes_in"` - MsgsIn uint64 `json:"msgs_in"` - MsgsOut uint64 `json:"msgs_out"` - BytesOut uint64 `json:"bytes_out"` - NodesOnline uint64 `json:"nodes_online"` -} - -// Responses represents HTTP response related stats. -type Responses struct { - Responses1xx uint64 `json:"1xx"` - Responses2xx uint64 `json:"2xx"` - Responses3xx uint64 `json:"3xx"` - Responses4xx uint64 `json:"4xx"` - Responses5xx uint64 `json:"5xx"` - Total uint64 -} - -// Sessions represents stream session related stats. -type Sessions struct { - Sessions2xx uint64 `json:"2xx"` - Sessions4xx uint64 `json:"4xx"` - Sessions5xx uint64 `json:"5xx"` - Total uint64 -} - -// Upstreams is a map of upstream stats by upstream name. -type Upstreams map[string]Upstream - -// Upstream represents upstream related stats. -type Upstream struct { - Peers []Peer - Keepalives int - Zombies int - Zone string - Queue Queue -} - -// StreamUpstreams is a map of stream upstream stats by upstream name. -type StreamUpstreams map[string]StreamUpstream - -// StreamUpstream represents stream upstream related stats. -type StreamUpstream struct { - Peers []StreamPeer - Zombies int - Zone string -} - -// Queue represents queue related stats for an upstream. -type Queue struct { - Size int - MaxSize int `json:"max_size"` - Overflows uint64 -} - -// Peer represents peer (upstream server) related stats. -type Peer struct { - ID int - Server string - Service string - Name string - Backup bool - Weight int - State string - Active uint64 - MaxConns int `json:"max_conns"` - Requests uint64 - Responses Responses - Sent uint64 - Received uint64 - Fails uint64 - Unavail uint64 - HealthChecks HealthChecks `json:"health_checks"` - Downtime uint64 - Downstart string - Selected string - HeaderTime uint64 `json:"header_time"` - ResponseTime uint64 `json:"response_time"` -} - -// StreamPeer represents peer (stream upstream server) related stats. -type StreamPeer struct { - ID int - Server string - Service string - Name string - Backup bool - Weight int - State string - Active uint64 - MaxConns int `json:"max_conns"` - Connections uint64 - ConnectTime int `json:"connect_time"` - FirstByteTime int `json:"first_byte_time"` - ResponseTime uint64 `json:"response_time"` - Sent uint64 - Received uint64 - Fails uint64 - Unavail uint64 - HealthChecks HealthChecks `json:"health_checks"` - Downtime uint64 - Downstart string - Selected string -} - -// HealthChecks represents health check related stats for a peer. -type HealthChecks struct { - Checks uint64 - Fails uint64 - Unhealthy uint64 - LastPassed bool `json:"last_passed"` -} - -// LocationZones represents location_zones related stats -type LocationZones map[string]LocationZone - -// Resolvers represents resolvers related stats -type Resolvers map[string]Resolver - -// LocationZone represents location_zones related stats -type LocationZone struct { - Requests int64 - Responses Responses - Discarded int64 - Received int64 - Sent int64 -} - -// Resolver represents resolvers related stats -type Resolver struct { - Requests ResolverRequests `json:"requests"` - Responses ResolverResponses `json:"responses"` -} - -// ResolverRequests represents resolver requests -type ResolverRequests struct { - Name int64 - Srv int64 - Addr int64 -} - -// ResolverResponses represents resolver responses -type ResolverResponses struct { - Noerror int64 - Formerr int64 - Servfail int64 - Nxdomain int64 - Notimp int64 - Refused int64 - Timedout int64 - Unknown int64 -} - -// Processes represents processes related stats -type Processes struct { - Respawned int64 -} - -// NewNginxClient creates an NginxClient. -func NewNginxClient(httpClient *http.Client, apiEndpoint string) (*NginxClient, error) { - versions, err := getAPIVersions(httpClient, apiEndpoint) - if err != nil { - return nil, fmt.Errorf("error accessing the API: %v", err) - } - - found := false - for _, v := range *versions { - if v == APIVersion { - found = true - break - } - } - - if !found { - return nil, fmt.Errorf("API version %v of the client is not supported by API versions of NGINX Plus: %v", APIVersion, *versions) - } - - return &NginxClient{ - apiEndpoint: apiEndpoint, - httpClient: httpClient, - }, nil -} - -func getAPIVersions(httpClient *http.Client, endpoint string) (*versions, error) { - resp, err := httpClient.Get(endpoint) - if err != nil { - return nil, fmt.Errorf("%v is not accessible: %v", endpoint, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%v is not accessible: expected %v response, got %v", endpoint, http.StatusOK, resp.StatusCode) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("error while reading body of the response: %v", err) - } - - var vers versions - err = json.Unmarshal(body, &vers) - if err != nil { - return nil, fmt.Errorf("error unmarshalling versions, got %q response: %v", string(body), err) - } - - return &vers, nil -} - -func createResponseMismatchError(respBody io.ReadCloser) *internalError { - apiErrResp, err := readAPIErrorResponse(respBody) - if err != nil { - return &internalError{ - err: fmt.Sprintf("failed to read the response body: %v", err), - } - } - - return &internalError{ - err: apiErrResp.toString(), - apiError: apiErrResp.Error, - } -} - -func readAPIErrorResponse(respBody io.ReadCloser) (*apiErrorResponse, error) { - body, err := ioutil.ReadAll(respBody) - if err != nil { - return nil, fmt.Errorf("failed to read the response body: %v", err) - } - - var apiErr apiErrorResponse - err = json.Unmarshal(body, &apiErr) - if err != nil { - return nil, fmt.Errorf("error unmarshalling apiErrorResponse: got %q response: %v", string(body), err) - } - - return &apiErr, nil -} - -// CheckIfUpstreamExists checks if the upstream exists in NGINX. If the upstream doesn't exist, it returns the error. -func (client *NginxClient) CheckIfUpstreamExists(upstream string) error { - _, err := client.GetHTTPServers(upstream) - return err -} - -// GetHTTPServers returns the servers of the upstream from NGINX. -func (client *NginxClient) GetHTTPServers(upstream string) ([]UpstreamServer, error) { - path := fmt.Sprintf("http/upstreams/%v/servers", upstream) - - var servers []UpstreamServer - err := client.get(path, &servers) - if err != nil { - return nil, fmt.Errorf("failed to get the HTTP servers of upstream %v: %v", upstream, err) - } - - return servers, nil -} - -// AddHTTPServer adds the server to the upstream. -func (client *NginxClient) AddHTTPServer(upstream string, server UpstreamServer) error { - id, err := client.getIDOfHTTPServer(upstream, server.Server) - if err != nil { - return fmt.Errorf("failed to add %v server to %v upstream: %v", server.Server, upstream, err) - } - if id != -1 { - return fmt.Errorf("failed to add %v server to %v upstream: server already exists", server.Server, upstream) - } - - path := fmt.Sprintf("http/upstreams/%v/servers/", upstream) - err = client.post(path, &server) - if err != nil { - return fmt.Errorf("failed to add %v server to %v upstream: %v", server.Server, upstream, err) - } - - return nil -} - -// DeleteHTTPServer the server from the upstream. -func (client *NginxClient) DeleteHTTPServer(upstream string, server string) error { - id, err := client.getIDOfHTTPServer(upstream, server) - if err != nil { - return fmt.Errorf("failed to remove %v server from %v upstream: %v", server, upstream, err) - } - if id == -1 { - return fmt.Errorf("failed to remove %v server from %v upstream: server doesn't exist", server, upstream) - } - - path := fmt.Sprintf("http/upstreams/%v/servers/%v", upstream, id) - err = client.delete(path, http.StatusOK) - if err != nil { - return fmt.Errorf("failed to remove %v server from %v upstream: %v", server, upstream, err) - } - - return nil -} - -// UpdateHTTPServers updates the servers of the upstream. -// Servers that are in the slice, but don't exist in NGINX will be added to NGINX. -// Servers that aren't in the slice, but exist in NGINX, will be removed from NGINX. -// Servers that are in the slice and exist in NGINX, but have different parameters, will be updated. -func (client *NginxClient) UpdateHTTPServers(upstream string, servers []UpstreamServer) (added []UpstreamServer, deleted []UpstreamServer, updated []UpstreamServer, err error) { - serversInNginx, err := client.GetHTTPServers(upstream) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to update servers of %v upstream: %v", upstream, err) - } - - // We assume port 80 if no port is set for servers. - var formattedServers []UpstreamServer - for _, server := range servers { - server.Server = addPortToServer(server.Server) - formattedServers = append(formattedServers, server) - } - - toAdd, toDelete, toUpdate := determineUpdates(formattedServers, serversInNginx) - - for _, server := range toAdd { - err := client.AddHTTPServer(upstream, server) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to update servers of %v upstream: %v", upstream, err) - } - } - - for _, server := range toDelete { - err := client.DeleteHTTPServer(upstream, server.Server) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to update servers of %v upstream: %v", upstream, err) - } - } - - for _, server := range toUpdate { - err := client.UpdateHTTPServer(upstream, server) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to update servers of %v upstream: %v", upstream, err) - } - } - - return toAdd, toDelete, toUpdate, nil -} - -// haveSameParameters checks if a given server has the same parameters as a server already present in NGINX. Order matters -func haveSameParameters(newServer UpstreamServer, serverNGX UpstreamServer) bool { - newServer.ID = serverNGX.ID - - if serverNGX.MaxConns != nil && newServer.MaxConns == nil { - newServer.MaxConns = &defaultMaxConns - } - - if serverNGX.MaxFails != nil && newServer.MaxFails == nil { - newServer.MaxFails = &defaultMaxFails - } - - if serverNGX.FailTimeout != "" && newServer.FailTimeout == "" { - newServer.FailTimeout = defaultFailTimeout - } - - if serverNGX.SlowStart != "" && newServer.SlowStart == "" { - newServer.SlowStart = defaultSlowStart - } - - if serverNGX.Backup != nil && newServer.Backup == nil { - newServer.Backup = &defaultBackup - } - - if serverNGX.Down != nil && newServer.Down == nil { - newServer.Down = &defaultDown - } - - if serverNGX.Weight != nil && newServer.Weight == nil { - newServer.Weight = &defaultWeight - } - - return reflect.DeepEqual(newServer, serverNGX) -} - -func determineUpdates(updatedServers []UpstreamServer, nginxServers []UpstreamServer) (toAdd []UpstreamServer, toRemove []UpstreamServer, toUpdate []UpstreamServer) { - for _, server := range updatedServers { - updateFound := false - for _, serverNGX := range nginxServers { - if server.Server == serverNGX.Server && !haveSameParameters(server, serverNGX) { - server.ID = serverNGX.ID - updateFound = true - break - } - } - if updateFound { - toUpdate = append(toUpdate, server) - } - } - - for _, server := range updatedServers { - found := false - for _, serverNGX := range nginxServers { - if server.Server == serverNGX.Server { - found = true - break - } - } - if !found { - toAdd = append(toAdd, server) - } - } - - for _, serverNGX := range nginxServers { - found := false - for _, server := range updatedServers { - if serverNGX.Server == server.Server { - found = true - break - } - } - if !found { - toRemove = append(toRemove, serverNGX) - } - } - - return -} - -func (client *NginxClient) getIDOfHTTPServer(upstream string, name string) (int, error) { - servers, err := client.GetHTTPServers(upstream) - if err != nil { - return -1, fmt.Errorf("error getting id of server %v of upstream %v: %v", name, upstream, err) - } - - for _, s := range servers { - if s.Server == name { - return s.ID, nil - } - } - - return -1, nil -} - -func (client *NginxClient) get(path string, data interface{}) error { - url := fmt.Sprintf("%v/%v/%v", client.apiEndpoint, APIVersion, path) - resp, err := client.httpClient.Get(url) - if err != nil { - return fmt.Errorf("failed to get %v: %v", path, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return createResponseMismatchError(resp.Body).Wrap(fmt.Sprintf( - "expected %v response, got %v", - http.StatusOK, resp.StatusCode)) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("failed to read the response body: %v", err) - } - - err = json.Unmarshal(body, data) - if err != nil { - return fmt.Errorf("error unmarshaling response %q: %v", string(body), err) - } - return nil -} - -func (client *NginxClient) post(path string, input interface{}) error { - url := fmt.Sprintf("%v/%v/%v", client.apiEndpoint, APIVersion, path) - - jsonInput, err := json.Marshal(input) - if err != nil { - return fmt.Errorf("failed to marshall input: %v", err) - } - - resp, err := client.httpClient.Post(url, "application/json", bytes.NewBuffer(jsonInput)) - if err != nil { - return fmt.Errorf("failed to post %v: %v", path, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusCreated { - return createResponseMismatchError(resp.Body).Wrap(fmt.Sprintf( - "expected %v response, got %v", - http.StatusCreated, resp.StatusCode)) - } - - return nil -} - -func (client *NginxClient) delete(path string, expectedStatusCode int) error { - path = fmt.Sprintf("%v/%v/%v/", client.apiEndpoint, APIVersion, path) - - req, err := http.NewRequest(http.MethodDelete, path, nil) - if err != nil { - return fmt.Errorf("failed to create a delete request: %v", err) - } - - resp, err := client.httpClient.Do(req) - if err != nil { - return fmt.Errorf("failed to create delete request: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != expectedStatusCode { - return createResponseMismatchError(resp.Body).Wrap(fmt.Sprintf( - "failed to complete delete request: expected %v response, got %v", - expectedStatusCode, resp.StatusCode)) - } - return nil -} - -func (client *NginxClient) patch(path string, input interface{}, expectedStatusCode int) error { - path = fmt.Sprintf("%v/%v/%v/", client.apiEndpoint, APIVersion, path) - - jsonInput, err := json.Marshal(input) - if err != nil { - return fmt.Errorf("failed to marshall input: %v", err) - } - - req, err := http.NewRequest(http.MethodPatch, path, bytes.NewBuffer(jsonInput)) - if err != nil { - return fmt.Errorf("failed to create a patch request: %v", err) - } - - resp, err := client.httpClient.Do(req) - if err != nil { - return fmt.Errorf("failed to create patch request: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != expectedStatusCode { - return createResponseMismatchError(resp.Body).Wrap(fmt.Sprintf( - "failed to complete patch request: expected %v response, got %v", - expectedStatusCode, resp.StatusCode)) - } - return nil -} - -// CheckIfStreamUpstreamExists checks if the stream upstream exists in NGINX. If the upstream doesn't exist, it returns the error. -func (client *NginxClient) CheckIfStreamUpstreamExists(upstream string) error { - _, err := client.GetStreamServers(upstream) - return err -} - -// GetStreamServers returns the stream servers of the upstream from NGINX. -func (client *NginxClient) GetStreamServers(upstream string) ([]StreamUpstreamServer, error) { - path := fmt.Sprintf("stream/upstreams/%v/servers", upstream) - - var servers []StreamUpstreamServer - err := client.get(path, &servers) - if err != nil { - return nil, fmt.Errorf("failed to get stream servers of upstream server %v: %v", upstream, err) - } - return servers, nil -} - -// AddStreamServer adds the stream server to the upstream. -func (client *NginxClient) AddStreamServer(upstream string, server StreamUpstreamServer) error { - id, err := client.getIDOfStreamServer(upstream, server.Server) - if err != nil { - return fmt.Errorf("failed to add %v stream server to %v upstream: %v", server.Server, upstream, err) - } - if id != -1 { - return fmt.Errorf("failed to add %v stream server to %v upstream: server already exists", server.Server, upstream) - } - - path := fmt.Sprintf("stream/upstreams/%v/servers/", upstream) - err = client.post(path, &server) - if err != nil { - return fmt.Errorf("failed to add %v stream server to %v upstream: %v", server.Server, upstream, err) - } - return nil -} - -// DeleteStreamServer the server from the upstream. -func (client *NginxClient) DeleteStreamServer(upstream string, server string) error { - id, err := client.getIDOfStreamServer(upstream, server) - if err != nil { - return fmt.Errorf("failed to remove %v stream server from %v upstream: %v", server, upstream, err) - } - if id == -1 { - return fmt.Errorf("failed to remove %v stream server from %v upstream: server doesn't exist", server, upstream) - } - - path := fmt.Sprintf("stream/upstreams/%v/servers/%v", upstream, id) - err = client.delete(path, http.StatusOK) - if err != nil { - return fmt.Errorf("failed to remove %v stream server from %v upstream: %v", server, upstream, err) - } - return nil -} - -// UpdateStreamServers updates the servers of the upstream. -// Servers that are in the slice, but don't exist in NGINX will be added to NGINX. -// Servers that aren't in the slice, but exist in NGINX, will be removed from NGINX. -// Servers that are in the slice and exist in NGINX, but have different parameters, will be updated. -func (client *NginxClient) UpdateStreamServers(upstream string, servers []StreamUpstreamServer) (added []StreamUpstreamServer, deleted []StreamUpstreamServer, updated []StreamUpstreamServer, err error) { - serversInNginx, err := client.GetStreamServers(upstream) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to update stream servers of %v upstream: %v", upstream, err) - } - - var formattedServers []StreamUpstreamServer - for _, server := range servers { - server.Server = addPortToServer(server.Server) - formattedServers = append(formattedServers, server) - } - - toAdd, toDelete, toUpdate := determineStreamUpdates(formattedServers, serversInNginx) - - for _, server := range toAdd { - err := client.AddStreamServer(upstream, server) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to update stream servers of %v upstream: %v", upstream, err) - } - } - - for _, server := range toDelete { - err := client.DeleteStreamServer(upstream, server.Server) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to update stream servers of %v upstream: %v", upstream, err) - } - } - - for _, server := range toUpdate { - err := client.UpdateStreamServer(upstream, server) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to update stream servers of %v upstream: %v", upstream, err) - } - } - - return toAdd, toDelete, toUpdate, nil -} - -func (client *NginxClient) getIDOfStreamServer(upstream string, name string) (int, error) { - servers, err := client.GetStreamServers(upstream) - if err != nil { - return -1, fmt.Errorf("error getting id of stream server %v of upstream %v: %v", name, upstream, err) - } - - for _, s := range servers { - if s.Server == name { - return s.ID, nil - } - } - - return -1, nil -} - -// haveSameParametersForStream checks if a given server has the same parameters as a server already present in NGINX. Order matters -func haveSameParametersForStream(newServer StreamUpstreamServer, serverNGX StreamUpstreamServer) bool { - newServer.ID = serverNGX.ID - if serverNGX.MaxConns != nil && newServer.MaxConns == nil { - newServer.MaxConns = &defaultMaxConns - } - - if serverNGX.MaxFails != nil && newServer.MaxFails == nil { - newServer.MaxFails = &defaultMaxFails - } - - if serverNGX.FailTimeout != "" && newServer.FailTimeout == "" { - newServer.FailTimeout = defaultFailTimeout - } - - if serverNGX.SlowStart != "" && newServer.SlowStart == "" { - newServer.SlowStart = defaultSlowStart - } - - if serverNGX.Backup != nil && newServer.Backup == nil { - newServer.Backup = &defaultBackup - } - - if serverNGX.Down != nil && newServer.Down == nil { - newServer.Down = &defaultDown - } - - if serverNGX.Weight != nil && newServer.Weight == nil { - newServer.Weight = &defaultWeight - } - - return reflect.DeepEqual(newServer, serverNGX) -} - -func determineStreamUpdates(updatedServers []StreamUpstreamServer, nginxServers []StreamUpstreamServer) (toAdd []StreamUpstreamServer, toRemove []StreamUpstreamServer, toUpdate []StreamUpstreamServer) { - for _, server := range updatedServers { - updateFound := false - for _, serverNGX := range nginxServers { - if server.Server == serverNGX.Server && !haveSameParametersForStream(server, serverNGX) { - server.ID = serverNGX.ID - updateFound = true - break - } - } - if updateFound { - toUpdate = append(toUpdate, server) - } - } - - for _, server := range updatedServers { - found := false - for _, serverNGX := range nginxServers { - if server.Server == serverNGX.Server { - found = true - break - } - } - if !found { - toAdd = append(toAdd, server) - } - } - - for _, serverNGX := range nginxServers { - found := false - for _, server := range updatedServers { - if serverNGX.Server == server.Server { - found = true - break - } - } - if !found { - toRemove = append(toRemove, serverNGX) - } - } - - return -} - -// GetStats gets process, slab, connection, request, ssl, zone, stream zone, upstream and stream upstream related stats from the NGINX Plus API. -func (client *NginxClient) GetStats() (*Stats, error) { - info, err := client.GetNginxInfo() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - processes, err := client.GetProcesses() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - slabs, err := client.GetSlabs() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - cons, err := client.GetConnections() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - requests, err := client.GetHTTPRequests() - if err != nil { - return nil, fmt.Errorf("Failed to get stats: %v", err) - } - - ssl, err := client.GetSSL() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - zones, err := client.GetServerZones() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - upstreams, err := client.GetUpstreams() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - streamZones, err := client.GetStreamServerZones() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - streamUpstreams, err := client.GetStreamUpstreams() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - streamZoneSync, err := client.GetStreamZoneSync() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - locationZones, err := client.GetLocationZones() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - resolvers, err := client.GetResolvers() - if err != nil { - return nil, fmt.Errorf("failed to get stats: %v", err) - } - - return &Stats{ - NginxInfo: *info, - Processes: *processes, - Slabs: *slabs, - Connections: *cons, - HTTPRequests: *requests, - SSL: *ssl, - ServerZones: *zones, - StreamServerZones: *streamZones, - Upstreams: *upstreams, - StreamUpstreams: *streamUpstreams, - StreamZoneSync: streamZoneSync, - LocationZones: *locationZones, - Resolvers: *resolvers, - }, nil -} - -// GetNginxInfo returns Nginx stats. -func (client *NginxClient) GetNginxInfo() (*NginxInfo, error) { - var info NginxInfo - err := client.get("nginx", &info) - if err != nil { - return nil, fmt.Errorf("failed to get info: %v", err) - } - return &info, nil -} - -// GetSlabs returns Slabs stats. -func (client *NginxClient) GetSlabs() (*Slabs, error) { - var slabs Slabs - err := client.get("slabs", &slabs) - if err != nil { - return nil, fmt.Errorf("failed to get slabs: %v", err) - } - return &slabs, nil -} - -// GetConnections returns Connections stats. -func (client *NginxClient) GetConnections() (*Connections, error) { - var cons Connections - err := client.get("connections", &cons) - if err != nil { - return nil, fmt.Errorf("failed to get connections: %v", err) - } - return &cons, nil -} - -// GetHTTPRequests returns http/requests stats. -func (client *NginxClient) GetHTTPRequests() (*HTTPRequests, error) { - var requests HTTPRequests - err := client.get("http/requests", &requests) - if err != nil { - return nil, fmt.Errorf("failed to get http requests: %v", err) - } - return &requests, nil -} - -// GetSSL returns SSL stats. -func (client *NginxClient) GetSSL() (*SSL, error) { - var ssl SSL - err := client.get("ssl", &ssl) - if err != nil { - return nil, fmt.Errorf("failed to get ssl: %v", err) - } - return &ssl, nil -} - -// GetServerZones returns http/server_zones stats. -func (client *NginxClient) GetServerZones() (*ServerZones, error) { - var zones ServerZones - err := client.get("http/server_zones", &zones) - if err != nil { - return nil, fmt.Errorf("failed to get server zones: %v", err) - } - return &zones, err -} - -// GetStreamServerZones returns stream/server_zones stats. -func (client *NginxClient) GetStreamServerZones() (*StreamServerZones, error) { - var zones StreamServerZones - err := client.get("stream/server_zones", &zones) - if err != nil { - if err, ok := err.(*internalError); ok { - if err.Code == pathNotFoundCode { - return &zones, nil - } - } - return nil, fmt.Errorf("failed to get stream server zones: %v", err) - } - return &zones, err -} - -// GetUpstreams returns http/upstreams stats. -func (client *NginxClient) GetUpstreams() (*Upstreams, error) { - var upstreams Upstreams - err := client.get("http/upstreams", &upstreams) - if err != nil { - return nil, fmt.Errorf("failed to get upstreams: %v", err) - } - return &upstreams, nil -} - -// GetStreamUpstreams returns stream/upstreams stats. -func (client *NginxClient) GetStreamUpstreams() (*StreamUpstreams, error) { - var upstreams StreamUpstreams - err := client.get("stream/upstreams", &upstreams) - if err != nil { - if err, ok := err.(*internalError); ok { - if err.Code == pathNotFoundCode { - return &upstreams, nil - } - } - return nil, fmt.Errorf("failed to get stream upstreams: %v", err) - } - return &upstreams, nil -} - -// GetStreamZoneSync returns stream/zone_sync stats. -func (client *NginxClient) GetStreamZoneSync() (*StreamZoneSync, error) { - var streamZoneSync StreamZoneSync - err := client.get("stream/zone_sync", &streamZoneSync) - if err != nil { - if err, ok := err.(*internalError); ok { - if err.Code == pathNotFoundCode { - return nil, nil - } - } - return nil, fmt.Errorf("failed to get stream zone sync: %v", err) - } - - return &streamZoneSync, err -} - -// GetLocationZones returns http/location_zones stats. -func (client *NginxClient) GetLocationZones() (*LocationZones, error) { - var locationZones LocationZones - err := client.get("http/location_zones", &locationZones) - if err != nil { - return nil, fmt.Errorf("failed to get location zones: %v", err) - } - - return &locationZones, err -} - -// GetResolvers returns Resolvers stats. -func (client *NginxClient) GetResolvers() (*Resolvers, error) { - var resolvers Resolvers - err := client.get("resolvers", &resolvers) - if err != nil { - return nil, fmt.Errorf("failed to get resolvers: %v", err) - } - - return &resolvers, err -} - -// GetProcesses returns Processes stats. -func (client *NginxClient) GetProcesses() (*Processes, error) { - var processes Processes - err := client.get("processes", &processes) - if err != nil { - return nil, fmt.Errorf("failed to get processes: %v", err) - } - - return &processes, err -} - -// KeyValPairs are the key-value pairs stored in a zone. -type KeyValPairs map[string]string - -// KeyValPairsByZone are the KeyValPairs for all zones, by zone name. -type KeyValPairsByZone map[string]KeyValPairs - -// GetKeyValPairs fetches key/value pairs for a given HTTP zone. -func (client *NginxClient) GetKeyValPairs(zone string) (KeyValPairs, error) { - return client.getKeyValPairs(zone, httpContext) -} - -// GetStreamKeyValPairs fetches key/value pairs for a given Stream zone. -func (client *NginxClient) GetStreamKeyValPairs(zone string) (KeyValPairs, error) { - return client.getKeyValPairs(zone, streamContext) -} - -func (client *NginxClient) getKeyValPairs(zone string, stream bool) (KeyValPairs, error) { - base := "http" - if stream { - base = "stream" - } - if zone == "" { - return nil, fmt.Errorf("zone required") - } - - path := fmt.Sprintf("%v/keyvals/%v", base, zone) - var keyValPairs KeyValPairs - err := client.get(path, &keyValPairs) - if err != nil { - return nil, fmt.Errorf("failed to get keyvals for %v/%v zone: %v", base, zone, err) - } - return keyValPairs, nil -} - -// GetAllKeyValPairs fetches all key/value pairs for all HTTP zones. -func (client *NginxClient) GetAllKeyValPairs() (KeyValPairsByZone, error) { - return client.getAllKeyValPairs(httpContext) -} - -// GetAllStreamKeyValPairs fetches all key/value pairs for all Stream zones. -func (client *NginxClient) GetAllStreamKeyValPairs() (KeyValPairsByZone, error) { - return client.getAllKeyValPairs(streamContext) -} - -func (client *NginxClient) getAllKeyValPairs(stream bool) (KeyValPairsByZone, error) { - base := "http" - if stream { - base = "stream" - } - - path := fmt.Sprintf("%v/keyvals", base) - var keyValPairsByZone KeyValPairsByZone - err := client.get(path, &keyValPairsByZone) - if err != nil { - return nil, fmt.Errorf("failed to get keyvals for all %v zones: %v", base, err) - } - return keyValPairsByZone, nil -} - -// AddKeyValPair adds a new key/value pair to a given HTTP zone. -func (client *NginxClient) AddKeyValPair(zone string, key string, val string) error { - return client.addKeyValPair(zone, key, val, httpContext) -} - -// AddStreamKeyValPair adds a new key/value pair to a given Stream zone. -func (client *NginxClient) AddStreamKeyValPair(zone string, key string, val string) error { - return client.addKeyValPair(zone, key, val, streamContext) -} - -func (client *NginxClient) addKeyValPair(zone string, key string, val string, stream bool) error { - base := "http" - if stream { - base = "stream" - } - if zone == "" { - return fmt.Errorf("zone required") - } - - path := fmt.Sprintf("%v/keyvals/%v", base, zone) - input := KeyValPairs{key: val} - err := client.post(path, &input) - if err != nil { - return fmt.Errorf("failed to add key value pair for %v/%v zone: %v", base, zone, err) - } - return nil -} - -// ModifyKeyValPair modifies the value of an existing key in a given HTTP zone. -func (client *NginxClient) ModifyKeyValPair(zone string, key string, val string) error { - return client.modifyKeyValPair(zone, key, val, httpContext) -} - -// ModifyStreamKeyValPair modifies the value of an existing key in a given Stream zone. -func (client *NginxClient) ModifyStreamKeyValPair(zone string, key string, val string) error { - return client.modifyKeyValPair(zone, key, val, streamContext) -} - -func (client *NginxClient) modifyKeyValPair(zone string, key string, val string, stream bool) error { - base := "http" - if stream { - base = "stream" - } - if zone == "" { - return fmt.Errorf("zone required") - } - - path := fmt.Sprintf("%v/keyvals/%v", base, zone) - input := KeyValPairs{key: val} - err := client.patch(path, &input, http.StatusNoContent) - if err != nil { - return fmt.Errorf("failed to update key value pair for %v/%v zone: %v", base, zone, err) - } - return nil -} - -// DeleteKeyValuePair deletes the key/value pair for a key in a given HTTP zone. -func (client *NginxClient) DeleteKeyValuePair(zone string, key string) error { - return client.deleteKeyValuePair(zone, key, httpContext) -} - -// DeleteStreamKeyValuePair deletes the key/value pair for a key in a given Stream zone. -func (client *NginxClient) DeleteStreamKeyValuePair(zone string, key string) error { - return client.deleteKeyValuePair(zone, key, streamContext) -} - -// To delete a key/value pair you set the value to null via the API, -// then NGINX+ will delete the key. -func (client *NginxClient) deleteKeyValuePair(zone string, key string, stream bool) error { - base := "http" - if stream { - base = "stream" - } - if zone == "" { - return fmt.Errorf("zone required") - } - - // map[string]string can't have a nil value so we use a different type here. - keyval := make(map[string]interface{}) - keyval[key] = nil - - path := fmt.Sprintf("%v/keyvals/%v", base, zone) - err := client.patch(path, &keyval, http.StatusNoContent) - if err != nil { - return fmt.Errorf("failed to remove key values pair for %v/%v zone: %v", base, zone, err) - } - return nil -} - -// DeleteKeyValPairs deletes all the key-value pairs in a given HTTP zone. -func (client *NginxClient) DeleteKeyValPairs(zone string) error { - return client.deleteKeyValPairs(zone, httpContext) -} - -// DeleteStreamKeyValPairs deletes all the key-value pairs in a given Stream zone. -func (client *NginxClient) DeleteStreamKeyValPairs(zone string) error { - return client.deleteKeyValPairs(zone, streamContext) -} - -func (client *NginxClient) deleteKeyValPairs(zone string, stream bool) error { - base := "http" - if stream { - base = "stream" - } - if zone == "" { - return fmt.Errorf("zone required") - } - - path := fmt.Sprintf("%v/keyvals/%v", base, zone) - err := client.delete(path, http.StatusNoContent) - if err != nil { - return fmt.Errorf("failed to remove all key value pairs for %v/%v zone: %v", base, zone, err) - } - return nil -} - -// UpdateHTTPServer updates the server of the upstream. -func (client *NginxClient) UpdateHTTPServer(upstream string, server UpstreamServer) error { - path := fmt.Sprintf("http/upstreams/%v/servers/%v", upstream, server.ID) - server.ID = 0 - err := client.patch(path, &server, http.StatusOK) - if err != nil { - return fmt.Errorf("failed to update %v server to %v upstream: %v", server.Server, upstream, err) - } - - return nil -} - -// UpdateStreamServer updates the stream server of the upstream. -func (client *NginxClient) UpdateStreamServer(upstream string, server StreamUpstreamServer) error { - path := fmt.Sprintf("stream/upstreams/%v/servers/%v", upstream, server.ID) - server.ID = 0 - err := client.patch(path, &server, http.StatusOK) - if err != nil { - return fmt.Errorf("failed to update %v stream server to %v upstream: %v", server.Server, upstream, err) - } - - return nil -} - -func addPortToServer(server string) string { - if len(strings.Split(server, ":")) == 2 { - return server - } - - if len(strings.Split(server, "]:")) == 2 { - return server - } - - if strings.HasPrefix(server, "unix:") { - return server - } - - return fmt.Sprintf("%v:%v", server, defaultServerPort) -} diff --git a/src/reload.go b/src/reload.go index 529126b..9c0bd6c 100644 --- a/src/reload.go +++ b/src/reload.go @@ -15,6 +15,7 @@ import ( "time" "github.com/sirupsen/logrus" + nplus "github.com/nginxinc/nginx-plus-go-client/client" ) type NamespaceRenderingData struct { @@ -271,8 +272,7 @@ func nginxPlus(data *RenderingData) error { } client := &http.Client{Transport: tr} - c := NginxClient{endpoint, client} - nginxClient, error := NewNginxClient(c.httpClient, c.apiEndpoint) + nginxClient, error := nplus.NewNginxClient(endpoint,nplus.WithHTTPClient(client),nplus.WithAPIVersion(6)) if error != nil { logger.WithFields(logrus.Fields{ "error": error, @@ -280,10 +280,10 @@ func nginxPlus(data *RenderingData) error { return error } upstreamtocheck := app.Vhost - var finalformattedServers []UpstreamServer + var finalformattedServers []nplus.UpstreamServer - for _, server := range newFormattedServers { - formattedServer := UpstreamServer{Server: server, MaxFails: config.MaxFailsUpstream, FailTimeout: config.FailTimeoutUpstream, SlowStart: config.SlowStartUpstream} + for _, server := range nplus.newFormattedServers { + formattedServer := nplus.UpstreamServer{Server: server, MaxFails: config.MaxFailsUpstream, FailTimeout: config.FailTimeoutUpstream, SlowStart: config.SlowStartUpstream} finalformattedServers = append(finalformattedServers, formattedServer) } From 61782abeab88d0edcfd5168496de03f521c20e7a Mon Sep 17 00:00:00 2001 From: Vishnu Naini Date: Thu, 26 Dec 2024 23:55:40 +0530 Subject: [PATCH 19/29] merge fix --- src/reload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/reload.go b/src/reload.go index d8db527..ca33fa5 100644 --- a/src/reload.go +++ b/src/reload.go @@ -290,7 +290,7 @@ func nginxPlus(data *RenderingData) error { upstreamtocheck := app.Vhost var finalformattedServers []nplus.UpstreamServer - for _, server := range nplus.newFormattedServers { + for _, server := range newFormattedServers { formattedServer := nplus.UpstreamServer{Server: server, MaxFails: config.MaxFailsUpstream, FailTimeout: config.FailTimeoutUpstream, SlowStart: config.SlowStartUpstream} finalformattedServers = append(finalformattedServers, formattedServer) } From 427773e199026757cff509b04e5866ce41a1d6ae Mon Sep 17 00:00:00 2001 From: Vishnu Naini Date: Fri, 27 Dec 2024 13:45:11 +0530 Subject: [PATCH 20/29] fix merge comment --- src/reload.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/reload.go b/src/reload.go index ca33fa5..707a6a0 100644 --- a/src/reload.go +++ b/src/reload.go @@ -15,8 +15,8 @@ import ( "text/template" "time" - "github.com/sirupsen/logrus" nplus "github.com/nginxinc/nginx-plus-go-client/client" + "github.com/sirupsen/logrus" ) type NamespaceRenderingData struct { @@ -294,6 +294,7 @@ func nginxPlus(data *RenderingData) error { formattedServer := nplus.UpstreamServer{Server: server, MaxFails: config.MaxFailsUpstream, FailTimeout: config.FailTimeoutUpstream, SlowStart: config.SlowStartUpstream} finalformattedServers = append(finalformattedServers, formattedServer) } + // If upstream has no servers, UpdateHTTPServers returns error as in-line GetHTTPServers returns error. server ID 0 needs to be explicitly initiated by a PATCH err := nginxClient.CheckIfUpstreamExists(upstreamtocheck) if err != nil { // First add atleast one server to initialise upstream to support UpdateHTTPServers @@ -440,7 +441,7 @@ func reloadWorker() { for { select { case <-ticker.C: - <-reloadSignalQueue + <-upstreamsUpdateSignalQueue reload() } } From afa0080613c7358b0122e57f2e2387a9d3d467d0 Mon Sep 17 00:00:00 2001 From: Vishnu Naini Date: Fri, 27 Dec 2024 13:49:28 +0530 Subject: [PATCH 21/29] fix merge comment --- src/reload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/reload.go b/src/reload.go index 707a6a0..91346c5 100644 --- a/src/reload.go +++ b/src/reload.go @@ -441,7 +441,7 @@ func reloadWorker() { for { select { case <-ticker.C: - <-upstreamsUpdateSignalQueue + <-reloadSignalQueue reload() } } From a65fc966d36c88131a34574003f0b6761a9cf6b1 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Fri, 27 Dec 2024 16:29:42 +0530 Subject: [PATCH 22/29] added fix for nixy plus --- src/reload.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/reload.go b/src/reload.go index 91346c5..5689bc2 100644 --- a/src/reload.go +++ b/src/reload.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "strings" "text/template" "time" @@ -57,13 +58,17 @@ func reload() error { if config.NginxReloadDisabled { logger.Warn("Template reload has been disabled") } else { - logger.Info("Need to reload config") - err = updateAndReloadConfig(&data) - if err != nil { - logger.WithFields(logrus.Fields{ - "error": err.Error(), - }).Error("unable to update and reload nginx config. NPlus api calls will be skipped.") - return err + vhosts := db.ReadAllKnownVhosts() + lastKnownVhosts := db.ReadLastKnownVhosts() + if !reflect.DeepEqual(vhosts, lastKnownVhosts) { + logger.Info("Need to reload config") + err = updateAndReloadConfig(&data) + if err != nil { + logger.WithFields(logrus.Fields{ + "error": err.Error(), + }).Error("unable to update and reload nginx config. NPlus api calls will be skipped.") + return err + } } else { logger.Debug("No changes detected in vhosts. No config update is necessary. Upstream updates will happen via nplus apis") } From 1715cfa008fd4c84f054acf0c5b24906531ddec3 Mon Sep 17 00:00:00 2001 From: Tushar Mandar Date: Fri, 27 Dec 2024 17:07:29 +0530 Subject: [PATCH 23/29] bug fix --- src/drove.go | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/drove.go b/src/drove.go index 8e4243c..7c3975c 100644 --- a/src/drove.go +++ b/src/drove.go @@ -562,25 +562,15 @@ func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) return false } equal := syncAppsAndVhosts(droveConfig, &jsonapps, &vhosts) - lastConfigUpdated := true - namespaceLastUpdated, err := db.ReadLastTimestamp(namespace) - if err == nil { - if db.ReadLastReloadTimestamp().Before(namespaceLastUpdated) { - logger.Info("Last reload still not applied") - lastConfigUpdated = false - } - } - //Ideally only lastConfigUpdated can dictate if reload is required - if equal && !leaderShifted && lastConfigUpdated { + if equal && !leaderShifted { logger.Debug("no config changes") return false } logger.WithFields(logrus.Fields{ - "namespace": namespace, - "lastConfigUpdated": lastConfigUpdated, - "leaderShifted": leaderShifted, - "appsChanged": !equal, + "namespace": namespace, + "leaderShifted": leaderShifted, + "appsChanged": !equal, }).Info("Config reload required") //logging exact reason of reload elapsed := time.Since(start) From cb54967277d596fc339639c602a4e514751ab1c3 Mon Sep 17 00:00:00 2001 From: Vishnu Naini Date: Fri, 27 Dec 2024 17:18:47 +0530 Subject: [PATCH 24/29] reduce log verbosity level --- src/datamanager.go | 26 +++++++++++++------------- src/drove.go | 2 +- src/nixy.go | 4 ++-- src/reload.go | 10 +++++----- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/datamanager.go b/src/datamanager.go index 9ab89f9..d61a3e4 100644 --- a/src/datamanager.go +++ b/src/datamanager.go @@ -146,7 +146,7 @@ func (dm *DataManager) ReadLastTimestamp(namespace string) (time.Time, error) { "operation": operation, "namespace": namespace, "Timestamp": ns.Timestamp, - }).Info("ReadLastTimestamps successfully") + }).Debug("ReadLastTimestamps successfully") return ns.Timestamp, nil //returning copy } @@ -173,7 +173,7 @@ func (dm *DataManager) ReadLeader(namespace string) (LeaderController, error) { "operation": operation, "namespace": namespace, "leader": ns.Leader, - }).Info("ReadLeader successfully") + }).Debug("ReadLeader successfully") return ns.Leader, nil //returning copy } @@ -203,7 +203,7 @@ func (dm *DataManager) UpdateLeader(namespace string, leader LeaderController) e "namespace": namespace, "leader": ns.Leader, "time": ns.Timestamp, - }).Info("UpdateLeader data finished successfully") + }).Debug("UpdateLeader data finished successfully") return nil } @@ -229,7 +229,7 @@ func (dm *DataManager) ReadApps(namespace string) (map[string]App, error) { "operation": operation, "namespace": namespace, "apps": ns.Apps, - }).Info("ReadApp successfully") + }).Debug("ReadApp successfully") return ns.Apps, nil //returning copy } @@ -259,7 +259,7 @@ func (dm *DataManager) UpdateApps(namespace string, apps map[string]App) error { "namespace": namespace, "apps": dm.namespaces[namespace].Apps, "time": dm.namespaces[namespace].Timestamp, - }).Info("UpdateApps finished successfully") + }).Debug("UpdateApps finished successfully") return nil } @@ -285,7 +285,7 @@ func (dm *DataManager) ReadKnownVhosts(namespace string) (Vhosts, error) { "operation": operation, "namespace": namespace, "knownVHosts": ns.KnownVHosts, - }).Info("ReadKnownVhosts successfully") + }).Debug("ReadKnownVhosts successfully") return ns.KnownVHosts, nil //returning copy } @@ -308,7 +308,7 @@ func (dm *DataManager) ReadAllKnownVhosts() Vhosts { logger.WithFields(logrus.Fields{ "operation": operation, "allApps": allKnownVhosts, - }).Info("ReadAllKnownVhosts successfully") + }).Debug("ReadAllKnownVhosts successfully") return allKnownVhosts //returning copy } @@ -338,7 +338,7 @@ func (dm *DataManager) UpdateKnownVhosts(namespace string, KnownVHosts Vhosts) e "namespace": namespace, "knownHosts": dm.namespaces[namespace].KnownVHosts, "time": dm.namespaces[namespace].Timestamp, - }).Info("UpdateKnownVhosts finished successfully") + }).Debug("UpdateKnownVhosts finished successfully") return nil } @@ -351,7 +351,7 @@ func (dm *DataManager) ReadLastReloadTimestamp() time.Time { logger.WithFields(logrus.Fields{ "operation": operation, "LastReloadTimestamp": dm.LastReloadTimestamp, - }).Info("ReadLoadedTime successfully") + }).Debug("ReadLoadedTime successfully") return dm.LastReloadTimestamp //returning copy } @@ -365,7 +365,7 @@ func (dm *DataManager) ReadLastKnownVhosts() Vhosts { logger.WithFields(logrus.Fields{ "operation": operation, "LastKnownVhosts": dm.LastKnownVhosts, - }).Info("LastKnownVhosts successfully") + }).Debug("LastKnownVhosts successfully") return dm.LastKnownVhosts //returning copy } @@ -381,7 +381,7 @@ func (dm *DataManager) UpdateLastKnownVhosts(inLastKnownVhosts Vhosts) error { logger.WithFields(logrus.Fields{ "operation": operation, "LastKnownVhosts": dm.LastKnownVhosts, - }).Info("UpdateLastKnownVhosts finished successfully") + }).Debug("UpdateLastKnownVhosts finished successfully") return nil } @@ -393,7 +393,7 @@ func (dm *DataManager) ReadAllNamespace() map[string]NamespaceData { // Start the operation log logger.WithFields(logrus.Fields{ "operation": operation, - }).Info("ReadAllNamespace data successfully") + }).Debug("ReadAllNamespace data successfully") return dm.namespaces //returning copy } @@ -408,7 +408,7 @@ func (dm *DataManager) ReadStaticData() StaticConfig { logger.WithFields(logrus.Fields{ "operation": operation, "staticData": dm.StaticData, - }).Info("ReadStaticData successfully") + }).Debug("ReadStaticData successfully") return dm.StaticData //returning copy } diff --git a/src/drove.go b/src/drove.go index 8e4243c..88d52a5 100644 --- a/src/drove.go +++ b/src/drove.go @@ -183,7 +183,7 @@ func refreshLeaderData(namespace string) bool { }).Info("New leader being set") return true } else { - logrus.Warn("Leade struct generation failed") + logrus.Error("Leader struct generation failed") } } return false diff --git a/src/nixy.go b/src/nixy.go index 91424dc..d4b92c4 100644 --- a/src/nixy.go +++ b/src/nixy.go @@ -153,7 +153,7 @@ func setloglevel() { case "error": logLevel = logrus.ErrorLevel default: - logger.Error("unknown loglevel") + logger.Error("unknown loglevel. Defaulting to info") logLevel = logrus.InfoLevel } @@ -224,7 +224,7 @@ func validateConfig() error { func nixyReload(w http.ResponseWriter, r *http.Request) { logger.WithFields(logrus.Fields{ "client": r.RemoteAddr, - }).Info("Reload triggered") + }).Info("Reload triggered via /v1/reload") queued := true select { case refreshSignalQueue <- true: // Add referesh to our signal channel, unless it is full of course. diff --git a/src/reload.go b/src/reload.go index 5689bc2..c6e9881 100644 --- a/src/reload.go +++ b/src/reload.go @@ -56,12 +56,12 @@ func reload() error { } else { //Nginx plus is enabled if config.NginxReloadDisabled { - logger.Warn("Template reload has been disabled") + logger.Debug("Template reload has been disabled") } else { vhosts := db.ReadAllKnownVhosts() lastKnownVhosts := db.ReadLastKnownVhosts() if !reflect.DeepEqual(vhosts, lastKnownVhosts) { - logger.Info("Need to reload config") + logger.Info("Vhost changed detected. Need to reload config") err = updateAndReloadConfig(&data) if err != nil { logger.WithFields(logrus.Fields{ @@ -86,7 +86,7 @@ func reload() error { elapsed := time.Since(start) logger.WithFields(logrus.Fields{ "took": elapsed, - }).Info("config reloaded successfully") + }).Debug("reload worker completed") return nil } @@ -116,7 +116,7 @@ func updateAndReloadConfig(data *RenderingData) error { elapsed := time.Since(start) logger.WithFields(logrus.Fields{ "took": elapsed, - }).Info("config updated successfully") + }).Debug("config updated and reloaded successfully") go statsCount("reload.success", 1) go statsTiming("reload.time", elapsed) go countSuccessfulReloads.Inc() @@ -261,7 +261,7 @@ func nginxPlus(data *RenderingData) error { return error } - logger.WithFields(logrus.Fields{"apps": data.Apps}).Info("Updating upstreams for the whitelisted drove tags") + logger.WithFields(logrus.Fields{"apps": data.Apps}).Debug("Updating upstreams for the whitelisted drove vhosts") for _, app := range data.Apps { var newFormattedServers []string for _, t := range app.Hosts { From 4264dcb5c025c0a54ea5df6dcf7bed4bd9cbdc0f Mon Sep 17 00:00:00 2001 From: Vishnu Naini Date: Fri, 27 Dec 2024 17:35:37 +0530 Subject: [PATCH 25/29] rename symbols for clarity on reloads as it is not universal,log verbosity --- src/drove.go | 38 +++++++++++++++++++------------------- src/nixy.go | 6 +++--- src/reload.go | 4 ++-- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/drove.go b/src/drove.go index 88d52a5..0cd2ee3 100644 --- a/src/drove.go +++ b/src/drove.go @@ -202,7 +202,7 @@ func newDroveClient(name string) *DroveClient { } } -func pollingHandler(droveClient *DroveClient, reloadChannel chan<- bool, waitGroup *sync.WaitGroup) { +func pollingHandler(droveClient *DroveClient, appsConfigUpdateChannel chan<- bool, waitGroup *sync.WaitGroup) { defer waitGroup.Done() appsRefreshed := false namespace := droveClient.namespace @@ -228,14 +228,14 @@ func pollingHandler(droveClient *DroveClient, reloadChannel chan<- bool, waitGro "namespace": namespace, "localTime": time.Now(), }).Info("Events received") - reloadNeeded := false + appsConfigUpdateNeeded := false if _, ok := eventSummary.EventsCount["APP_STATE_CHANGE"]; ok { - reloadNeeded = true + appsConfigUpdateNeeded = true } if _, ok := eventSummary.EventsCount["INSTANCE_STATE_CHANGE"]; ok { - reloadNeeded = true + appsConfigUpdateNeeded = true } - if reloadNeeded || leaderShifted { + if appsConfigUpdateNeeded || leaderShifted { appsRefreshed = refreshApps(droveClient.httpClient, namespace, leaderShifted) } else { logger.Debug("Irrelevant events ignored") @@ -247,34 +247,34 @@ func pollingHandler(droveClient *DroveClient, reloadChannel chan<- bool, waitGro }).Debug("New Events received") } } - reloadChannel <- appsRefreshed + appsConfigUpdateChannel <- appsRefreshed } func pollingEvents() { var waitGroup sync.WaitGroup - reloadChannel := make(chan bool, len(droveClients)) + appsUpdateChannel := make(chan bool, len(droveClients)) for _, droveClient := range droveClients { waitGroup.Add(1) - go pollingHandler(droveClient, reloadChannel, &waitGroup) + go pollingHandler(droveClient, appsUpdateChannel, &waitGroup) } waitGroup.Wait() - close(reloadChannel) + close(appsUpdateChannel) - reloadConfig := false - for result := range reloadChannel { + appsUpdated := false + for result := range appsUpdateChannel { if result { - reloadConfig = true + appsUpdated = true break } } logger.WithFields(logrus.Fields{ - "reloadConfig": reloadConfig, + "upstreamsUpdated": appsUpdated, }).Info("Drove poll event result") - if reloadConfig { - reloadSignalQueue <- true + if appsUpdated { + appsUpdateSignalQueue <- true } } @@ -287,7 +287,7 @@ func schedulePollDroveEvents() { case <-ticker.C: logger.Info("Refreshing drove events as per schedule") pollingEvents() - case <-refreshSignalQueue: + case <-eventRefreshSignalQueue: logger.Info("Refreshing drove events due to force referesh") pollingEvents() } @@ -464,7 +464,7 @@ func syncAppsAndVhosts(droveConfig DroveConfig, jsonapps *DroveApps, vhosts *Vho "tag": droveConfig.RoutingTag, "vhost": newapp.Vhost, "value": tagValue, - }).Info("routing tag found") + }).Debug("routing tag found") groupName = tagValue } else { @@ -572,7 +572,7 @@ func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) } //Ideally only lastConfigUpdated can dictate if reload is required if equal && !leaderShifted && lastConfigUpdated { - logger.Debug("no config changes") + logger.Debug("no config or vhost changes") return false } @@ -581,7 +581,7 @@ func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) "lastConfigUpdated": lastConfigUpdated, "leaderShifted": leaderShifted, "appsChanged": !equal, - }).Info("Config reload required") //logging exact reason of reload + }).Info("Config or vhosts update required") //logging exact reason of reload elapsed := time.Since(start) go observeAppRefreshTimeMetric(namespace, elapsed) diff --git a/src/nixy.go b/src/nixy.go index d4b92c4..b165661 100644 --- a/src/nixy.go +++ b/src/nixy.go @@ -171,8 +171,8 @@ func setupDataManager() { } // Reload signal with buffer of two, because we dont really need more. -var reloadSignalQueue = make(chan bool, 2) -var refreshSignalQueue = make(chan bool, 2) +var appsUpdateSignalQueue = make(chan bool, 2) +var eventRefreshSignalQueue = make(chan bool, 2) // Global http transport for connection reuse var tr = &http.Transport{MaxIdleConnsPerHost: 10, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} @@ -227,7 +227,7 @@ func nixyReload(w http.ResponseWriter, r *http.Request) { }).Info("Reload triggered via /v1/reload") queued := true select { - case refreshSignalQueue <- true: // Add referesh to our signal channel, unless it is full of course. + case eventRefreshSignalQueue <- true: // Add referesh to our signal channel, unless it is full of course. default: queued = false } diff --git a/src/reload.go b/src/reload.go index c6e9881..13b290f 100644 --- a/src/reload.go +++ b/src/reload.go @@ -197,7 +197,7 @@ func createRenderingData(data *RenderingData) { data.Apps = allApps logger.WithFields(logrus.Fields{ "data": data, - }).Info("Rendering data generated") + }).Debug("Rendering data generated") return } @@ -446,7 +446,7 @@ func reloadWorker() { for { select { case <-ticker.C: - <-reloadSignalQueue + <-appsUpdateSignalQueue reload() } } From c2cfcabfbe5d6a9f70956984ba4d257a97a0aa50 Mon Sep 17 00:00:00 2001 From: Vishnu Naini Date: Fri, 27 Dec 2024 17:51:21 +0530 Subject: [PATCH 26/29] nginx-plus: reduce log verbosity --- src/reload.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/reload.go b/src/reload.go index 13b290f..d5ab917 100644 --- a/src/reload.go +++ b/src/reload.go @@ -282,15 +282,11 @@ func nginxPlus(data *RenderingData) error { logger.WithFields(logrus.Fields{ "vhost": app.Vhost, - }).Info("app.vhost") + }).Debug("app.vhost") logger.WithFields(logrus.Fields{ "upstreams": newFormattedServers, - }).Info("nginx upstreams") - - logger.WithFields(logrus.Fields{ - "nginx": config.Nginxplusapiaddr, - }).Info("endpoint") + }).Debug("nginx upstreams") upstreamtocheck := app.Vhost var finalformattedServers []nplus.UpstreamServer @@ -335,21 +331,25 @@ func nginxPlus(data *RenderingData) error { if added != nil { logger.WithFields(logrus.Fields{ - "nginx upstreams added": added, + "vhost": upstreamtocheck, + "upstreams added": added, }).Info("nginx upstreams added") } if deleted != nil { logger.WithFields(logrus.Fields{ - "nginx upstreams deleted": deleted, + "vhost": upstreamtocheck, + "upstreams deleted": deleted, }).Info("nginx upstreams deleted") } if updated != nil { logger.WithFields(logrus.Fields{ - "nginx upsteams updated": updated, + "vhost": upstreamtocheck, + "upstreams updated": updated, }).Info("nginx upstreams updated") } if error != nil { logger.WithFields(logrus.Fields{ + "vhost": upstreamtocheck, "error": error, }).Error("unable to update nginx upstreams") return error From 1d9e72497e39613806edde27f2c8b447f216731a Mon Sep 17 00:00:00 2001 From: Vishnu Naini Date: Fri, 27 Dec 2024 18:13:28 +0530 Subject: [PATCH 27/29] further logging changes --- src/drove.go | 8 ++++---- src/reload.go | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/drove.go b/src/drove.go index e1fed13..dd7936f 100644 --- a/src/drove.go +++ b/src/drove.go @@ -270,7 +270,7 @@ func pollingEvents() { } } logger.WithFields(logrus.Fields{ - "upstreamsUpdated": appsUpdated, + "appsUpdated": appsUpdated, }).Info("Drove poll event result") if appsUpdated { @@ -563,7 +563,7 @@ func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) } equal := syncAppsAndVhosts(droveConfig, &jsonapps, &vhosts) if equal && !leaderShifted { - logger.Debug("no config changes") + logger.Debug("no relevant App Data changes") return false } @@ -571,12 +571,12 @@ func refreshApps(httpClient *http.Client, namespace string, leaderShifted bool) "namespace": namespace, "leaderShifted": leaderShifted, "appsChanged": !equal, - }).Info("Config or vhosts update required") //logging exact reason of reload + }).Debug("Apps Data change required because of") //logging exact reason of potential reload elapsed := time.Since(start) go observeAppRefreshTimeMetric(namespace, elapsed) logger.WithFields(logrus.Fields{ "took": elapsed, - }).Info("Apps updated") + }).Debug("Apps update") return true } diff --git a/src/reload.go b/src/reload.go index d5ab917..cf91a5b 100644 --- a/src/reload.go +++ b/src/reload.go @@ -61,7 +61,7 @@ func reload() error { vhosts := db.ReadAllKnownVhosts() lastKnownVhosts := db.ReadLastKnownVhosts() if !reflect.DeepEqual(vhosts, lastKnownVhosts) { - logger.Info("Vhost changed detected. Need to reload config") + logger.Info("Vhost changes detected. Need to reload config") err = updateAndReloadConfig(&data) if err != nil { logger.WithFields(logrus.Fields{ @@ -227,6 +227,9 @@ func writeConf(data *RenderingData) error { logger.Error("Error in config generated") return err } + logger.WithFields(logrus.Fields{ + "file": config.NginxConfig, + }).Info("Writing new config") err = os.Rename(tmpFile.Name(), config.NginxConfig) if err != nil { return err From 62e4317c91a66d6a5e3fe70df318f2abbed8e1eb Mon Sep 17 00:00:00 2001 From: Vishnu Naini Date: Fri, 27 Dec 2024 18:21:25 +0530 Subject: [PATCH 28/29] symbol fixes --- src/drove.go | 10 +++++----- src/nixy.go | 2 +- src/reload.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/drove.go b/src/drove.go index dd7936f..c89bc8f 100644 --- a/src/drove.go +++ b/src/drove.go @@ -252,18 +252,18 @@ func pollingHandler(droveClient *DroveClient, appsConfigUpdateChannel chan<- boo func pollingEvents() { var waitGroup sync.WaitGroup - appsUpdateChannel := make(chan bool, len(droveClients)) + appsConfigUpdateChannel := make(chan bool, len(droveClients)) for _, droveClient := range droveClients { waitGroup.Add(1) - go pollingHandler(droveClient, appsUpdateChannel, &waitGroup) + go pollingHandler(droveClient, appsConfigUpdateChannel, &waitGroup) } waitGroup.Wait() - close(appsUpdateChannel) + close(appsConfigUpdateChannel) appsUpdated := false - for result := range appsUpdateChannel { + for result := range appsConfigUpdateChannel { if result { appsUpdated = true break @@ -274,7 +274,7 @@ func pollingEvents() { }).Info("Drove poll event result") if appsUpdated { - appsUpdateSignalQueue <- true + appsConfigUpdateSignalQueue <- true } } diff --git a/src/nixy.go b/src/nixy.go index b165661..4dcd39c 100644 --- a/src/nixy.go +++ b/src/nixy.go @@ -171,7 +171,7 @@ func setupDataManager() { } // Reload signal with buffer of two, because we dont really need more. -var appsUpdateSignalQueue = make(chan bool, 2) +var appsConfigUpdateSignalQueue = make(chan bool, 2) var eventRefreshSignalQueue = make(chan bool, 2) // Global http transport for connection reuse diff --git a/src/reload.go b/src/reload.go index cf91a5b..d6f06b2 100644 --- a/src/reload.go +++ b/src/reload.go @@ -449,7 +449,7 @@ func reloadWorker() { for { select { case <-ticker.C: - <-appsUpdateSignalQueue + <-appsConfigUpdateSignalQueue reload() } } From 5c48b9776006a703a333e577ac93c2856030c0a5 Mon Sep 17 00:00:00 2001 From: vishnunaini Date: Sun, 5 Jan 2025 23:45:48 +0530 Subject: [PATCH 29/29] /v1/health: fix Template healthcheck --- src/reload.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/reload.go b/src/reload.go index d6f06b2..e7bb649 100644 --- a/src/reload.go +++ b/src/reload.go @@ -371,7 +371,9 @@ func checkTmpl() error { if err != nil { return err } - err = t.Execute(ioutil.Discard, &config) + data := RenderingData{} + createRenderingData(&data) + err = t.Execute(ioutil.Discard, &data) if err != nil { return err }