Merge branch 'master' into TT-13890-request-debug-endpoint

apidef/api_definitions.go

@@ -751,7 +751,6 @@ type APIDefinition struct {
 	IPAccessControlDisabled              bool                   `mapstructure:"ip_access_control_disabled" bson:"ip_access_control_disabled" json:"ip_access_control_disabled"`
 	DontSetQuotasOnCreate                bool                   `mapstructure:"dont_set_quota_on_create" bson:"dont_set_quota_on_create" json:"dont_set_quota_on_create"`
 	ExpireAnalyticsAfter                 int64                  `mapstructure:"expire_analytics_after" bson:"expire_analytics_after" json:"expire_analytics_after"` // must have an expireAt TTL index set (http://docs.mongodb.org/manual/tutorial/expire-data/)
-	DisableExpireAnalytics               bool                   `mapstructure:"disable_expire_analytics" bson:"disable_expire_analytics" json:"disable_expire_analytics"`
 	ResponseProcessors                   []ResponseProcessor    `bson:"response_processors" json:"response_processors"`
 	CORS                                 CORSConfig             `bson:"CORS" json:"CORS"`
 	Domain                               string                 `bson:"domain" json:"domain"`

apidef/migration.go

@@ -438,8 +438,6 @@ func (a *APIDefinition) SetDisabledFlags() {
 	a.ConfigDataDisabled = true
 	a.Proxy.ServiceDiscovery.CacheDisabled = true
 	a.UptimeTests.Config.ServiceDiscovery.CacheDisabled = true
-	a.DisableExpireAnalytics = true
-
 	for i := 0; i < len(a.CustomMiddleware.Pre); i++ {
 		a.CustomMiddleware.Pre[i].Disabled = true
 	}

apidef/migration_test.go

@@ -642,7 +642,6 @@ func TestSetDisabledFlags(t *testing.T) {
 		},
 	}
 	expectedAPIDef := APIDefinition{
-		DisableExpireAnalytics: true,
 		CustomMiddleware: MiddlewareSection{
 			AuthCheck: MiddlewareDefinition{
 				Disabled: true,

apidef/oas/linter_test.go

@@ -86,13 +86,13 @@ func TestXTykGateway_Lint(t *testing.T) {
 
 		settings.Upstream.RateLimit.Per = ReadableDuration(10 * time.Second)
 		settings.Server.Authentication.CustomKeyLifetime.Value = ReadableDuration(10 * time.Second)
+		settings.Middleware.Global.TrafficLogs.CustomRetentionPeriod = ReadableDuration(10 * time.Second)
 
 		settings.Upstream.Authentication = &UpstreamAuth{
 			Enabled:   false,
 			BasicAuth: nil,
 			OAuth:     nil,
 		}
-		settings.Middleware.Global.TrafficLogs.RetentionPeriod.Value = ReadableDuration(time.Minute * 10)
 	}
 
 	// Encode data to json

apidef/oas/middleware.go

@@ -1581,9 +1581,9 @@ type TrafficLogs struct {
 	// TagHeaders is a string array of HTTP headers that can be extracted
 	// and transformed into analytics tags (statistics aggregated by tag, per hour).
 	TagHeaders []string `bson:"tagHeaders" json:"tagHeaders,omitempty"`
-	// RetentionPeriod holds the configuration for the analytics retention, it contains configuration
-	// for how long you would like analytics data to last for.
-	RetentionPeriod *RetentionPeriod `bson:"retentionPeriod" json:"retentionPeriod,omitempty"`
+	// CustomRetentionPeriod configures a custom value for how long the analytics is retained for,
+	// defaults to 100 years.
+	CustomRetentionPeriod ReadableDuration `bson:"customRetentionPeriod,omitempty" json:"customRetentionPeriod,omitempty"`
 	// Plugins configures custom plugins to allow for extensive modifications to analytics records
 	// The plugins would be executed in the order of configuration in the list.
 	Plugins CustomPlugins `bson:"plugins,omitempty" json:"plugins,omitempty"`
@@ -1593,14 +1593,7 @@ type TrafficLogs struct {
 func (t *TrafficLogs) Fill(api apidef.APIDefinition) {
 	t.Enabled = !api.DoNotTrack
 	t.TagHeaders = api.TagHeaders
-
-	if t.RetentionPeriod == nil {
-		t.RetentionPeriod = &RetentionPeriod{}
-	}
-	t.RetentionPeriod.Fill(api)
-	if ShouldOmit(t.RetentionPeriod) {
-		t.RetentionPeriod = nil
-	}
+	t.CustomRetentionPeriod = ReadableDuration(time.Duration(api.ExpireAnalyticsAfter) * time.Second)
 
 	if len(api.CustomMiddleware.TrafficLogs) == 0 {
 		t.Plugins = nil
@@ -1614,14 +1607,7 @@ func (t *TrafficLogs) Fill(api apidef.APIDefinition) {
 func (t *TrafficLogs) ExtractTo(api *apidef.APIDefinition) {
 	api.DoNotTrack = !t.Enabled
 	api.TagHeaders = t.TagHeaders
-
-	if t.RetentionPeriod == nil {
-		t.RetentionPeriod = &RetentionPeriod{}
-		defer func() {
-			t.RetentionPeriod = nil
-		}()
-	}
-	t.RetentionPeriod.ExtractTo(api)
+	api.ExpireAnalyticsAfter = int64(t.CustomRetentionPeriod.Seconds())
 
 	if len(t.Plugins) == 0 {
 		api.CustomMiddleware.TrafficLogs = nil
@@ -1631,42 +1617,6 @@ func (t *TrafficLogs) ExtractTo(api *apidef.APIDefinition) {
 	}
 }
 
-// RetentionPeriod holds the configuration for analytics retention.
-type RetentionPeriod struct {
-	// Enabled enables log analytics retention for the API
-	//
-	// Tyk classic API definition: `!disable_expire_analytics`.
-	Enabled bool `bson:"enabled" json:"enabled"`
-	// Value is the interval to keep the analytics record for
-	// The value of Value is a string that specifies the interval in a compact form,
-	// where hours, minutes and seconds are denoted by 'h', 'm' and 's' respectively.
-	// Multiple units can be combined to represent the duration.
-	//
-	// Examples of valid shorthand notations:
-	// - "1h"   : one hour
-	// - "20m"  : twenty minutes
-	// - "30s"  : thirty seconds
-	// - "1m29s": one minute and twenty-nine seconds
-	// - "1h30m" : one hour and thirty minutes
-	//
-	// An empty value is interpreted as "0s"
-	//
-	// Tyk classic API definition: `expire_analytics_after`.
-	Value ReadableDuration `bson:"value" json:"value"`
-}
-
-// Fill fills *RetentionPeriod from apidef.APIDefinition
-func (r *RetentionPeriod) Fill(api apidef.APIDefinition) {
-	r.Enabled = !api.DisableExpireAnalytics
-	r.Value = ReadableDuration(time.Duration(api.ExpireAnalyticsAfter) * time.Second)
-}
-
-// ExtractTo extracts *RetentionPeriod into apidef.APIDefinition
-func (r *RetentionPeriod) ExtractTo(api *apidef.APIDefinition) {
-	api.DisableExpireAnalytics = !r.Enabled
-	api.ExpireAnalyticsAfter = int64(r.Value.Seconds())
-}
-
 // GlobalRequestSizeLimit holds configuration about the global limits for request sizes.
 type GlobalRequestSizeLimit struct {
 	// Enabled activates the Request Size Limit.

apidef/oas/middleware_test.go

@@ -228,44 +228,22 @@ func TestTrafficLogs(t *testing.T) {
 		assert.Empty(t, convertedAPI.TagHeaders)
 	})
 
-	t.Run("retention header enabled", func(t *testing.T) {
-		trafficLogs := TrafficLogs{
-			Enabled: true,
-			RetentionPeriod: &RetentionPeriod{
-				Enabled: true,
-				Value:   ReadableDuration(time.Minute * 2),
-			},
-		}
-
+	t.Run("enable with retention period", func(t *testing.T) {
 		var convertedAPI apidef.APIDefinition
 		var resultTrafficLogs TrafficLogs
-		convertedAPI.SetDisabledFlags()
-		trafficLogs.ExtractTo(&convertedAPI)
-
-		assert.Equal(t, trafficLogs.RetentionPeriod.Enabled, !convertedAPI.DisableExpireAnalytics)
-		assert.Equal(t, int64(120), convertedAPI.ExpireAnalyticsAfter)
-
-		resultTrafficLogs.Fill(convertedAPI)
-		assert.Equal(t, trafficLogs, resultTrafficLogs)
-	})
-
-	t.Run("retention header disabled", func(t *testing.T) {
 		trafficLogs := TrafficLogs{
-			Enabled:         true,
-			RetentionPeriod: nil,
+			Enabled:               true,
+			CustomRetentionPeriod: ReadableDuration(time.Minute * 2),
 		}
 
-		var convertedAPI apidef.APIDefinition
-		var resultTrafficLogs TrafficLogs
-
 		convertedAPI.SetDisabledFlags()
 		trafficLogs.ExtractTo(&convertedAPI)
 
-		assert.Equal(t, true, convertedAPI.DisableExpireAnalytics)
-		assert.Equal(t, int64(0), convertedAPI.ExpireAnalyticsAfter)
+		assert.Equal(t, int64(120), convertedAPI.ExpireAnalyticsAfter)
 
 		resultTrafficLogs.Fill(convertedAPI)
-		assert.Nil(t, resultTrafficLogs.RetentionPeriod)
+
+		assert.Equal(t, trafficLogs, resultTrafficLogs)
 	})
 
 	t.Run("with custom analytics plugin", func(t *testing.T) {

apidef/oas/oas_test.go

@@ -174,7 +174,6 @@ func TestOAS_ExtractTo_ResetAPIDefinition(t *testing.T) {
 	a.DisableRateLimit = false
 	a.DoNotTrack = false
 	a.IPAccessControlDisabled = false
-	a.DisableExpireAnalytics = false
 
 	// deprecated fields
 	a.Auth = apidef.AuthConfig{}
@@ -241,7 +240,6 @@ func TestOAS_ExtractTo_ResetAPIDefinition(t *testing.T) {
 		"APIDefinition.UptimeTests.Config.ExpireUptimeAnalyticsAfter",
 		"APIDefinition.UptimeTests.Config.ServiceDiscovery.CacheDisabled",
 		"APIDefinition.UptimeTests.Config.RecheckWait",
-		"APIDefinition.Proxy.PreserveHostHeader",
 		"APIDefinition.Proxy.DisableStripSlash",
 		"APIDefinition.Proxy.CheckHostAgainstUptimeTests",
 		"APIDefinition.Proxy.Transport.SSLInsecureSkipVerify",
@@ -941,9 +939,6 @@ func TestMigrateAndFillOAS_DropEmpties(t *testing.T) {
 			Global: &Global{
 				TrafficLogs: &TrafficLogs{
 					Enabled: true,
-					RetentionPeriod: &RetentionPeriod{
-						Enabled: true,
-					},
 				},
 			},
 		}, baseAPI.OAS.GetTykExtension().Middleware)

apidef/oas/schema/x-tyk-api-gateway.json

@@ -1346,6 +1346,9 @@
         },
         "loadBalancing": {
           "$ref": "#/definitions/X-Tyk-LoadBalancing"
+        },
+        "preserveHostHeader": {
+          "$ref": "#/definitions/X-Tyk-PreserveHostHeader"
         }
       },
       "required": [
@@ -2038,21 +2041,12 @@
             "type": "string"
           }
         },
-        "retentionPeriod": {
-          "type": "object",
-          "properties": {
-            "enabled": {
-              "type": "boolean"
-            },
-            "value": {
-              "type": "string",
-              "pattern": "^(\\d+h)?(\\d+m)?(\\d+s)?$"
-            }
-          },
-          "required": [
-            "enabled",
-            "value"
-          ]
+        "customRetentionPeriod": {
+          "type": "string",
+          "pattern": "^(\\d+h)?(\\d+m)?(\\d+s)?$",
+          "not": {
+            "pattern": "^(0h)?(0m)?(0s)?$"
+          }
         },
         "plugins": {
           "type": "array",
@@ -2292,6 +2286,17 @@
         "enabled"
       ]
     },
+    "X-Tyk-PreserveHostHeader": {
+      "type": "object",
+        "properties": {
+            "enabled": {
+            "type": "boolean"
+            }
+        },
+        "required": [
+            "enabled"
+        ]
+    },
     "X-Tyk-LoadBalancingTarget": {
       "type": "object",
       "properties": {

apidef/oas/upstream.go

@@ -35,6 +35,9 @@ type Upstream struct {
 
 	// LoadBalancing contains configuration for load balancing between multiple upstream targets.
 	LoadBalancing *LoadBalancing `bson:"loadBalancing,omitempty" json:"loadBalancing,omitempty"`
+
+	// PreserveHostHeader contains the configuration for preserving the host header.
+	PreserveHostHeader *PreserveHostHeader `bson:"preserveHostHeader,omitempty" json:"preserveHostHeader,omitempty"`
 }
 
 // Fill fills *Upstream from apidef.APIDefinition.
@@ -96,6 +99,20 @@ func (u *Upstream) Fill(api apidef.APIDefinition) {
 	}
 
 	u.fillLoadBalancing(api)
+
+	u.fillPreserveHostHeader(api)
+}
+
+func (u *Upstream) fillPreserveHostHeader(api apidef.APIDefinition) {
+	if u.PreserveHostHeader == nil {
+		u.PreserveHostHeader = &PreserveHostHeader{}
+	}
+
+	u.PreserveHostHeader.Fill(api)
+
+	if ShouldOmit(u.PreserveHostHeader) {
+		u.PreserveHostHeader = nil
+	}
 }
 
 // ExtractTo extracts *Upstream into *apidef.APIDefinition.
@@ -157,6 +174,19 @@ func (u *Upstream) ExtractTo(api *apidef.APIDefinition) {
 	u.Authentication.ExtractTo(&api.UpstreamAuth)
 
 	u.loadBalancingExtractTo(api)
+
+	u.preserveHostHeaderExtractTo(api)
+}
+
+func (u *Upstream) preserveHostHeaderExtractTo(api *apidef.APIDefinition) {
+	if u.PreserveHostHeader == nil {
+		u.PreserveHostHeader = &PreserveHostHeader{}
+		defer func() {
+			u.PreserveHostHeader = nil
+		}()
+	}
+
+	u.PreserveHostHeader.ExtractTo(api)
 }
 
 func (u *Upstream) fillLoadBalancing(api apidef.APIDefinition) {
@@ -905,3 +935,19 @@ func (l *LoadBalancing) ExtractTo(api *apidef.APIDefinition) {
 
 	api.Proxy.Targets = proxyConfTargets
 }
+
+// PreserveHostHeader holds the configuration for preserving the host header.
+type PreserveHostHeader struct {
+	// Enabled activates preserving the host header.
+	Enabled bool `json:"enabled" bson:"enabled"`
+}
+
+// Fill fills *PreserveHostHeader from apidef.APIDefinition.
+func (p *PreserveHostHeader) Fill(api apidef.APIDefinition) {
+	p.Enabled = api.Proxy.PreserveHostHeader
+}
+
+// ExtractTo extracts *PreserveHostHeader into *apidef.APIDefinition.
+func (p *PreserveHostHeader) ExtractTo(api *apidef.APIDefinition) {
+	api.Proxy.PreserveHostHeader = p.Enabled
+}