feature: Expand rate limiting capabilities

Previously, our rate limits were static in the sense that we could just turn them on or off and set specific namespaces where rate limiting is essentially turned off.
With this PR we expand the rate limiting capability so that the user can define its own rates per defined bucket. The schema in Quay's `config.yaml` should look as follows:

~~~
RATE_LIMITS:
  http1:
     v2: number
     registry: number
     api_resources: number
  http2:
     v2: number
     registry: number
     api_resources: number
  namespaced:
     http1_v2: number
     http2_v2: number
     http1_registry: number
     http2_registry: number
     http1_api_resources: number
     http2_api_resources: number
~~~

We define three separate buckets for access:

* `http1`: handles HTTP/1 connections
* `http2`: handles HTTP/2 connections
* `namespaced`: handles access to particular namespaces when rate limiting is turned on

The values represent different endpoints that Quay exposes:

* `v2`: denotes the Docker v2 endpoint exposed on `/v2/`
* `registry`: deals with registry operations on manifests and blobs
* `api_resources`: deals with connections to Quay's `v1` API as well as Docker v2 `tags` and `_catalog` endpoint

conf/init/nginx_conf_create.py

@@ -54,6 +54,28 @@
     "!KRB5-DES-CBC3-SHA",
 ]
 
+DEFAULT_RATE_LIMITS =   {
+    "http1":    {
+        "v2": 60,
+        "registry": 50,
+        "api_resources": 5
+    },     
+    "http2":    {
+        "v2": 600,
+        "registry": 500,
+        "api_resources": 50,
+    },
+    "namespaced":   {
+        "http1_v2": 60,
+        "http2_v2": 600,
+        "http1_registry": 50,
+        "http2_registry": 500,
+        "http1_api_resources": 5,
+        "http2_api_resources": 50,
+    },
+}
+
+
 
 def write_config(filename, **kwargs):
     with open(filename + ".jnj") as f:
@@ -121,10 +143,22 @@ def generate_rate_limiting_config(config):
     config = config or {}
     non_rate_limited_namespaces = config.get("NON_RATE_LIMITED_NAMESPACES") or set()
     enable_rate_limits = config.get("FEATURE_RATE_LIMITS", False)
+    if enable_rate_limits == True:
+        http1_bucket = [value for _, value in config['RATE_LIMITS']['http1'].items()]
+        http2_bucket = [value for _, value in config['RATE_LIMITS']['http2'].items()]
+        namespaced_bucket = [value for _, value in config['RATE_LIMITS']['namespaced'].items()]
+    else:
+        http1_bucket = [value for _, value in DEFAULT_RATE_LIMITS['http1'].items()]
+        http2_bucket = [value for _, value in DEFAULT_RATE_LIMITS['http2'].items()]
+        namespaced_bucket = [value for _, value in DEFAULT_RATE_LIMITS['namespaced'].items()]
+
     write_config(
         os.path.join(QUAYCONF_DIR, "nginx/rate-limiting.conf"),
         non_rate_limited_namespaces=non_rate_limited_namespaces,
         enable_rate_limits=enable_rate_limits,
+        http1_bucket=http1_bucket,
+        http2_bucket=http2_bucket,
+        namespaced_bucket=namespaced_bucket,
         static_dir=STATIC_DIR,
     )
 

conf/nginx/rate-limiting.conf.jnj

@@ -49,24 +49,27 @@ limit_req_zone $http_authorization zone=staticauth:10m rate=30r/s;
 limit_req_zone $request_id zone=staticauth:10m rate=300r/s;
 {% endif %}
 
-limit_req_zone $http1_bucket zone=dynamicauth_very_light_http1:10m rate=60r/s;
-limit_req_zone $http2_bucket zone=dynamicauth_very_light_http2:10m rate=600r/s;
-limit_req_zone $namespaced_http1_bucket zone=namespaced_dynamicauth_very_light_http1:10m rate=60r/s;
-limit_req_zone $namespaced_http2_bucket zone=namespaced_dynamicauth_very_light_http2:10m rate=600r/s;
+# Rate limits for the v2 API endpoint.
+limit_req_zone $http1_bucket zone=v2_endpoint_http1:10m rate={{ http1_bucket[0] }}r/s;
+limit_req_zone $http2_bucket zone=v2_endpoint_http2:10m rate={{ http2_bucket[0] }}r/s;
+limit_req_zone $namespaced_http1_bucket zone=namespaced_v2_endpoint_http1:10m rate={{ namespaced_bucket[0] }}r/s;
+limit_req_zone $namespaced_http2_bucket zone=namespaced_v2_endpoint_http2:10m rate={{ namespaced_bucket[1] }}r/s;
 
-limit_req_zone $http1_bucket zone=dynamicauth_light_http1:10m rate=50r/s;
-limit_req_zone $http2_bucket zone=dynamicauth_light_http2:10m rate=500r/s;
-limit_req_zone $namespaced_http1_bucket zone=namespaced_dynamicauth_light_http1:10m rate=50r/s;
-limit_req_zone $namespaced_http2_bucket zone=namespaced_dynamicauth_light_http2:10m rate=500r/s;
+# Rate limits for the registry operations on manifests and blobs.
+limit_req_zone $http1_bucket zone=registry_http1:10m rate={{ http1_bucket[1] }}r/s;
+limit_req_zone $http2_bucket zone=registry_http2:10m rate={{ http2_bucket[1] }}r/s;
+limit_req_zone $namespaced_http1_bucket zone=namespaced_registry_http1:10m rate={{ namespaced_bucket[2] }}r/s;
+limit_req_zone $namespaced_http2_bucket zone=namespaced_registry_http2:10m rate={{ namespaced_bucket[3] }}r/s;
 
+# Rate limits for API resources such as access to Quay's v1 API, tag list and _catalog.
 # This zone should always be used with burst=<number> (nodelay|delay) as the
 # limit is very low on purpose but should allow for the burst of traffic
 # required for a registry operation. The burst number should also vary per
 # endpoint.
-limit_req_zone $http1_bucket zone=dynamicauth_heavy_http1:10m rate=5r/s;
-limit_req_zone $http2_bucket zone=dynamicauth_heavy_http2:10m rate=50r/s;
-limit_req_zone $namespaced_http1_bucket zone=namespaced_dynamicauth_heavy_http1:10m rate=5r/s;
-limit_req_zone $namespaced_http2_bucket zone=namespaced_dynamicauth_heavy_http2:10m rate=50r/s;
+limit_req_zone $http1_bucket zone=api_resources_heavy_http1:10m rate={{ http1_bucket[2] }}r/s;
+limit_req_zone $http2_bucket zone=api_resources_heavy_http2:10m rate={{ http2_bucket[2] }}r/s;
+limit_req_zone $namespaced_http1_bucket zone=namespaced_api_resources_http1:10m rate={{ namespaced_bucket[4] }}r/s;
+limit_req_zone $namespaced_http2_bucket zone=namespaced_api_resources_http2:10m rate={{ namespaced_bucket[5] }}r/s;
 
 limit_req_status 429;
 limit_req_log_level warn;

conf/nginx/server-base.conf.jnj

@@ -112,8 +112,8 @@ location ~ ^/v2/_catalog(.*)$ {
     keepalive_timeout 0; # Disables HTTP 1.1 keep-alive and forces round-robin.
 
     {% if enable_rate_limits %}
-    limit_req zone=dynamicauth_heavy_http1 burst=1 nodelay;
-    limit_req zone=dynamicauth_heavy_http2 burst=5 nodelay;
+    limit_req zone=api_resources_heavy_http1 burst=1 nodelay;
+    limit_req zone=api_resources_heavy_http2 burst=5 nodelay;
     {% endif %}
 }
 
@@ -133,8 +133,8 @@ location /api/ {
     proxy_pass   http://web_app_server;
 
     {% if enable_rate_limits %}
-    limit_req zone=dynamicauth_heavy_http1 burst=25 nodelay;
-    limit_req zone=dynamicauth_heavy_http2 burst=100 nodelay;
+    limit_req zone=api_resources_heavy_http1 burst=25 nodelay;
+    limit_req zone=api_resources_heavy_http2 burst=100 nodelay;
     {% endif %}
 
     keepalive_timeout 0; # Disables HTTP 1.1 keep-alive and forces round-robin.
@@ -184,8 +184,8 @@ location ~ /v2/([^/]+)(/[^/]+)+/blobs/ {
     set $namespace $1;
 
     {% if enable_rate_limits %}
-    limit_req zone=namespaced_dynamicauth_light_http1 burst=50 nodelay;
-    limit_req zone=namespaced_dynamicauth_light_http2 burst=100 nodelay;
+    limit_req zone=namespaced_registry_http1 burst=50 nodelay;
+    limit_req zone=namespaced_registry_http2 burst=100 nodelay;
     {% endif %}
 
     keepalive_timeout 0; # Disables HTTP 1.1 keep-alive and forces round-robin.
@@ -211,8 +211,8 @@ location ~ /v2/([^/]+)\/[^/]+/tags/ {
     set $namespace $1;
 
     {% if enable_rate_limits %}
-    limit_req zone=namespaced_dynamicauth_heavy_http1 burst=2 nodelay;
-    limit_req zone=namespaced_dynamicauth_heavy_http2 burst=2 nodelay;
+    limit_req zone=namespaced_api_resources_http1 burst=2 nodelay;
+    limit_req zone=namespaced_api_resources_http2 burst=2 nodelay;
     {% endif %}
 
     keepalive_timeout 0; # Disables HTTP 1.1 keep-alive and forces round-robin.
@@ -238,8 +238,8 @@ location ~ /v2/([^/]+)\/[^/]+/manifests/ {
     set $namespace $1;
 
     {% if enable_rate_limits %}
-    limit_req zone=namespaced_dynamicauth_light_http1 burst=10 nodelay;
-    limit_req zone=namespaced_dynamicauth_light_http2 burst=50 nodelay;
+    limit_req zone=namespaced_registry_http1 burst=10 nodelay;
+    limit_req zone=namespaced_registry_http2 burst=50 nodelay;
     {% endif %}
 
     keepalive_timeout 0; # Disables HTTP 1.1 keep-alive and forces round-robin.
@@ -292,8 +292,8 @@ location ~ ^/v2 {
     proxy_pass   http://registry_app_server;
 
     {% if enable_rate_limits %}
-    limit_req zone=dynamicauth_very_light_http1 burst=20 nodelay;
-    limit_req zone=dynamicauth_very_light_http2 burst=80 nodelay;
+    limit_req zone=namespaced_v2_endpoint_http1 burst=20 nodelay;
+    limit_req zone=namespaced_v2_endpoint_http2 burst=80 nodelay;
     {% endif %}
 
     keepalive_timeout 0; # Disables HTTP 1.1 keep-alive and forces round-robin.
@@ -317,8 +317,8 @@ location /v1/ {
     client_max_body_size {{ maximum_layer_size }};
 
     {% if enable_rate_limits %}
-    limit_req zone=dynamicauth_heavy_http1 burst=5 nodelay;
-    limit_req zone=dynamicauth_heavy_http2 burst=25 nodelay;
+    limit_req zone=api_resources_heavy_http1 burst=5 nodelay;
+    limit_req zone=api_resources_heavy_http2 burst=25 nodelay;
     {% endif %}
 
     keepalive_timeout 0; # Disables HTTP 1.1 keep-alive and forces round-robin.

util/config/schema.py

@@ -93,6 +93,7 @@
     "RESET_CHILD_MANIFEST_EXPIRATION",
     "PERMANENTLY_DELETE_TAGS",
     "FEATURE_RH_MARKETPLACE",
+    "RATE_LIMITS",
 }
 
 CONFIG_SCHEMA = {