Implement custom healthcheck middleware

This extension of swift's healthcheck middleware counts the number of
requests currently being processed. The counter can be retrieved from
a JSON object generated when calling the '/_status' route (can be
changed by configuration).

oioswift processes can be registered in oio-sds' conscience with
watchers configured like this:

    host: 10.0.3.1
    port: 5000
    type: oioswift
    service_id: oioswift-1
    location: rack.127-0-0-1.novol
    checks:
        - {type: http, uri: "/healthcheck"}
    slots:
        - oioswift
    stats:
        - {type: system}
        - {type: http, path: "/_status", parser: json}

conf/default.cfg

@@ -302,7 +302,12 @@ use = egg:swift#catch_errors
 use = egg:swift#ratelimit
 
 [filter:healthcheck]
-use = egg:swift#healthcheck
+# Original healthcheck middleware
+#use = egg:swift#healthcheck
+# OpenIO version, counting requests
+use = egg:oioswift#healthcheck
+# Customize the path used to get the counters.
+#status_path = /_status
 
 [filter:cache]
 use = egg:swift#memcache

conf/s3-container-hierarchy.cfg

@@ -43,7 +43,7 @@ use = egg:swift#catch_errors
 use = egg:swift#ratelimit
 
 [filter:healthcheck]
-use = egg:swift#healthcheck
+use = egg:oioswift#healthcheck
 
 [filter:cache]
 use = egg:swift#memcache

conf/s3-default.cfg

@@ -35,7 +35,7 @@ use = egg:swift#catch_errors
 use = egg:swift#ratelimit
 
 [filter:healthcheck]
-use = egg:swift#healthcheck
+use = egg:oioswift#healthcheck
 
 [filter:cache]
 use = egg:swift#memcache

conf/swift-flatns-skip-metadata.cfg

@@ -8,7 +8,7 @@ eventlet_debug = true
 sds_default_account = AUTH_demo
 
 [pipeline:main]
-pipeline = catch_errors cache proxy-logging slo hashedcontainer proxy-server
+pipeline = catch_errors healthcheck cache proxy-logging slo hashedcontainer proxy-server
 
 [app:proxy-server]
 use = egg:oioswift#main
@@ -36,6 +36,9 @@ skip_metadata = true
 [filter:gatekeeper]
 use = egg:swift#gatekeeper
 
+[filter:healthcheck]
+use = egg:oioswift#healthcheck
+
 [filter:proxy-logging]
 use = egg:swift#proxy_logging
 

oioswift/__init__.py

@@ -1 +1 @@
-__version__ = '1.10.1'
+__version__ = '1.11.0'

oioswift/common/middleware/healthcheck.py

@@ -0,0 +1,78 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
+# Copyright (c) 2019 OpenIO SAS
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from swift.common.middleware.healthcheck import HealthCheckMiddleware as HCM
+from swift.common.swob import HTTPOk
+from swift.common.utils import config_auto_int_value
+from oio.common.json import json
+
+STATUS_PATH = '/_status'
+
+
+class HealthCheckMiddleware(HCM):
+    """
+    Extension of swift's HealthCheckMiddleware counting the number of
+    requests currently being processed.
+    """
+
+    def __init__(self, app, conf):
+        super(HealthCheckMiddleware, self).__init__(app, conf)
+        self.status_path = conf.get('status_path', STATUS_PATH)
+        counters = conf.get('oioswift_counters', {})
+        self.cur_reqs = counters.get('current_requests')
+        self.workers = config_auto_int_value(conf.get('workers'), 1)
+
+    def dump_status(self):
+        """
+        Build a response with the current status of the server
+        as a json object.
+        """
+        cur_reqs = self.cur_reqs.value if self.cur_reqs else 0
+        status = {
+            'stat.cur_reqs': cur_reqs,
+            'stat.workers': self.workers,
+        }
+        return HTTPOk(body=json.dumps(status),
+                      headers={'Content-Type': 'application/json'})
+
+    def __call__(self, env, start_response):
+        path = env.get('PATH_INFO')
+        if path == self.status_path:
+            return self.dump_status()(env, start_response)
+        elif path == '/healthcheck':
+            # Do not count health check requests
+            return super(HealthCheckMiddleware, self).__call__(
+                env, start_response)
+
+        if self.cur_reqs:
+            with self.cur_reqs.get_lock():
+                self.cur_reqs.value += 1
+        try:
+            return super(HealthCheckMiddleware, self).__call__(
+                env, start_response)
+        finally:
+            if self.cur_reqs:
+                with self.cur_reqs.get_lock():
+                    self.cur_reqs.value -= 1
+
+
+def filter_factory(global_conf, **local_conf):
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def healthcheck_filter(app):
+        return HealthCheckMiddleware(app, conf)
+    return healthcheck_filter

oioswift/server.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2016-2018 OpenIO SAS
+# Copyright (c) 2016-2019 OpenIO SAS
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import multiprocessing
 from swift.common import request_helpers, storage_policy
 from oioswift.common.request_helpers import OioSegmentedIterable
 from oioswift.common.storage_policy import POLICIES
@@ -101,6 +102,25 @@ def __init__(self, conf, memcache=None, logger=None, account_ring=None,
             config_true_value(conf.get('check_state', False))
 
 
+def global_conf_callback(preloaded_app_conf, global_conf):
+    """
+    Callback for swift.common.wsgi.run_wsgi during the global_conf
+    creation so that we can add our shared memory manager.
+
+    :param preloaded_app_conf: The preloaded conf for the WSGI app.
+                               This conf instance will go away, so
+                               just read from it, don't write.
+    :param global_conf: The global conf that will eventually be
+                        passed to the app_factory function later.
+                        This conf is created before the worker
+                        subprocesses are forked, so can be useful to
+                        set up semaphores, shared memory, etc.
+    """
+    global_conf['oioswift_counters'] = {
+        'current_requests': multiprocessing.Value('i', 0),
+    }
+
+
 def app_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)

setup.py

@@ -33,6 +33,7 @@
             'autocontainer=oioswift.common.middleware.autocontainer:filter_factory',
             'encryption=oioswift.common.middleware.crypto:filter_factory',
             'hashedcontainer=oioswift.common.middleware.hashedcontainer:filter_factory',
+            'healthcheck=oioswift.common.middleware.healthcheck:filter_factory',
             'keymaster=oioswift.common.middleware.crypto.keymaster:filter_factory',
             'regexcontainer=oioswift.common.middleware.regexcontainer:filter_factory',
             'versioned_writes=oioswift.common.middleware.versioned_writes:filter_factory',

tests/functional/run-swift-tests.sh

@@ -10,7 +10,9 @@ run_sds || exit 1
 
 RET=0
 
-run_functional_test swift-flatns-skip-metadata.cfg swift-skip-metadata.sh
+run_functional_test swift-flatns-skip-metadata.cfg \
+    swift-skip-metadata.sh \
+    swift-healthcheck.sh
 
 # TODO(FVE): gridinit_cmd stop
 exit $RET

tests/functional/swift-healthcheck.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+echo "# Running tests against the custom healthcheck middleware"
+
+set -e
+DATA=$(curl -s http://localhost:5000/_status)
+WORKERS=$(echo "$DATA" | jq -r '."stat.workers"')
+CUR_REQS=$(echo "$DATA" | jq -r '."stat.cur_reqs"')
+echo "healthcheck is reporting $WORKERS workers and $CUR_REQS requests"
+echo "OK"