diff --git a/conf/default.cfg b/conf/default.cfg index 8c092e31..76fe2c94 100644 --- a/conf/default.cfg +++ b/conf/default.cfg @@ -302,7 +302,12 @@ use = egg:swift#catch_errors use = egg:swift#ratelimit [filter:healthcheck] -use = egg:swift#healthcheck +# Original healthcheck middleware +#use = egg:swift#healthcheck +# OpenIO version, counting requests +use = egg:oioswift#healthcheck +# Customize the path used to get the counters. +#status_path = /_status [filter:cache] use = egg:swift#memcache diff --git a/conf/s3-container-hierarchy.cfg b/conf/s3-container-hierarchy.cfg index e01c4a29..397299b7 100644 --- a/conf/s3-container-hierarchy.cfg +++ b/conf/s3-container-hierarchy.cfg @@ -43,7 +43,7 @@ use = egg:swift#catch_errors use = egg:swift#ratelimit [filter:healthcheck] -use = egg:swift#healthcheck +use = egg:oioswift#healthcheck [filter:cache] use = egg:swift#memcache diff --git a/conf/s3-default.cfg b/conf/s3-default.cfg index 9c40de3b..fc778807 100644 --- a/conf/s3-default.cfg +++ b/conf/s3-default.cfg @@ -35,7 +35,7 @@ use = egg:swift#catch_errors use = egg:swift#ratelimit [filter:healthcheck] -use = egg:swift#healthcheck +use = egg:oioswift#healthcheck [filter:cache] use = egg:swift#memcache diff --git a/conf/swift-flatns-skip-metadata.cfg b/conf/swift-flatns-skip-metadata.cfg index 70a4641d..cc58edec 100644 --- a/conf/swift-flatns-skip-metadata.cfg +++ b/conf/swift-flatns-skip-metadata.cfg @@ -8,7 +8,7 @@ eventlet_debug = true sds_default_account = AUTH_demo [pipeline:main] -pipeline = catch_errors cache proxy-logging slo hashedcontainer proxy-server +pipeline = catch_errors healthcheck cache proxy-logging slo hashedcontainer proxy-server [app:proxy-server] use = egg:oioswift#main @@ -36,6 +36,9 @@ skip_metadata = true [filter:gatekeeper] use = egg:swift#gatekeeper +[filter:healthcheck] +use = egg:oioswift#healthcheck + [filter:proxy-logging] use = egg:swift#proxy_logging diff --git a/oioswift/__init__.py b/oioswift/__init__.py index ff987d28..da77e85c 100644 --- a/oioswift/__init__.py +++ b/oioswift/__init__.py @@ -1 +1 @@ -__version__ = '1.10.1' +__version__ = '1.11.0' diff --git a/oioswift/common/middleware/healthcheck.py b/oioswift/common/middleware/healthcheck.py new file mode 100644 index 00000000..bae98ec0 --- /dev/null +++ b/oioswift/common/middleware/healthcheck.py @@ -0,0 +1,78 @@ +# Copyright (c) 2010-2012 OpenStack Foundation +# Copyright (c) 2019 OpenIO SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from swift.common.middleware.healthcheck import HealthCheckMiddleware as HCM +from swift.common.swob import HTTPOk +from swift.common.utils import config_auto_int_value +from oio.common.json import json + +STATUS_PATH = '/_status' + + +class HealthCheckMiddleware(HCM): + """ + Extension of swift's HealthCheckMiddleware counting the number of + requests currently being processed. + """ + + def __init__(self, app, conf): + super(HealthCheckMiddleware, self).__init__(app, conf) + self.status_path = conf.get('status_path', STATUS_PATH) + counters = conf.get('oioswift_counters', {}) + self.cur_reqs = counters.get('current_requests') + self.workers = config_auto_int_value(conf.get('workers'), 1) + + def dump_status(self): + """ + Build a response with the current status of the server + as a json object. + """ + cur_reqs = self.cur_reqs.value if self.cur_reqs else 0 + status = { + 'stat.cur_reqs': cur_reqs, + 'stat.workers': self.workers, + } + return HTTPOk(body=json.dumps(status), + headers={'Content-Type': 'application/json'}) + + def __call__(self, env, start_response): + path = env.get('PATH_INFO') + if path == self.status_path: + return self.dump_status()(env, start_response) + elif path == '/healthcheck': + # Do not count health check requests + return super(HealthCheckMiddleware, self).__call__( + env, start_response) + + if self.cur_reqs: + with self.cur_reqs.get_lock(): + self.cur_reqs.value += 1 + try: + return super(HealthCheckMiddleware, self).__call__( + env, start_response) + finally: + if self.cur_reqs: + with self.cur_reqs.get_lock(): + self.cur_reqs.value -= 1 + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def healthcheck_filter(app): + return HealthCheckMiddleware(app, conf) + return healthcheck_filter diff --git a/oioswift/server.py b/oioswift/server.py index 6434789f..c1783ad3 100644 --- a/oioswift/server.py +++ b/oioswift/server.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2018 OpenIO SAS +# Copyright (c) 2016-2019 OpenIO SAS # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import multiprocessing from swift.common import request_helpers, storage_policy from oioswift.common.request_helpers import OioSegmentedIterable from oioswift.common.storage_policy import POLICIES @@ -101,6 +102,25 @@ def __init__(self, conf, memcache=None, logger=None, account_ring=None, config_true_value(conf.get('check_state', False)) +def global_conf_callback(preloaded_app_conf, global_conf): + """ + Callback for swift.common.wsgi.run_wsgi during the global_conf + creation so that we can add our shared memory manager. + + :param preloaded_app_conf: The preloaded conf for the WSGI app. + This conf instance will go away, so + just read from it, don't write. + :param global_conf: The global conf that will eventually be + passed to the app_factory function later. + This conf is created before the worker + subprocesses are forked, so can be useful to + set up semaphores, shared memory, etc. + """ + global_conf['oioswift_counters'] = { + 'current_requests': multiprocessing.Value('i', 0), + } + + def app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) diff --git a/setup.py b/setup.py index a8417f9b..1a3a2025 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ 'autocontainer=oioswift.common.middleware.autocontainer:filter_factory', 'encryption=oioswift.common.middleware.crypto:filter_factory', 'hashedcontainer=oioswift.common.middleware.hashedcontainer:filter_factory', + 'healthcheck=oioswift.common.middleware.healthcheck:filter_factory', 'keymaster=oioswift.common.middleware.crypto.keymaster:filter_factory', 'regexcontainer=oioswift.common.middleware.regexcontainer:filter_factory', 'versioned_writes=oioswift.common.middleware.versioned_writes:filter_factory', diff --git a/tests/functional/run-swift-tests.sh b/tests/functional/run-swift-tests.sh index 134c67d0..0556e550 100755 --- a/tests/functional/run-swift-tests.sh +++ b/tests/functional/run-swift-tests.sh @@ -10,7 +10,9 @@ run_sds || exit 1 RET=0 -run_functional_test swift-flatns-skip-metadata.cfg swift-skip-metadata.sh +run_functional_test swift-flatns-skip-metadata.cfg \ + swift-skip-metadata.sh \ + swift-healthcheck.sh # TODO(FVE): gridinit_cmd stop exit $RET diff --git a/tests/functional/swift-healthcheck.sh b/tests/functional/swift-healthcheck.sh new file mode 100755 index 00000000..cdde75bb --- /dev/null +++ b/tests/functional/swift-healthcheck.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "# Running tests against the custom healthcheck middleware" + +set -e +DATA=$(curl -s http://localhost:5000/_status) +WORKERS=$(echo "$DATA" | jq -r '."stat.workers"') +CUR_REQS=$(echo "$DATA" | jq -r '."stat.cur_reqs"') +echo "healthcheck is reporting $WORKERS workers and $CUR_REQS requests" +echo "OK"