diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c852f1c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.coverage +.tox/ +htmlcov/ +.mypy_cache +*egg-info +**/__pycache__/ +*.py[cod] + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b803309 --- /dev/null +++ b/.gitignore @@ -0,0 +1,77 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# vim +*.swp + +# pycharm +.idea + +# testing +config.json +server_config.json +script_config.json +work_dir +artifact_dir +build-tools + +beetmoverscript/test/test_artifact_dir/ + +# misc +.DS_Store diff --git a/.taskcluster.yml b/.taskcluster.yml new file mode 100644 index 0000000..511aaaa --- /dev/null +++ b/.taskcluster.yml @@ -0,0 +1,128 @@ +version: 1 +policy: + pullRequests: public +tasks: + $let: + head_rev: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.sha} + else: + $if: 'tasks_for == "github-push"' + then: ${event.after} + else: ${event.release.tag_name} + + repository: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.repo.html_url} + else: ${event.repository.html_url} + + owner: ${event.sender.login}@users.noreply.github.com + + docker_tag: + $if: 'tasks_for == "github-pull-request"' + then: pull-request + else: + $if: 'tasks_for == "github-push"' + then: + $if: 'event.ref[:10] == "refs/tags/"' + then: ${event.ref[10:]} # strip "refs/tags/" + else: ${event.ref[11:]} # strip "refs/heads/" + else: "v${event.release.tag_name}" # prefix the version with "v" + + in: + - taskId: '${as_slugid("py37")}' + provisionerId: aws-provisioner-v1 + workerType: github-worker + created: {$fromNow: ''} + deadline: {$fromNow: '4 hours'} + payload: + maxRunTime: 3600 + image: python:3.7 + command: + - sh + - -lxce + - >- + git clone ${repository} /src && + cd /src && + git config advice.detachedHead false && + git checkout ${head_rev} && + pip install tox && + tox + metadata: + name: tox py37 (${docker_tag}) + description: code linting & unit tests on py37 (${docker_tag}) + owner: ${owner} + source: ${repository}/raw/${head_rev}/.taskcluster.yml + + - taskId: '${as_slugid("docker_build")}' + dependencies: + - '${as_slugid("py37")}' + provisionerId: aws-provisioner-v1 + workerType: github-worker + created: {$fromNow: ''} + deadline: {$fromNow: '4 hours'} + payload: + features: + dind: true + maxRunTime: 3600 + image: mozillareleases/python-test-runner + command: + - bash + - -ce + - >- + git clone ${repository} /src && + cd /src && + git config advice.detachedHead false && + git checkout ${head_rev} && + docker build -f Dockerfile -t mozilla/releng-k8s-autoscale:${docker_tag} . && + docker save mozilla/releng-k8s-autoscale:${docker_tag} > /tmp/image.tar + artifacts: + public/image.tar: + expires: {$fromNow: '2 weeks'} + path: /tmp/image.tar + type: file + metadata: + name: docker build (${docker_tag}) + description: build latest docker image (${docker_tag}) + owner: ${owner} + source: ${repository}/raw/${head_rev}/.taskcluster.yml + + - $if: 'tasks_for != "github-pull-request"' + then: + $if: 'repository == "https://github.com/mozilla-releng/k8s-autoscale"' + then: + taskId: '${as_slugid("docker_push")}' + dependencies: + - '${as_slugid("docker_build")}' + provisionerId: aws-provisioner-v1 + workerType: github-worker + created: {$fromNow: ''} + deadline: {$fromNow: '4 hours'} + payload: + features: + # Needed for access to secret + taskclusterProxy: true + dind: true + maxRunTime: 3600 + image: mozillareleases/python-test-runner + command: + - bash + - -ce + - >- + git clone ${repository} /src && + cd /src && + git config advice.detachedHead false && + git checkout ${head_rev} && + IMAGE_TASK_ID=${as_slugid("docker_build")} + SECRET_URL=http://taskcluster/secrets/v1/secret/project/releng/k8s-autoscale/deploy + DOCKERHUB_EMAIL=release+dockerhub+services@mozilla.com + DOCKERHUB_USER=mozillarelengservices + TAG=mozilla/releng-k8s-autoscale:${docker_tag} + ./docker.d/push_image.sh + scopes: + - secrets:get:project/releng/k8s-autoscale/deploy + metadata: + name: docker push (${docker_tag}) + description: push docker image (${docker_tag}) + owner: ${owner} + source: ${repository}/raw/${head_rev}/.taskcluster.yml diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..498baa3 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,15 @@ +# Community Participation Guidelines + +This repository is governed by Mozilla's code of conduct and etiquette guidelines. +For more details, please read the +[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). + +## How to Report +For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. + + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1098d02 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.7 + +RUN groupadd --gid 10001 app && \ + useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app + +USER app +WORKDIR /app + +COPY . /app +# generate /app/version.json according to https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md +RUN ./docker.d/generate_version_json.sh + +RUN python -m venv /app +RUN ./bin/pip install -r requirements/base.txt +RUN ./bin/pip install -e . + +COPY docker.d/healthcheck /bin/healthcheck +COPY docker.d/init.sh /app/bin/init.sh + +CMD ["/app/bin/init.sh"] diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..68d4d93 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,8 @@ +======= +History +======= + +0.1.0 (2019-05-07) +------------------ + +* First release diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..39b9813 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,27 @@ +include CONTRIBUTING.rst +include HISTORY.rst +include LICENSE +include README.rst +include CODE_OF_CONDUCT.md +include tox.ini +include config-example.yaml +include version.txt + +recursive-include requirements *.in +recursive-include requirements *.txt +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.rst conf.py + +# added by check_manifest.py +include *.txt + +# added by check_manifest.py +include Dockerfile +recursive-include configs *.yml +recursive-include docker.d * + +exclude .dockerignore +exclude .taskcluster.yml diff --git a/README.md b/README.md deleted file mode 100644 index 5e3826c..0000000 --- a/README.md +++ /dev/null @@ -1 +0,0 @@ -# k8s-autoscale diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..e7f211f --- /dev/null +++ b/README.rst @@ -0,0 +1,16 @@ +==================== +Kubernetes Autoscale +==================== + +Autoscale scriptworkers in Kubernetes + +* Free software: MPL2 +* Documentation: https://k8s-autoscale.readthedocs.io. + +Features +-------- + +* TODO + +Credits +------- diff --git a/config-example.yaml b/config-example.yaml new file mode 100644 index 0000000..e6e48fc --- /dev/null +++ b/config-example.yaml @@ -0,0 +1,18 @@ +worker_types: + - name: shipitscript-v1 + provisioner: scriptworker-prov-v1 + poll_interval: 180 + deployment_namespace: default + deployment_name: nginx-1 + # remove kube_config and kube_connfig_context to use in-cluster auth + kube_connfig: ~/.kube/config + kube_connfig_context: xx + autoscale: + algorithm: sla + args: + max_replicas: 20 + avg_task_duration: 60 + boot_time: 60 + tolerance_seconds: 300 + # cover 100% of pending + capacity_ratio: 1.0 diff --git a/configs/production-beetmover.yml b/configs/production-beetmover.yml new file mode 100644 index 0000000..0dcffc9 --- /dev/null +++ b/configs/production-beetmover.yml @@ -0,0 +1,15 @@ +worker_types: + - name: beetmoverworker-v1 + provisioner: scriptworker-prov-v1 + poll_interval: 180 + deployment_namespace: TODO + deployment_name: TODO + autoscale: + algorithm: sla + args: + max_replicas: 20 + avg_task_duration: 60 + boot_time: 60 + tolerance_seconds: 300 + # cover 100% of pending + capacity_ratio: 1.0 diff --git a/configs/stage-beetmover.yml b/configs/stage-beetmover.yml new file mode 100644 index 0000000..6d51e34 --- /dev/null +++ b/configs/stage-beetmover.yml @@ -0,0 +1,15 @@ +worker_types: + - name: beetmoverworker-dev + provisioner: scriptworker-prov-v1 + poll_interval: 180 + deployment_namespace: TODO + deployment_name: TODO + autoscale: + algorithm: sla + args: + max_replicas: 2 + avg_task_duration: 60 + boot_time: 60 + tolerance_seconds: 300 + # cover 100% of pending + capacity_ratio: 1.0 diff --git a/docker.d/generate_version_json.sh b/docker.d/generate_version_json.sh new file mode 100755 index 0000000..c993fc2 --- /dev/null +++ b/docker.d/generate_version_json.sh @@ -0,0 +1,14 @@ +#/bin/bash + +set -xe +commit=$(git rev-parse HEAD) +version=$(cat version.txt) + +cat > version.json < args["avg_task_duration"] + # In case we don't want to cover all the pending tasks + pending = int(math.ceil(pending * args["capacity_ratio"])) + # Scale down only when we have no pending tasks + if pending == 0: + return -(running + booting) + # How many tasks a replica can process within our tolerance period + new_tasks_per_replica = math.floor( + (args["sla_seconds"] - args["boot_time"]) / args["avg_task_duration"] + ) + # how many tasks can be covered by the running replicas, assuming they are + # busy and can only take new tasks after they are done with the current one + running_tasks_per_replica = ( + math.floor(args["sla_seconds"] / args["avg_task_duration"]) - 1 + ) + # how many tasks the booting replicas can cover, assuming they will be + # available for new jobs as soon as they boot up + booting_tasks_per_replica = math.floor( + (args["sla_seconds"] - args["boot_time"]) / args["avg_task_duration"] + ) + running_can_cover = running * running_tasks_per_replica + booting_can_cover = booting * booting_tasks_per_replica + still_pending = pending - (running_can_cover + booting_can_cover) + if still_pending > 0: + new_replicas_needed = math.ceil(still_pending / new_tasks_per_replica) + return min([new_replicas_needed, args["max_replicas"]]) + else: + return 0 diff --git a/requirements/base.in b/requirements/base.in new file mode 100644 index 0000000..337ba88 --- /dev/null +++ b/requirements/base.in @@ -0,0 +1,5 @@ +click +kubernetes +taskcluster +pyyaml +structlog diff --git a/requirements/base.txt b/requirements/base.txt new file mode 100644 index 0000000..81da9af --- /dev/null +++ b/requirements/base.txt @@ -0,0 +1,181 @@ +# SHA1:133859141b0570bf860131c249f2098eda5c8f4c +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +aiohttp==3.5.4 \ + --hash=sha256:00d198585474299c9c3b4f1d5de1a576cc230d562abc5e4a0e81d71a20a6ca55 \ + --hash=sha256:0155af66de8c21b8dba4992aaeeabf55503caefae00067a3b1139f86d0ec50ed \ + --hash=sha256:09654a9eca62d1bd6d64aa44db2498f60a5c1e0ac4750953fdd79d5c88955e10 \ + --hash=sha256:199f1d106e2b44b6dacdf6f9245493c7d716b01d0b7fbe1959318ba4dc64d1f5 \ + --hash=sha256:296f30dedc9f4b9e7a301e5cc963012264112d78a1d3094cd83ef148fdf33ca1 \ + --hash=sha256:368ed312550bd663ce84dc4b032a962fcb3c7cae099dbbd48663afc305e3b939 \ + --hash=sha256:40d7ea570b88db017c51392349cf99b7aefaaddd19d2c78368aeb0bddde9d390 \ + --hash=sha256:629102a193162e37102c50713e2e31dc9a2fe7ac5e481da83e5bb3c0cee700aa \ + --hash=sha256:6d5ec9b8948c3d957e75ea14d41e9330e1ac3fed24ec53766c780f82805140dc \ + --hash=sha256:87331d1d6810214085a50749160196391a712a13336cd02ce1c3ea3d05bcf8d5 \ + --hash=sha256:9a02a04bbe581c8605ac423ba3a74999ec9d8bce7ae37977a3d38680f5780b6d \ + --hash=sha256:9c4c83f4fa1938377da32bc2d59379025ceeee8e24b89f72fcbccd8ca22dc9bf \ + --hash=sha256:9cddaff94c0135ee627213ac6ca6d05724bfe6e7a356e5e09ec57bd3249510f6 \ + --hash=sha256:a25237abf327530d9561ef751eef9511ab56fd9431023ca6f4803f1994104d72 \ + --hash=sha256:a5cbd7157b0e383738b8e29d6e556fde8726823dae0e348952a61742b21aeb12 \ + --hash=sha256:a97a516e02b726e089cffcde2eea0d3258450389bbac48cbe89e0f0b6e7b0366 \ + --hash=sha256:acc89b29b5f4e2332d65cd1b7d10c609a75b88ef8925d487a611ca788432dfa4 \ + --hash=sha256:b05bd85cc99b06740aad3629c2585bda7b83bd86e080b44ba47faf905fdf1300 \ + --hash=sha256:c2bec436a2b5dafe5eaeb297c03711074d46b6eb236d002c13c42f25c4a8ce9d \ + --hash=sha256:cc619d974c8c11fe84527e4b5e1c07238799a8c29ea1c1285149170524ba9303 \ + --hash=sha256:d4392defd4648badaa42b3e101080ae3313e8f4787cb517efd3f5b8157eaefd6 \ + --hash=sha256:e1c3c582ee11af7f63a34a46f0448fca58e59889396ffdae1f482085061a2889 \ + # via taskcluster +async-timeout==3.0.1 \ + --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \ + --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3 \ + # via aiohttp, taskcluster +attrs==19.1.0 \ + --hash=sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79 \ + --hash=sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399 \ + # via aiohttp +cachetools==3.1.0 \ + --hash=sha256:219b7dc6024195b6f2bc3d3f884d1fef458745cd323b04165378622dcc823852 \ + --hash=sha256:9efcc9fab3b49ab833475702b55edd5ae07af1af7a4c627678980b45e459c460 \ + # via google-auth +certifi==2019.3.9 \ + --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \ + --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae \ + # via kubernetes, requests +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ + # via aiohttp, requests +click==7.0 \ + --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ + --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 +google-auth==1.6.3 \ + --hash=sha256:0f7c6a64927d34c1a474da92cfc59e552a5d3b940d3266606c6a28b72888b9e4 \ + --hash=sha256:20705f6803fd2c4d1cc2dcb0df09d4dfcb9a7d51fd59e94a3a28231fd93119ed \ + # via kubernetes +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ + # via requests, yarl +kubernetes==9.0.0 \ + --hash=sha256:a8b0aed55ba946faea660712595a52ae53a8854df773d96f47a63fa0c9d4e3bf \ + --hash=sha256:f56137a298cb1453dd908b49dd4169347287c971e8cabd11b32f27570fec314c +mohawk==1.0.0 \ + --hash=sha256:aa57e6626a6ea323ab714779f23734de1d1feca8cb6fc00b65e65ce115c1696a \ + --hash=sha256:fca4e34d8f5492f1c33141c98b96e168a089e5692ce65fb747e4bb613f5fe552 \ + # via taskcluster +multidict==4.5.2 \ + --hash=sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f \ + --hash=sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3 \ + --hash=sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef \ + --hash=sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b \ + --hash=sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73 \ + --hash=sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc \ + --hash=sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3 \ + --hash=sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd \ + --hash=sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351 \ + --hash=sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941 \ + --hash=sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d \ + --hash=sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1 \ + --hash=sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b \ + --hash=sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a \ + --hash=sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3 \ + --hash=sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7 \ + --hash=sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0 \ + --hash=sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0 \ + --hash=sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014 \ + --hash=sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5 \ + --hash=sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036 \ + --hash=sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d \ + --hash=sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a \ + --hash=sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce \ + --hash=sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1 \ + --hash=sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a \ + --hash=sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9 \ + --hash=sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7 \ + --hash=sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b \ + # via aiohttp, yarl +oauthlib==3.0.1 \ + --hash=sha256:0ce32c5d989a1827e3f1148f98b9085ed2370fc939bf524c9c851d8714797298 \ + --hash=sha256:3e1e14f6cde7e5475128d30e97edc3bfb4dc857cb884d8714ec161fdbb3b358e \ + # via requests-oauthlib +pyasn1-modules==0.2.5 \ + --hash=sha256:ef721f68f7951fab9b0404d42590f479e30d9005daccb1699b0a51bb4177db96 \ + --hash=sha256:f309b6c94724aeaf7ca583feb1cc70430e10d7551de5e36edfc1ae6909bcfb3c \ + # via google-auth +pyasn1==0.4.5 \ + --hash=sha256:da2420fe13a9452d8ae97a0e478adde1dee153b11ba832a95b223a2ba01c10f7 \ + --hash=sha256:da6b43a8c9ae93bc80e2739efb38cc776ba74a886e3e9318d65fe81a8b8a2c6e \ + # via pyasn1-modules, rsa +python-dateutil==2.8.0 \ + --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \ + --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \ + # via kubernetes +pyyaml==5.1 \ + --hash=sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c \ + --hash=sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95 \ + --hash=sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2 \ + --hash=sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4 \ + --hash=sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad \ + --hash=sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba \ + --hash=sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1 \ + --hash=sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e \ + --hash=sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673 \ + --hash=sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13 \ + --hash=sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19 +requests-oauthlib==1.2.0 \ + --hash=sha256:bd6533330e8748e94bf0b214775fed487d309b8b8fe823dc45641ebcd9a32f57 \ + --hash=sha256:d3ed0c8f2e3bbc6b344fa63d6f933745ab394469da38db16bdddb461c7e25140 \ + # via kubernetes +requests==2.22.0 \ + --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \ + --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 \ + # via kubernetes, requests-oauthlib, taskcluster +rsa==4.0 \ + --hash=sha256:14ba45700ff1ec9eeb206a2ce76b32814958a98e372006c8fb76ba820211be66 \ + --hash=sha256:1a836406405730121ae9823e19c6e806c62bbad73f890574fff50efa4122c487 \ + # via google-auth +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ + # via google-auth, kubernetes, mohawk, python-dateutil, structlog, taskcluster, websocket-client +slugid==2.0.0 \ + --hash=sha256:a950d98b72691178bdd4d6c52743c4a2aa039207cf7a97d71060a111ff9ba297 \ + --hash=sha256:aec8b0e01c4ad32e38e12d609eab3ec912fd129aaf6b2ded0199b56a5f8fd67c \ + # via taskcluster +structlog==19.1.0 \ + --hash=sha256:5feae03167620824d3ae3e8915ea8589fc28d1ad6f3edf3cc90ed7c7cb33fab5 \ + --hash=sha256:db441b81c65b0f104a7ce5d86c5432be099956b98b8a2c8be0b3fb3a7a0b1536 +taskcluster-urls==11.0.0 \ + --hash=sha256:18dcaa9c2412d34ff6c78faca33f0dd8f2384e3f00a98d5832c62d6d664741f0 \ + --hash=sha256:2aceab7cf5b1948bc197f2e5e50c371aa48181ccd490b8bada00f1e3baf0c5cc \ + --hash=sha256:74bd2110b5daaebcec5e1d287bf137b61cb8cf6b2d8f5f2b74183e32bc4e7c87 \ + # via taskcluster +taskcluster==7.0.1 \ + --hash=sha256:93027ec6949289d8267595c5770c3d3f0902d9461d98081544dce60764f94f46 \ + --hash=sha256:97e19674738515e6891f4d7928280aecb2686def8b9d72a15477d7297f34c18c \ + --hash=sha256:c24a8a67db9306e5ff0fd140e5bb6489ba3dcaf0632b6a2052b2d649a30e7617 +urllib3==1.25.2 \ + --hash=sha256:a53063d8b9210a7bdec15e7b272776b9d42b2fd6816401a0d43006ad2f9902db \ + --hash=sha256:d363e3607d8de0c220d31950a8f38b18d5ba7c0830facd71a1c6b1036b7ce06c \ + # via kubernetes, requests +websocket-client==0.56.0 \ + --hash=sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9 \ + --hash=sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a \ + # via kubernetes +yarl==1.3.0 \ + --hash=sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9 \ + --hash=sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f \ + --hash=sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb \ + --hash=sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320 \ + --hash=sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842 \ + --hash=sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0 \ + --hash=sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829 \ + --hash=sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310 \ + --hash=sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4 \ + --hash=sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8 \ + --hash=sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1 \ + # via aiohttp diff --git a/requirements/test.in b/requirements/test.in new file mode 100644 index 0000000..da85881 --- /dev/null +++ b/requirements/test.in @@ -0,0 +1,4 @@ +-r base.in + +pytest +pytest-coverage diff --git a/requirements/test.txt b/requirements/test.txt new file mode 100644 index 0000000..0cbc53c --- /dev/null +++ b/requirements/test.txt @@ -0,0 +1,75 @@ +# SHA1:130fc19af7d15836ea1559a85d8216832b029187 +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +-r base.txt +atomicwrites==1.3.0 \ + --hash=sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4 \ + --hash=sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6 \ + # via pytest +coverage==4.5.3 \ + --hash=sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9 \ + --hash=sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74 \ + --hash=sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390 \ + --hash=sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8 \ + --hash=sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe \ + --hash=sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf \ + --hash=sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e \ + --hash=sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741 \ + --hash=sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09 \ + --hash=sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd \ + --hash=sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034 \ + --hash=sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420 \ + --hash=sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c \ + --hash=sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab \ + --hash=sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba \ + --hash=sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e \ + --hash=sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609 \ + --hash=sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2 \ + --hash=sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49 \ + --hash=sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b \ + --hash=sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d \ + --hash=sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce \ + --hash=sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9 \ + --hash=sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4 \ + --hash=sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773 \ + --hash=sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723 \ + --hash=sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c \ + --hash=sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f \ + --hash=sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1 \ + --hash=sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260 \ + --hash=sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a \ + # via pytest-cov +more-itertools==7.0.0 \ + --hash=sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7 \ + --hash=sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a \ + # via pytest +pluggy==0.11.0 \ + --hash=sha256:25a1bc1d148c9a640211872b4ff859878d422bccb59c9965e04eed468a0aa180 \ + --hash=sha256:964cedd2b27c492fbf0b7f58b3284a09cf7f99b0f715941fb24a439b3af1bd1a \ + # via pytest +py==1.8.0 \ + --hash=sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa \ + --hash=sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53 \ + # via pytest +pytest-cov==2.7.1 \ + --hash=sha256:2b097cde81a302e1047331b48cadacf23577e431b61e9c6f49a1170bbe3d3da6 \ + --hash=sha256:e00ea4fdde970725482f1f35630d12f074e121a23801aabf2ae154ec6bdd343a \ + # via pytest-cover +pytest-cover==3.0.0 \ + --hash=sha256:578249955eb3b5f3991209df6e532bb770b647743b7392d3d97698dc02f39ebb \ + --hash=sha256:5bdb6c1cc3dd75583bb7bc2c57f5e1034a1bfcb79d27c71aceb0b16af981dbf4 \ + # via pytest-coverage +pytest-coverage==0.0 \ + --hash=sha256:db6af2cbd7e458c7c9fd2b4207cee75258243c8a81cad31a7ee8cfad5be93c05 \ + --hash=sha256:dedd084c5e74d8e669355325916dc011539b190355021b037242514dee546368 +pytest==4.5.0 \ + --hash=sha256:1a8aa4fa958f8f451ac5441f3ac130d9fc86ea38780dd2715e6d5c5882700b24 \ + --hash=sha256:b8bf138592384bd4e87338cb0f256bf5f615398a649d4bd83915f0e4047a5ca6 +wcwidth==0.1.7 \ + --hash=sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e \ + --hash=sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c \ + # via pytest diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..8419de4 --- /dev/null +++ b/setup.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""The setup script.""" + +from glob import glob +from os.path import basename, splitext + +from setuptools import find_packages, setup + +with open("README.rst") as readme_file: + readme = readme_file.read() + +with open("version.txt") as f: + version = f.read().rstrip() + +with open("requirements/base.in") as f: + install_requires = f.readlines() + +setup( + author="Rail Aliiev", + author_email="rail@mozilla.com", + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)" + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + ], + description="Autoscale scriptworkers in Kubernetes", + install_requires=install_requires, + license="MPL2.0", + long_description=readme, + include_package_data=True, + keywords="k8s-autoscale", + name="k8s_autoscale", + packages=find_packages(), + test_suite="tests", + url="https://github.com/mozilla-releng/k8s-autoscale", + version=version, + zip_safe=False, + entry_points={"console_scripts": ["k8s_autoscale=k8s_autoscale.cli:main"]}, +) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_sla.py b/tests/test_sla.py new file mode 100644 index 0000000..fd9609d --- /dev/null +++ b/tests/test_sla.py @@ -0,0 +1,53 @@ +import pytest +from k8s_autoscale.sla import get_new_worker_count + +args = { + "max_replicas": 10, + "avg_task_duration": 60, + "sla_seconds": 300, + "boot_time": 30, + "capacity_ratio": 1.0, +} +args_capacity = args.copy() +args_capacity["capacity_ratio"] = 0.5 + + +@pytest.mark.parametrize( + "pending, running, booting, args, expected", + [ + (0, 0, 0, args, 0), + (1, 0, 0, args, 1), + (10000, 0, 0, args, 10), + (0, 10, 0, args, -10), + (10, 20, 0, args, 0), + (30, 0, 0, args, 8), + (30, 2, 0, args, 6), + (30, 5, 0, args, 3), + (30, 6, 0, args, 2), + (30, 7, 0, args, 1), + (30, 8, 0, args, 0), + (30, 8, 2, args, 0), + (30, 8, 10, args, 0), + (30, 0, 10, args, 0), + (30, 5, 10, args, 0), + (30, 2, 2, args, 4), + (0, 2, 2, args, -4), + (0, 0, 2, args, -2), + (30, 2, 2, args_capacity, 0), + (30, 2, 0, args_capacity, 2), + (30, 0, 2, args_capacity, 2), + (0, 0, 2, args_capacity, -2), + (0, 4, 2, args_capacity, -6), + ], +) +def test_process(pending, running, booting, args, expected): + assert get_new_worker_count(pending, running, booting, args) == expected + + +@pytest.mark.parametrize( + "pending, running, booting, args, exception_type", + [(0, 0, 0, {"sla_seconds": 10, "avg_task_duration": 20}, AssertionError)], +) +def test_process_raises(pending, running, booting, args, exception_type): + with pytest.raises(exception_type): + get_new_worker_count(pending, running, booting, args) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..dc967f6 --- /dev/null +++ b/tox.ini @@ -0,0 +1,45 @@ +[tox] +envlist = clean, py37, report, check + +[travis] +python = + 3.7: py37 + +[testenv] +depends = clean +setenv = + PYTHONPATH = {toxinidir}/tests +usedevelop = true +deps = + -r {toxinidir}/requirements/test.txt +commands = + {posargs:py.test --cov-config=tox.ini --cov-append --cov=k8s_autoscale --cov-report term-missing tests} + +[testenv:clean] +skip_install = true +deps = coverage +commands = coverage erase +depends = + +[testenv:report] +skip_install = true +deps = coverage +commands = coverage report -m +depends = py37 +parallel_show_output = true + +[testenv:check] +skip_install = true +deps = + black + check-manifest + pip-compile-multi + isort +commands = + black --diff --check {toxinidir} + isort --check -rc -df {toxinidir} + pip-compile-multi verify + check-manifest -u -v {toxinidir} + +[coverage:run] +branch = true diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +0.1.0