Skip to content

Commit

Permalink
Initial import (#1)
Browse files Browse the repository at this point in the history
* Initial import

* Use docker save/load/push
  • Loading branch information
Rail Aliiev authored May 25, 2019
1 parent fb31d73 commit 81b4454
Show file tree
Hide file tree
Showing 30 changed files with 964 additions and 1 deletion.
8 changes: 8 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
.coverage
.tox/
htmlcov/
.mypy_cache
*egg-info
**/__pycache__/
*.py[cod]

77 changes: 77 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]

# C extensions
*.so

# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.pytest_cache/

# Translations
*.mo
*.pot

# Django stuff:
*.log

# Sphinx documentation
docs/_build/

# PyBuilder
target/

# vim
*.swp

# pycharm
.idea

# testing
config.json
server_config.json
script_config.json
work_dir
artifact_dir
build-tools

beetmoverscript/test/test_artifact_dir/

# misc
.DS_Store
128 changes: 128 additions & 0 deletions .taskcluster.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
version: 1
policy:
pullRequests: public
tasks:
$let:
head_rev:
$if: 'tasks_for == "github-pull-request"'
then: ${event.pull_request.head.sha}
else:
$if: 'tasks_for == "github-push"'
then: ${event.after}
else: ${event.release.tag_name}

repository:
$if: 'tasks_for == "github-pull-request"'
then: ${event.pull_request.head.repo.html_url}
else: ${event.repository.html_url}

owner: ${event.sender.login}@users.noreply.github.com

docker_tag:
$if: 'tasks_for == "github-pull-request"'
then: pull-request
else:
$if: 'tasks_for == "github-push"'
then:
$if: 'event.ref[:10] == "refs/tags/"'
then: ${event.ref[10:]} # strip "refs/tags/"
else: ${event.ref[11:]} # strip "refs/heads/"
else: "v${event.release.tag_name}" # prefix the version with "v"

in:
- taskId: '${as_slugid("py37")}'
provisionerId: aws-provisioner-v1
workerType: github-worker
created: {$fromNow: ''}
deadline: {$fromNow: '4 hours'}
payload:
maxRunTime: 3600
image: python:3.7
command:
- sh
- -lxce
- >-
git clone ${repository} /src &&
cd /src &&
git config advice.detachedHead false &&
git checkout ${head_rev} &&
pip install tox &&
tox
metadata:
name: tox py37 (${docker_tag})
description: code linting & unit tests on py37 (${docker_tag})
owner: ${owner}
source: ${repository}/raw/${head_rev}/.taskcluster.yml

- taskId: '${as_slugid("docker_build")}'
dependencies:
- '${as_slugid("py37")}'
provisionerId: aws-provisioner-v1
workerType: github-worker
created: {$fromNow: ''}
deadline: {$fromNow: '4 hours'}
payload:
features:
dind: true
maxRunTime: 3600
image: mozillareleases/python-test-runner
command:
- bash
- -ce
- >-
git clone ${repository} /src &&
cd /src &&
git config advice.detachedHead false &&
git checkout ${head_rev} &&
docker build -f Dockerfile -t mozilla/releng-k8s-autoscale:${docker_tag} . &&
docker save mozilla/releng-k8s-autoscale:${docker_tag} > /tmp/image.tar
artifacts:
public/image.tar:
expires: {$fromNow: '2 weeks'}
path: /tmp/image.tar
type: file
metadata:
name: docker build (${docker_tag})
description: build latest docker image (${docker_tag})
owner: ${owner}
source: ${repository}/raw/${head_rev}/.taskcluster.yml

- $if: 'tasks_for != "github-pull-request"'
then:
$if: 'repository == "https://github.com/mozilla-releng/k8s-autoscale"'
then:
taskId: '${as_slugid("docker_push")}'
dependencies:
- '${as_slugid("docker_build")}'
provisionerId: aws-provisioner-v1
workerType: github-worker
created: {$fromNow: ''}
deadline: {$fromNow: '4 hours'}
payload:
features:
# Needed for access to secret
taskclusterProxy: true
dind: true
maxRunTime: 3600
image: mozillareleases/python-test-runner
command:
- bash
- -ce
- >-
git clone ${repository} /src &&
cd /src &&
git config advice.detachedHead false &&
git checkout ${head_rev} &&
IMAGE_TASK_ID=${as_slugid("docker_build")}
SECRET_URL=http://taskcluster/secrets/v1/secret/project/releng/k8s-autoscale/deploy
[email protected]
DOCKERHUB_USER=mozillarelengservices
TAG=mozilla/releng-k8s-autoscale:${docker_tag}
./docker.d/push_image.sh
scopes:
- secrets:get:project/releng/k8s-autoscale/deploy
metadata:
name: docker push (${docker_tag})
description: push docker image (${docker_tag})
owner: ${owner}
source: ${repository}/raw/${head_rev}/.taskcluster.yml
15 changes: 15 additions & 0 deletions CODE_OF_CONDUCT.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Community Participation Guidelines

This repository is governed by Mozilla's code of conduct and etiquette guidelines.
For more details, please read the
[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/).

## How to Report
For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page.

<!--
## Project Specific Etiquette
In some cases, there will be additional project etiquette i.e.: (https://bugzilla.mozilla.org/page.cgi?id=etiquette.html).
Please update for your project.
-->
20 changes: 20 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
FROM python:3.7

RUN groupadd --gid 10001 app && \
useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app

USER app
WORKDIR /app

COPY . /app
# generate /app/version.json according to https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md
RUN ./docker.d/generate_version_json.sh

RUN python -m venv /app
RUN ./bin/pip install -r requirements/base.txt
RUN ./bin/pip install -e .

COPY docker.d/healthcheck /bin/healthcheck
COPY docker.d/init.sh /app/bin/init.sh

CMD ["/app/bin/init.sh"]
8 changes: 8 additions & 0 deletions HISTORY.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
=======
History
=======

0.1.0 (2019-05-07)
------------------

* First release
27 changes: 27 additions & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
include CONTRIBUTING.rst
include HISTORY.rst
include LICENSE
include README.rst
include CODE_OF_CONDUCT.md
include tox.ini
include config-example.yaml
include version.txt

recursive-include requirements *.in
recursive-include requirements *.txt
recursive-include tests *
recursive-exclude * __pycache__
recursive-exclude * *.py[co]

recursive-include docs *.rst conf.py

# added by check_manifest.py
include *.txt

# added by check_manifest.py
include Dockerfile
recursive-include configs *.yml
recursive-include docker.d *

exclude .dockerignore
exclude .taskcluster.yml
1 change: 0 additions & 1 deletion README.md

This file was deleted.

16 changes: 16 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
====================
Kubernetes Autoscale
====================

Autoscale scriptworkers in Kubernetes

* Free software: MPL2
* Documentation: https://k8s-autoscale.readthedocs.io.

Features
--------

* TODO

Credits
-------
18 changes: 18 additions & 0 deletions config-example.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
worker_types:
- name: shipitscript-v1
provisioner: scriptworker-prov-v1
poll_interval: 180
deployment_namespace: default
deployment_name: nginx-1
# remove kube_config and kube_connfig_context to use in-cluster auth
kube_connfig: ~/.kube/config
kube_connfig_context: xx
autoscale:
algorithm: sla
args:
max_replicas: 20
avg_task_duration: 60
boot_time: 60
tolerance_seconds: 300
# cover 100% of pending
capacity_ratio: 1.0
15 changes: 15 additions & 0 deletions configs/production-beetmover.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
worker_types:
- name: beetmoverworker-v1
provisioner: scriptworker-prov-v1
poll_interval: 180
deployment_namespace: TODO
deployment_name: TODO
autoscale:
algorithm: sla
args:
max_replicas: 20
avg_task_duration: 60
boot_time: 60
tolerance_seconds: 300
# cover 100% of pending
capacity_ratio: 1.0
15 changes: 15 additions & 0 deletions configs/stage-beetmover.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
worker_types:
- name: beetmoverworker-dev
provisioner: scriptworker-prov-v1
poll_interval: 180
deployment_namespace: TODO
deployment_name: TODO
autoscale:
algorithm: sla
args:
max_replicas: 2
avg_task_duration: 60
boot_time: 60
tolerance_seconds: 300
# cover 100% of pending
capacity_ratio: 1.0
14 changes: 14 additions & 0 deletions docker.d/generate_version_json.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#/bin/bash

set -xe
commit=$(git rev-parse HEAD)
version=$(cat version.txt)

cat > version.json <<EOF
{
"commit": "${commit}",
"version": "${version}",
"source": "https://github.com/mozilla-releng/k8s-autoscale",
"build": "https://github.com/mozilla-releng/k8s-autoscale/commit/${commit}"
}
EOF
2 changes: 2 additions & 0 deletions docker.d/healthcheck
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#!/bin/bash
pgrep k8s_autoscale
5 changes: 5 additions & 0 deletions docker.d/init.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash
set -e
test $CONFIG

exec /app/bin/k8s_autoscale --config /app/configs/$CONFIG
Loading

0 comments on commit 81b4454

Please sign in to comment.