PB-363 💎 Release v0.4.0

.github/workflows/development.yml → .github/workflows/development_pull_request.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - development
+
 jobs:
   development:
     runs-on: ubuntu-18.04
@@ -30,11 +31,11 @@ jobs:
 
       - name: black
         run: |
-          black --line-length 100 --check setup.py xain_fl tests
+          black --check setup.py xain_fl tests
 
       - name: isort
         run: |
-          isort --check-only --indent=4 -rc setup.py xain_fl tests
+          isort --check-only --settings-path=.isort.cfg -rc setup.py xain_fl tests
 
       - name: pylint
         run: |

.github/workflows/master.yml → .github/workflows/master_pull_request.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - master
+
 jobs:
   main:
     runs-on: ubuntu-18.04
@@ -30,11 +31,11 @@ jobs:
 
       - name: black
         run: |
-          black --line-length 100 --check setup.py xain_fl tests
+          black --check setup.py xain_fl tests
 
       - name: isort
         run: |
-          isort --check-only --indent=4 -rc setup.py xain_fl tests
+          isort --check-only --settings-path=.isort.cfg -rc setup.py xain_fl tests
 
       - name: pylint
         run: |
@@ -55,3 +56,6 @@ jobs:
       - name: PyTest
         run: |
           pytest -v
+
+      # TODO: Add E2E tests here!!!
+

.github/workflows/master_tag.yml

@@ -0,0 +1,44 @@
+name: Package release
+
+on:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+
+jobs:
+  main:
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python 3.6
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.6
+
+      - name: Cache pip packages
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: pip-master-${{ hashFiles('setup.py') }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools
+          pip install wheel twine
+
+      - name: Create packages
+        run: |
+          python setup.py sdist bdist_wheel
+
+      - name: Output tag name
+        id: get_tag_name
+        run: echo ::set-output name=TAG_NAME::${GITHUB_REF/refs\/tags\//}
+
+      - name: Upload packages
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ steps.get_tag_name.outputs.TAG_NAME }}-release
+          path: dist

.gitignore

@@ -129,3 +129,7 @@ dmypy.json
 # Editors
 .idea
 .vscode
+
+# In the README, we document how to use custom config. Let's ignore
+# the files mentioned in that section
+custom_config.toml

.isort.cfg

@@ -1,8 +1,10 @@
 [settings]
+combine_as_imports=True
 force_grid_wrap=0
 force_sort_within_sections=True
 include_trailing_comma=True
+indent=4
 known_third_party=xain_proto,xain_sdk
-line_length=100
+line_length=88
 multi_line_output=3
 use_parentheses=True

CHANGELOG.md

@@ -20,6 +20,38 @@ For reference, the possible headings are:
 
 ## [Unreleased]
 
+## [0.4.0] - 2020-02-04
+
+Flatten model weights instead of using lists.
+Fix minor issues, update documentation.
+
+- PB-116: pin docutils version (#259) [Corentin Henry]
+- PB-119 update isort config and calls (#260) [janpetschexain]
+- PB-351 Store participant metrics (#244) [Robert Steiner]
+- Adjust isort config (#258) [Robert Steiner]
+- PB-366 flatten weights (#253) [janpetschexain]
+- PB-379 Update black setup (#255) [Anastasiia Tymoshchuk]
+- PB-387 simplify serve module (#251) [Corentin Henry]
+- PB-104: make the tests fast again (#252) [Corentin Henry]
+- PB-122: handle sigint properly (#250) [Corentin Henry]
+- PB-383 write aggregated weights after each round (#246) [Corentin Henry]
+- PB-104: Fix exception in monitor_hearbeats() (#248) [Corentin Henry]
+- DO-57 Update docker-compose files for provisioning InfluxDB (#249) [Ricardo Saffi Marques]
+- DO-59 Provision Redis 5.x for persisting states for the Coordinator (#247) [Ricardo Saffi Marques]
+- PB-381: make the log level configurable (#243) [Corentin Henry]
+- PB-382: cleanup storage (#245) [Corentin Henry]
+- PB-380: split get_logger() (#242) [Corentin Henry]
+- XP-332: grpc resource exhausted (#238) [Robert Steiner]
+- XP-456: fix coordinator command (#241) [Corentin Henry]
+- XP-485 Document revised state machine (#240) [kwok]
+- XP-456: replace CLI argument with a config file (#221) [Corentin Henry]
+- DO-48 :snowflake: :rocket: Build stable package on git tag with SemVer (#234) [Daniel Kravetz]
+- XP-407 update documentation (#239) [janpetschexain]
+- XP-406 remove numpy file cli (#237) [janpetschexain]
+- XP-544 fix aggregate module (#235) [janpetschexain]
+- DO-58: cache xain-fl dependencies in Docker (#232) [Corentin Henry]
+- XP-479 Start training rounds from 0 (#226) [kwok]
+
 ## [0.3.0] - 2020-01-21
 
 - XP-505 cleanup docstrings in xain_fl.coordinator (#228)

Dockerfile

@@ -4,6 +4,7 @@ ENV USER="xain"
 ENV HOST="0.0.0.0"
 ENV PORT="50051"
 ENV PATH="/home/${USER}/.local/bin:${PATH}"
+ENV CONFIG_FILE="/app/xain-fl.toml"
 
 RUN addgroup -S ${USER} && adduser -S ${USER} -G ${USER}
 RUN apk update && apk add python3-dev build-base git
@@ -19,10 +20,11 @@ RUN pip install -v .
 # Remove everything, including dot files
 RUN rm -rf ..?* .[!.]* *
 
+COPY configs/xain-fl.toml ${CONFIG_FILE}
+
 # Drop down to a non-root user
 USER ${USER}
 
-COPY --chown=${USER}:${USER} test_array.npy test_array.npy
 COPY --chown=${USER}:${USER} docker/entrypoint.sh entrypoint.sh
 RUN chmod +x entrypoint.sh
 

Dockerfile.dev

@@ -3,10 +3,27 @@ FROM python:3.6-alpine
 RUN apk update && apk add python3-dev build-base git
 
 WORKDIR /app
+
+# Some dependencies require a very long compilation: protobuf, numpy,
+# grpcio. To avoid having to re-install these packages every time, we
+# pre-install them so that they are cached.
+#
+# However, we cannot just pre-install a few packages, because we don't
+# know exactly which version `pip` will pick for them. Instead, we
+# give `pip` all the package's dependencies, and let it resolve and
+# install them.
 COPY setup.py .
-COPY xain_fl xain_fl/
+RUN mkdir xain_fl && \
+    printf '__version__ = "0"\n__short_version__ = "0"' > xain_fl/__version__.py && \
+    touch README.md && \
+    python setup.py egg_info && \
+    cat *.egg-info/requires.txt | grep -v '^\[' | uniq | pip install -r /dev/stdin
+
+RUN rm -rf xain_fl README.md
 COPY README.md .
+COPY xain_fl xain_fl/
+RUN pip install -e .
 
-RUN pip install -v -e .
+COPY configs/xain-fl.toml /app/xain-fl.toml
 
-CMD ["python3", "setup.py", "--fullname"]
+CMD ["coordinator", "--config", "/app/xain-fl.toml"]

README.md

@@ -68,16 +68,17 @@ $ make show
 
 ### Running the Coordinator locally
 
-To run the Coordinator on your local machine, use the command:
+To run the Coordinator on your local machine, you can use the
+`example-config.toml` file:
 
 ```shell
-$ python xain_fl/cli.py -f test_array.npy
-```
-
-For more information about the CLI and its arguments, run:
+# If you have installed the xain_fl package,
+# the `coordinator` command should be directly available
+coordinator --config configs/example-config.toml
 
-```shell
-$ python xain_fl/cli.py --help
+# otherwise the coordinator can be started by executing the
+# `xain_fl` package:
+python xain_fl --config configs/example-config.toml
 ```
 
 ### Run the Coordinator from a Docker image
@@ -92,13 +93,28 @@ To run the coordinator's development image, first build the Docker image:
 $ docker build -t xain-fl-dev -f Dockerfile.dev .
 ```
 
-Then run the image, mounting the directory as a Docker volume, and call the
-entrypoint:
+Then run the image, mounting the directory as a Docker volume:
 
 ```shell
 $ docker run -v $(pwd):/app -v '/app/xain_fl.egg-info' xain-fl-dev coordinator
 ```
 
+The command above uses a default configuration but you can also use a
+custom config file:
+
+For instance, if you have a `./custom_config.toml` file that you'd
+like to use, you can mount it in the container and run the coordinator
+with:
+
+```shell
+docker run \
+  -v $(pwd)/custom_config.toml:/custom_config.toml \
+  -v $(pwd):/app \
+  -v '/app/xain_fl.egg-info' \
+  xain-fl-dev \
+  coordinator --config /custom_config.toml
+```
+
 #### Release image
 
 To run the coordinator's release image, first build it:
@@ -115,10 +131,23 @@ $ docker run -p 50051:50051 xain-fl
 
 ### Docker-compose
 
+The coordinator needs a storage service that provides an AWS S3
+API. For development, we use `minio`. We provide `docker-compose`
+files that start coordinator container along with a `minio` container,
+and pre-populate the appropriate storage buckets.
+
 #### Development
 
+To start both the coordinator and the `minio` service use:
+
+```shell
+docker-compose -f docker-compose-dev.yml up
+```
+
+It is also possible to only start the storage service:
+
 ```shell
-$ docker-compose -f docker-compose-dev.yml up
+docker-compose -f docker-compose-dev.yml up minio-dev initial-buckets
 ```
 
 #### Release

configs/example-config.toml

@@ -0,0 +1,63 @@
+# An example configfiguration file. 
+# The values that are used for the optional keys correspond to their default values.
+
+# (Optional)
+[server]
+# (Optional) Address to listen on for incoming gRPC connections
+host = "localhost"
+# (Optional) Port to listen on for incoming gRPC connections
+port = 50051
+
+# (Optional)
+[server.grpc_options]
+# More information about all available gRPC options can be found here: 
+# https://grpc.github.io/grpc/cpp/group__grpc__arg__keys.html#ga813f94f9ac3174571dd712c96cdbbdc1
+
+# Example
+# Maximum message length that the channel can receive. -1 means unlimited. 
+# "grpc.max_receive_message_length" = -1
+
+# (Required)
+[ai]
+# (Required) Number of global rounds the model is going to be trained for. This
+# must be a positive integer.
+rounds = 2
+# (Required) Number of local epochs per round
+epochs = 1
+# (Optional) Minimum number of participants to be selected for a round.
+min_participants = 1
+# (Optional) Fraction of total clients that participate in a training round. This
+# must be a float between 0 and 1.
+fraction_participants = 1.0
+
+# (Required)
+[storage]
+# (Required) URL to the storage service to use
+endpoint = "http://minio-dev:9000"
+# (Required) Name of the bucket for storing the aggregated models
+bucket = "xain-fl-aggregated-weights"
+# (Required) AWS access key ID to use to authenticate to the storage service
+access_key_id = "minio"
+# (Required) AWS secret access to use to authenticate to the storage service
+secret_access_key = "minio123"
+
+# (Optional)
+[logging]
+# (Optional) The log level can be one of "notset", "debug", "info", "warning",
+# "error" and "critical".
+level = "info"
+
+# (Optional)
+[metrics]
+# (Optional) If `true`, store metrics in InfluxDB, otherwise not.
+enable = false
+# (Optional) The hostname to connect to InfluxDB.
+host = "localhost"
+# (Optional) The port of InfluxDB.
+port = 8086
+# (Optional) The name of the InfluxDB user.
+user = "root"
+# (Optional) The password of the InfluxDB user.
+password = "root"
+# (Optional) The name of the database.
+db_name = "metrics"