first try use mamba instead of conda

eth-easl · Sep 18, 2023 · ce29d01 · ce29d01
1 parent 0c1aa65
commit ce29d01
Show file tree

Hide file tree

Showing 13 changed files with 98 additions and 94 deletions.
diff --git a/.github/actions/conda/action.yml b/.github/actions/conda/action.yml
@@ -6,61 +6,61 @@ runs:
         with:
           python-version: '3.10'
 
-      - name: Check if shared miniconda installation exists (we are on a Github Runner)
-        id: check_files
-        uses: andstor/file-existence-action@v2
-        with:
-          files: "/usr/share/miniconda/envs"
+#      - name: Check if shared miniconda installation exists (we are on a Github Runner)
+#        id: check_files
+#        uses: andstor/file-existence-action@v2
+#        with:
+#          files: "/usr/share/miniconda/envs"
 
-      - name: Set shared conda path
-        if: steps.check_files.outputs.files_exists == 'true'
-        run: echo "CONDAPATH=/usr/share/miniconda/envs" >> $GITHUB_ENV 
-        shell: bash
+#      - name: Set shared conda path
+#        if: steps.check_files.outputs.files_exists == 'true'
+#        run: echo "CONDAPATH=/usr/share/miniconda/envs" >> $GITHUB_ENV 
+#        shell: bash
 
-      - name: Miniconda exists, fix permissions
-        if: steps.check_files.outputs.files_exists == 'true'
-        run: sudo chown -R $(whoami):$(id -ng) ${CONDAPATH}
-        shell: bash
+#      - name: Miniconda exists, fix permissions
+#        if: steps.check_files.outputs.files_exists == 'true'
+#        run: sudo chown -R $(whoami):$(id -ng) ${CONDAPATH}
+#        shell: bash
 
-      - name: Setup miniconda if not exists
-        if: steps.check_files.outputs.files_exists == 'false'
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-update-conda: true
-          python-version: '3.10'
-          miniconda-version: "latest"
-
-      - name: Set custom conda path
-        if: steps.check_files.outputs.files_exists == 'false'
-        run: echo "CONDAPATH=${CONDA}" >> $GITHUB_ENV 
-        shell: bash
+#      - name: Set custom conda path
+#        if: steps.check_files.outputs.files_exists == 'false'
+#        run: echo "CONDAPATH=${CONDA}" >> $GITHUB_ENV 
+#        shell: bash
 
       - name: Set cache date
         run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
         shell: bash
 
       # increase to reset cache manually
       - name: Set cache number
-        run: echo "CACHE_NUMBER=8" >> $GITHUB_ENV 
+        run: echo "CACHE_NUMBER=0" >> $GITHUB_ENV 
         shell: bash
 
-      - uses: actions/cache@v3
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@v1
         with:
-          path: ${{ env.CONDAPATH }}
-          key: conda-${{ hashFiles('environment.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
-        id: cache
+          environment-file: environment.yml
+          cache-environment-key: environment-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
+          cache-downloads-key: downloads-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
 
-      - name: Setup requirements
-        run: |
-          conda update -n base -c defaults conda
-          conda update --all
-          conda env create -f ./environment.yml
-          conda run -n modyn pip install -e .
-          conda run -n modyn pip install -r dev-requirements.txt
-        if: steps.cache.outputs.cache-hit != 'true'
-        shell: bash
+#      - uses: actions/cache@v3
+#        with:
+#          path: ${{ env.CONDAPATH }}
+#          key: conda-${{ hashFiles('environment.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
+#        id: cache
+
+#      - name: Setup requirements
+#        run: |
+#          mamba update -n base -c defaults mamba
+#          mamba update --all
+#          mamba env create -f ./environment.yml
+#          mamba run -n modyn pip install -e .
+#          mamba run -n modyn pip install -r dev-requirements.txt
+#        if: steps.cache.outputs.cache-hit != 'true'
+#        shell: bash
 
-      - name: Reinstall modyn in case of cache
-        run: conda run -n modyn pip install -e .
-        if: steps.cache.outputs.cache-hit == 'true'
+      - name: Install dev requirements 
+        run: | 
+          mamba run -n modyn pip install -e .
+          mamba run -n modyn pip install -r dev-requirements.txt
         shell: bash
diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml
@@ -14,13 +14,13 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Setup Conda
-        uses: ./.github/actions/conda
+      - name: Setup mamba
+        uses: ./.github/actions/mamba
 
       - name: Flake8
         run: |
-          conda run -n modyn flake8 --version
-          conda run -n modyn flake8 modyn --statistics
+          mamba run -n modyn flake8 --version
+          mamba run -n modyn flake8 modyn --statistics
 
   mypy-typechecking:
     timeout-minutes: 20
@@ -30,13 +30,13 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Setup Conda
-        uses: ./.github/actions/conda
+      - name: Setup mamba
+        uses: ./.github/actions/mamba
 
       - name: Mypy
         run: |
-          conda run -n modyn mypy --version
-          conda run -n modyn mypy modyn
+          mamba run -n modyn mypy --version
+          mamba run -n modyn mypy modyn
 
   pylint:
     timeout-minutes: 20
@@ -46,13 +46,13 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Setup Conda
-        uses: ./.github/actions/conda
+      - name: Setup mamba
+        uses: ./.github/actions/mamba
 
       - name: Pylint
         run: |
-          conda run -n modyn pylint --version
-          conda run -n modyn pylint modyn
+          mamba run -n modyn pylint --version
+          mamba run -n modyn pylint modyn
 
   isort:
     timeout-minutes: 20
@@ -62,15 +62,15 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Setup Conda
-        uses: ./.github/actions/conda
+      - name: Setup mamba
+        uses: ./.github/actions/mamba
 
       - name: Isort
         run: |
-          conda run -n modyn isort --version
-          conda run -n modyn isort modyn --check --diff
-          conda run -n modyn isort integrationtests --check --diff
-          conda run -n modyn isort benchmark --check --diff
+          mamba run -n modyn isort --version
+          mamba run -n modyn isort modyn --check --diff
+          mamba run -n modyn isort integrationtests --check --diff
+          mamba run -n modyn isort benchmark --check --diff
 
   black:
     timeout-minutes: 20
@@ -80,13 +80,13 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Setup Conda
-        uses: ./.github/actions/conda
+      - name: Setup mamba
+        uses: ./.github/actions/mamba
 
       - name: Black
         run: |
-          conda run -n modyn black --version
-          conda run -n modyn black --check modyn --verbose --config black.toml
+          mamba run -n modyn black --version
+          mamba run -n modyn black --check modyn --verbose --config black.toml
 
   unittests:
     timeout-minutes: 20
@@ -96,13 +96,13 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Setup Conda
-        uses: ./.github/actions/conda
+      - name: Setup mamba
+        uses: ./.github/actions/mamba
 
       - name: Pytest
         run: |
-          conda run -n modyn pytest modyn --cov-reset --cache-clear --cov-fail-under=90
-          conda run -n modyn pytest > pytest-coverage.txt
+          mamba run -n modyn pytest modyn --cov-reset --cache-clear --cov-fail-under=90
+          mamba run -n modyn pytest > pytest-coverage.txt
 
       - name: Comment coverage
         uses: coroo/[email protected]
@@ -130,7 +130,7 @@ jobs:
       uses: ./.github/actions/base
 
     - name: Setup dev-requirements and run pytest within container
-      run: docker run modynbase conda run -n modyn bash -c "pip install -r dev-requirements.txt && echo Running pytest && pytest"
+      run: docker run modynbase mamba run -n modyn bash -c "pip install -r dev-requirements.txt && echo Running pytest && pytest"
 
 
 # Tests whether docker-compose up starts all components successfully and integration tests run through

diff --git a/README.md b/README.md
@@ -14,12 +14,17 @@ Modyn is an open-source platform for model training on dynamic datasets, i.e., d
 ## ⚡️ Quickstart
 
 For deploying and running integration tests, you will need [Docker](https://docs.docker.com/get-docker/).
-Furthermore, we use [conda](https://docs.conda.io/en/latest/) for local environments and [tmuxp](https://github.com/tmux-python/tmuxp) for easily managing components panes (optional).
+Furthermore, we use [mamba](https://mamba.readthedocs.io/en/latest/) for local environments and [tmuxp](https://github.com/tmux-python/tmuxp) for easily managing components panes (optional).
 For local development, run
 ```bash
+# In case you don't have mamba yet
+curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh"
+bash Mambaforge-$(uname)-$(uname -m).sh
+
+# Start here if you have Mamba already
 ./initial_setup.sh
-conda env create -f ./environment.yml
-conda activate modyn
+mamba env create -f ./environment.yml
+mamba activate modyn
 pip install -e .
 pip install -r dev-requirements.txt
 ```

diff --git a/docker/Base/Dockerfile b/docker/Base/Dockerfile
@@ -2,12 +2,12 @@ FROM modyndependencies
 
 # Copy source code into container
 ADD . /src
-RUN conda run -n modyn pip install -e /src
+RUN mamba run -n modyn pip install -e /src
 WORKDIR /src
 RUN chown -R appuser /src
 
 # Setup environment for user
 USER appuser
 ENV CONDA_DEFAULT_ENV modyn
-RUN /bin/bash -c "conda init"
-RUN echo "conda activate modyn" >> /home/appuser/.bashrc
+RUN /bin/bash -c "mamba init"
+RUN echo "mamba activate modyn" >> /home/appuser/.bashrc
diff --git a/docker/Dependencies/Dockerfile b/docker/Dependencies/Dockerfile
@@ -25,14 +25,13 @@ RUN apt-get update -yq \
 RUN adduser -u 5678 --disabled-password --gecos "" appuser
 ENV PATH="${PATH}:/home/appuser/.local/bin"
 
-# Install miniconda
-ENV CONDA_DIR /opt/conda
-RUN if [ "$(dpkg --print-architecture)" = "arm64" ]; then ARCHITECTURE=aarch64; else ARCHITECTURE=x86_64; fi \
-    && wget --quiet "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${ARCHITECTURE}.sh" -O ~/miniconda.sh && \
-    /bin/bash ~/miniconda.sh -b -p /opt/conda
-ENV PATH=$CONDA_DIR/bin:$PATH
-RUN conda update -n base -c defaults conda && conda update --all && conda init bash
+# Install mamba
+ENV mamba_DIR /opt/mamba
+RUN wget "https://github.com/mamba-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" -O ~/mamba.sh && \
+    /bin/bash ~/mamba.sh -b -p /opt/mamba
+ENV PATH=$mamba_DIR/bin:$PATH
+RUN mamba update -n base -c defaults mamba && mamba update --all && mamba init bash
 
 # Install dependencies
 COPY ./environment.yml /tmp/environment.yml
-RUN conda env create -f /tmp/environment.yml
+RUN mamba env create -f /tmp/environment.yml
diff --git a/docker/Evaluator/Dockerfile b/docker/Evaluator/Dockerfile
@@ -3,4 +3,4 @@ FROM modynbase:latest
 RUN chmod a+x /src/modyn/evaluator/modyn-evaluator
 
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD conda run -n modyn --no-capture-output ./modyn/evaluator/modyn-evaluator ./modyn/config/examples/modyn_config.yaml
+CMD mamba run -n modyn --no-capture-output ./modyn/evaluator/modyn-evaluator ./modyn/config/examples/modyn_config.yaml
diff --git a/docker/MetadataProcessor/Dockerfile b/docker/MetadataProcessor/Dockerfile
@@ -3,4 +3,4 @@ FROM modynbase
 RUN chmod a+x /src/modyn/metadata_processor/modyn-metadata-processor
 
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD conda run -n modyn ./modyn/metadata_processor/modyn-metadata-processor ./modyn/config/examples/modyn_config.yaml
+CMD mamba run -n modyn ./modyn/metadata_processor/modyn-metadata-processor ./modyn/config/examples/modyn_config.yaml
diff --git a/docker/Model_Storage/Dockerfile b/docker/Model_Storage/Dockerfile
@@ -3,4 +3,4 @@ FROM modynbase:latest
 RUN chmod a+x /src/modyn/model_storage/modyn-model-storage
 
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD conda run -n modyn --no-capture-output ./modyn/model_storage/modyn-model-storage ./modyn/config/examples/modyn_config.yaml
+CMD mamba run -n modyn --no-capture-output ./modyn/model_storage/modyn-model-storage ./modyn/config/examples/modyn_config.yaml
diff --git a/docker/Selector/Dockerfile b/docker/Selector/Dockerfile
@@ -5,4 +5,4 @@ RUN mkdir -p /tmp/trigger_samples
 RUN chown appuser /tmp/trigger_samples
 
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD conda run -n modyn --no-capture-output ./modyn/selector/modyn-selector ./modyn/config/examples/modyn_config.yaml
+CMD mamba run -n modyn --no-capture-output ./modyn/selector/modyn-selector ./modyn/config/examples/modyn_config.yaml
diff --git a/docker/Storage/Dockerfile b/docker/Storage/Dockerfile
@@ -3,4 +3,4 @@ FROM modynbase:latest
 RUN chmod a+x /src/modyn/storage/modyn-storage
 
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD conda run -n modyn --no-capture-output ./modyn/storage/modyn-storage ./modyn/config/examples/modyn_config.yaml
+CMD mamba run -n modyn --no-capture-output ./modyn/storage/modyn-storage ./modyn/config/examples/modyn_config.yaml
diff --git a/docker/Tests/Dockerfile b/docker/Tests/Dockerfile
@@ -10,4 +10,4 @@ USER appuser
 RUN chmod a+x /src/integrationtests/run.sh
 
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD conda run -n modyn --no-capture-output /src/integrationtests/run.sh
+CMD mamba run -n modyn --no-capture-output /src/integrationtests/run.sh
diff --git a/docker/Trainer_Server/Dockerfile b/docker/Trainer_Server/Dockerfile
@@ -3,16 +3,16 @@ FROM modyndependencies:latest as apex-image
 # Uncomment the following lines to install apex
 
 # TODO(#104): Make this easily configurable here
-# RUN conda run -n modyn pip install packaging
+# RUN mamba run -n modyn pip install packaging
 # RUN git clone https://github.com/NVIDIA/apex ./apex
-# RUN conda run -v -n modyn pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./apex
+# RUN mamba run -v -n modyn pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./apex
 
 FROM modynbase:latest AS trainerserver-image
 
-COPY --from=apex-image /opt/conda/envs/modyn /opt/conda/envs/modyn
+COPY --from=apex-image /opt/mamba/envs/modyn /opt/conda/envs/modyn
 RUN chmod a+x /src/modyn/trainer_server/modyn-trainer-server
 RUN mkdir -p /tmp/offline_dataset
 RUN chown appuser /tmp/offline_dataset
 
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD conda run -n modyn --no-capture-output ./modyn/trainer_server/modyn-trainer-server ./modyn/config/examples/modyn_config.yaml
+CMD mamba run -n modyn --no-capture-output ./modyn/trainer_server/modyn-trainer-server ./modyn/config/examples/modyn_config.yaml
diff --git a/tmuxp.yaml b/tmuxp.yaml
@@ -5,7 +5,7 @@ windows:
     panes:
       - shell_command:
         - docker exec -it $(docker compose ps -q supervisor) /bin/bash
-        - clear && conda activate modyn && echo "Welcome to the supervisor pane. You have access to the 'modyn-supervisor' command here to submit pipelines."
+        - clear && mamba activate modyn && echo "Welcome to the supervisor pane. You have access to the 'modyn-supervisor' command here to submit pipelines."
   - window_name: storage
     layout: tiled
     panes: