Merge pull request #136 from 3DBAG/pc-2024.12.06

3DBAG release 2024.12.16

.dockerignore

@@ -0,0 +1,45 @@
+# !!! .dockerignore files cannot be nested in subdirectories, thus it does not work like .gitignore
+
+# Local env
+.env_*
+.env
+
+# IDE config
+**/.run
+**/.idea
+
+# Build artifacts and cache
+**/*.egg-info
+**/build
+**/.pytest_cache
+**/*__pycache__*
+
+# Test data
+tmp
+tests/test_data
+
+# Local DAGSTER_HOME
+tests/dagster_home/.logs_queue/
+tests/dagster_home/.nux
+tests/dagster_home/.telemetry/
+tests/dagster_home/history/
+tests/dagster_home/logs/
+tests/dagster_home/schedules/
+tests/dagster_home/storage/
+
+# Virtual envs
+venvs
+
+# Documentation
+**/site
+**/docs
+
+# Tools
+**/.tox
+**/.coverage
+**/*.ruff_cache
+
+# Other
+experiments
+deployment
+scripts

.github/workflows/build-docker-develop.yaml

@@ -0,0 +1,75 @@
+name: Build and Publish Docker Images
+
+on:
+  push:
+    tags:
+      - '202[2-9].[0-9]+.[0-9]+'
+      - '202[2-9].[0-9]+-pc[0-9]+'
+  pull_request:
+    branches:
+      - "develop"
+    types: [ closed ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set docker image tag to 'develop'
+        if: github.ref == 'refs/heads/develop'
+        run: echo "DOCKER_TAG=develop" >> $GITHUB_ENV
+      - name: Set docker image tag to git tag
+        if: startsWith(github.ref, 'refs/tags/')
+        run: echo "DOCKER_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push core
+        uses: docker/build-push-action@v6
+        with:
+          context: ./
+          file: ./docker/pipeline/bag3d-core.dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            JOBS=2
+            VERSION=${{ env.DOCKER_TAG }}
+          push: true
+          tags: 3dgi/3dbag-pipeline-core:${{ env.DOCKER_TAG }}
+      - name: Build and push floors estimation
+        uses: docker/build-push-action@v6
+        with:
+          context: ./
+          file: ./docker/pipeline/bag3d-floors-estimation.dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            JOBS=2
+            VERSION=${{ env.DOCKER_TAG }}
+          push: true
+          tags: 3dgi/3dbag-pipeline-floors-estimation:${{ env.DOCKER_TAG }}
+      - name: Build and push party walls
+        uses: docker/build-push-action@v6
+        with:
+          context: ./
+          file: ./docker/pipeline/bag3d-party-walls.dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            JOBS=2
+            VERSION=${{ env.DOCKER_TAG }}
+          push: true
+          tags: 3dgi/3dbag-pipeline-party-walls:${{ env.DOCKER_TAG }}
+      - name: Build and push dagster
+        uses: docker/build-push-action@v6
+        with:
+          context: ./docker/dagster
+          file: ./docker/dagster/Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            JOBS=2
+            VERSION=${{ env.DOCKER_TAG }}
+          push: true
+          tags: 3dgi/3dbag-pipeline-dagster:${{ env.DOCKER_TAG }}

.github/workflows/build-docker-tools.yaml

@@ -0,0 +1,39 @@
+name: Build and Publish The Tools Docker Image
+
+on:
+  push:
+    tags:
+      - '*'
+    branches:
+      - "develop"
+      - "pc-*"
+    paths:
+      - docker/tools/Dockerfile
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set docker image tag
+        run: echo "DOCKER_TAG=$(date +'%Y.%m.%d')" >> $GITHUB_ENV
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push tools image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./
+          file: ./docker/tools/Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            JOBS=2
+            VERSION=${{ env.DOCKER_TAG }}
+          push: true
+          tags: 3dgi/3dbag-pipeline-tools:${{ env.DOCKER_TAG }}
+

.github/workflows/documentation.yaml

@@ -0,0 +1,31 @@
+name: Publish docs
+on:
+  push:
+    branches:
+      - main
+      - develop
+permissions:
+  contents: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - uses: actions/cache@v4
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - run: |
+          pip install -r requirements-dev.txt
+          pip install --no-deps packages/common
+      - run: mkdocs gh-deploy --force

.github/workflows/lint-and-formatting.yaml

@@ -0,0 +1,26 @@
+name: Lint & Format
+
+on:
+  pull_request:
+    branches: ["develop", "master"]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Intall uv
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install dependencies
+        run: uv pip install ruff --system
+      - name: Lint
+        run: ruff check
+      - name: Format
+        run: ruff format --check

.github/workflows/test.yaml

@@ -0,0 +1,41 @@
+name: Test
+
+on:
+  pull_request:
+    branches: [ "develop", "master" ]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+#  push:
+#    branches: [ "develop", "master" ]
+#    paths-ignore:
+#      - "docs/**"
+#      - "**.md"
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - run: |
+        touch .env
+        echo BAG3D_TEST_DATA=${PWD}/tests/test_data >> .env
+    - name: Build docker images
+      run: |
+        make download
+        make docker_volume_create
+        make docker_up
+    - name: Run unit tests
+      run: make test
+    - name: Recreate volumes
+      run: |
+        make docker_down
+        make docker_volume_recreate
+        make docker_up_nobuild
+    - name: Run integration tests
+      run: make test_integration

.gitignore

@@ -1,45 +1,37 @@
 # Local env
-/.env_*
-/.env
+.env_*
+.env
 
 # IDE config
-/.run/
+.run
 .idea
 
-# Python build artefacts and chache
-.pytest_cache/
-/src/bag3d_pipeline.egg-info/
-/.tox/
+# Build artifacts and cache
+.pytest_cache
 *__pycache__*
+*.log
 
 # Test data
-/tmp/*
-/tests/test_data/
-/packages/party_walls/tests/data/
-/packages/core/tests/data/
+tmp/*
+tests/test_data/
 
 # Local DAGSTER_HOME
-/tests/dagster_home/.logs_queue/
-/tests/dagster_home/.telemetry/
-/tests/dagster_home/history/
-/tests/dagster_home/schedules/
-/tests/dagster_home/storage/
-/tests/dagster_home/logs/
-/tests/dagster_home/.nux
-/tests/dagster_home/tmp*
-/tests/dagster_home/.env
+tests/dagster_home/.logs_queue/
+tests/dagster_home/.nux
+tests/dagster_home/.telemetry/
+tests/dagster_home/history/
+tests/dagster_home/logs/
+tests/dagster_home/schedules/
+tests/dagster_home/storage/
 
-/packages/common/src/bag3d_common.egg-info/
-/packages/common/.tox/
-/packages/common/tests/.pytest_cache/
-/venvs/venv_*
-/packages/core/src/bag3d_core.egg-info/
-/packages/core/tests/.pytest_cache/
-/packages/core/.tox/
-/packages/core/build/
-/packages/party_walls/src/bag3d_party_walls.egg-info/
-/packages/party_walls/tests/.pytest_cache/
-/packages/party_walls/build/
-/packages/floors_estimation/src/bag3d_floors_estimation.egg-info/
-/packages/floors_estimation/tests/.pytest_cache/
-/packages/floors_estimation/build/
+# Virtual env
+.venv/
+
+# Documentation
+site
+docs/reference
+
+# Tools
+.tox
+.coverage
+*.ruff_cache

CONTRIBUTING.md

@@ -0,0 +1 @@
+Please have a look at our contribution guidelines in our documentation: [https://innovation.3dbag.nl/3dbag-pipeline](https://innovation.3dbag.nl/3dbag-pipeline/development/guidelines).

NOTICE

@@ -1,2 +1,2 @@
 3dbag-pipeline software
-Copyright 2023 Balázs Dukai <[email protected]>, 3DGI
+Copyright 2023-2024 Balázs Dukai <[email protected]> (3DGI), Ravi Peters (3DGI), TU Delft 3D geoinformation group