Integrate Uffizzi

Signed-off-by: Vibhav Bobade <[email protected]>

.dockerignore

@@ -3,3 +3,4 @@
 !metadata
 !search
 !requirements*txt
+!databuilder

.github/workflows/uffizzi-build.yml

@@ -0,0 +1,181 @@
+name: Preview (build)
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, closed]
+jobs:
+  build-search:
+    name: Build PR Amundsend Search image
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' || github.event.action != 'closed' }}
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Generate UUID image name
+        id: uuid
+        run: echo "UUID_TAG_SEARCH=$(uuidgen)" >> $GITHUB_ENV
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: registry.uffizzi.com/${{ env.UUID_TAG_SEARCH }}
+          tags: type=raw,value=60d
+      - name: Build and Push Image to registry.uffizzi.com ephemeral registry
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          context: .
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          file: ./Dockerfile.search.public
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+  build-metadata:
+    name: Build PR Amundsend Metadata image
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' || github.event.action != 'closed' }}
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Generate UUID image name
+        id: uuid
+        run: echo "UUID_TAG_METADATA=$(uuidgen)" >> $GITHUB_ENV
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: registry.uffizzi.com/${{ env.UUID_TAG_METADATA }}
+          tags: type=raw,value=60d
+      - name: Build and Push Image to registry.uffizzi.com ephemeral registry
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          context: .
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          file: ./Dockerfile.metadata.public
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+  build-databuilder:
+    name: Build PR Amundsend Databuilder image
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' || github.event.action != 'closed' }}
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Generate UUID image name
+        id: uuid
+        run: echo "UUID_TAG_DATABUILDER=$(uuidgen)" >> $GITHUB_ENV
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: registry.uffizzi.com/${{ env.UUID_TAG_DATABUILDER }}
+          tags: type=raw,value=60d
+      - name: Build and Push Image to registry.uffizzi.com ephemeral registry
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          context: .
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          file: ./Dockerfile.databuilder-init.public
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+  build-frontend:
+    name: Build PR Amundsend Frontend image
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' || github.event.action != 'closed' }}
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Generate UUID image name
+        id: uuid
+        run: echo "UUID_TAG_FRONTEND=$(uuidgen)" >> $GITHUB_ENV
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: registry.uffizzi.com/${{ env.UUID_TAG_FRONTEND }}
+          tags: type=raw,value=60d
+      - name: Build and Push Image to registry.uffizzi.com ephemeral registry
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          context: .
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          file: ./Dockerfile.frontend.local
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          args: |
+            SEARCHSERVICE_BASE: http://localhost:5001
+            METADATASERVICE_BASE: http://localhost:5002
+  render-compose-file:
+    name: Render Docker Compose file
+    # Pass output of this workflow to another triggered by `workflow_run` event.
+    runs-on: ubuntu-latest
+    needs:
+      - build-metadata
+      - build-search
+      - build-frontend
+      - build-databuilder
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Render Compose File
+        run: |
+          export AMUNDSEN_SEARCH_IMAGE=${{ needs.build-search.outputs.tags }}
+          export AMUNDSEN_METADATA_IMAGE=${{ needs.build-metadata.outputs.tags }}
+          export AMUNDSEN_FRONTEND_IMAGE=${{ needs.build-frontend.outputs.tags }}
+          export AMUNDSEN_DATABUILDER_IMAGE=${{ needs.build-databuilder.outputs.tags }}
+          # Render simple template from environment variables.
+          envsubst < ./docker-compose.uffizzi.yml > docker-compose.rendered.yml
+          cat docker-compose.rendered.yml
+      - name: Upload Rendered Compose File as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: docker-compose.rendered.yml
+          retention-days: 2
+      - name: Serialize PR Event to File
+        run: |
+          cat << EOF > event.json
+          ${{ toJSON(github.event) }}
+          EOF
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2
+
+  delete-preview:
+    name: Mark preview for deletion
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'closed' }}
+    steps:
+      # If this PR is closing, we will not render a compose file nor pass it to the next workflow.
+      - name: Serialize PR Event to File
+        run: echo '${{ toJSON(github.event) }}' > event.json
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2

.github/workflows/uffizzi-preview.yml

@@ -0,0 +1,83 @@
+name: Preview (deploy)
+
+on:
+  workflow_run:
+    workflows:
+      - "Preview (build)"
+    types:
+      - completed
+
+jobs:
+  cache-compose-file:
+    name: Cache Docker Compose file
+    runs-on: ubuntu-latest
+    outputs:
+      compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }}
+      pr-number: ${{ env.PR_NUMBER }}
+    steps:
+      - name: "Download artifacts"
+        # Fetch output (zip archive) from the workflow run that triggered this workflow.
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "preview-spec"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
+      - name: "Unzip artifact"
+        run: unzip preview-spec.zip
+      - name: Read Event into ENV
+        run: |
+          echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
+          cat event.json >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - name: Hash Rendered Compose File
+        id: hash
+        # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.rendered.yml | awk '{ print $1 }')" >> $GITHUB_ENV
+      - name: Cache Rendered Compose File
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        uses: actions/cache@v3
+        with:
+          path: docker-compose.rendered.yml
+          key: ${{ env.COMPOSE_FILE_HASH }}
+
+      - name: Read PR Number From Event Object
+        id: pr
+        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
+
+      - name: DEBUG - Print Job Outputs
+        if: ${{ runner.debug }}
+        run: |
+          echo "PR number: ${{ env.PR_NUMBER }}"
+          echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}"
+          cat event.json
+  deploy-uffizzi-preview:
+    name: Run Uffizzi deployment
+    needs:
+      - cache-compose-file
+    uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@v2
+    with:
+      # If this workflow was triggered by a PR close event, cache-key will be an empty string
+      # and this reusable workflow will delete the preview deployment.
+      compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
+      compose-file-cache-path: docker-compose.rendered.yml
+      server: https://app.uffizzi.com
+      pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
+    permissions:
+      contents: read
+      pull-requests: write
+      id-token: write

Dockerfile.databuilder-init.public

@@ -0,0 +1,13 @@
+FROM python:3.7-slim
+WORKDIR /app
+RUN pip3 install gunicorn
+
+COPY ./databuilder/* /app/
+
+RUN ls && \
+    pip3 install --upgrade pip && \ 
+    pip3 install -r requirements.txt && \
+    pip3 install -e . && \
+    python3 setup.py install
+
+CMD [ "python3", " example/scripts/sample_data_loader.py" ]

docker-compose.uffizzi.yml

@@ -0,0 +1,73 @@
+version: '3'
+x-uffizzi:
+  ingress:
+    service: amundsenfrontend
+    port: 5000
+services:
+  neo4j:
+    image: neo4j:3.5.26
+    environment:
+      - NEO4J_AUTH=neo4j/test
+    ports:
+        - 7474:7474
+        - 7687:7687
+    volumes:
+        - ./example/docker/neo4j/conf:/var/lib/neo4j/conf
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+  elasticsearch:
+    image: elasticsearch:8.0.0
+    ports:
+        - 9200:9200
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+  amundsensearch:
+    image: '${AMUNDSEN_SEARCH_IMAGE}'
+    ports:
+      - 5001:5001
+    environment:
+      - PROXY_ENDPOINT=localhost:9200
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+  amundsenmetadata:
+    image: '${AMUNDSEN_METADATA_IMAGE}'
+    depends_on:
+      - neo4j
+    ports:
+      - 5002:5002
+    environment:
+        - PROXY_HOST=bolt://localhost
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+  amundsenfrontend:
+    image: '${AMUNDSEN_FRONTEND_IMAGE}'
+    container_name: amundsenfrontend
+    ports:
+      - 5000:5000
+    environment:
+      - SEARCHSERVICE_BASE=http://localhost:5001
+      - METADATASERVICE_BASE=http://localhost:5002
+      - FRONTEND_SVC_CONFIG_MODULE_CLASS=amundsen_application.config.TestConfig
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+  amundsendatabuilderinit:
+    image: '${AMUNDSEN_DATABUILDER_IMAGE}'
+    container_name: amundsendatabuilder
+    command: ["python3", " example/scripts/sample_data_loader.py", "&&", "sleep", "infinity"]
+    deploy:
+      resources:
+        limits:
+          memory: 500M