Feat: Add Kafka integration for Parseable server #936 .

.github/workflows/build-push-edge-kafka.yaml

@@ -0,0 +1,45 @@
+name: Build and push edge kafka tag
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths-ignore:
+      - 'docs/**'
+      - 'helm/**'
+      - 'assets/**'
+      - '**.md'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: parseable/parseable
+
+      - name: Build and push
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          file: ./Dockerfile.kafka
+          push: true
+          tags: parseable/parseable:edge-kafka
+          platforms: linux/amd64,linux/arm64

.github/workflows/coverage.yaml

@@ -27,6 +27,10 @@ jobs:
         with:
           tool: cargo-hack, cargo-llvm-cov, nextest
 
+      - name: Install dependencies
+        run: sudo apt-get install -y libsasl2-dev || echo "Skipping libsasl2-dev installation"
+        continue-on-error: true
+
       - name: Check with clippy
         run: cargo hack clippy --verbose --each-feature --no-dev-deps -- -D warnings
 
@@ -39,7 +43,7 @@ jobs:
       - name: Tests
         run: cargo hack --each-feature llvm-cov --no-report nextest
 
-      - name: Genrate coverage report
+      - name: Generate coverage report
         run: cargo llvm-cov report --lcov --output-path coverage.lcov
 
       - name: Upload Coverage Report

.github/workflows/integration-test.yaml

@@ -32,3 +32,27 @@ jobs:
       - name: Stop compose
         if: always()
         run: docker compose -f docker-compose-distributed-test.yaml down
+
+  docker-compose-test-with-kafka:
+    name: Quest Smoke and Load Tests for Standalone deployments with Kafka
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Start compose
+        run: docker compose -f docker-compose-test-with-kafka.yaml up --build --exit-code-from quest
+      - name: Stop compose
+        if: always()
+        run: docker compose -f docker-compose-test-with-kafka.yaml down
+
+  docker-compose-distributed-test-with-kafka:
+    name: Quest Smoke and Load Tests for Distributed deployments with Kafka
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Start compose
+        run: docker compose -f docker-compose-distributed-test-with-kafka.yaml up --build --exit-code-from quest
+      - name: Stop compose
+        if: always()
+        run: docker compose -f docker-compose-distributed-test-with-kafka.yaml down

.gitignore

@@ -14,3 +14,4 @@ parseable
 parseable_*
 parseable-env-secret
 cache
+.idea