feat: Add zillaproxy to expose Kafka via REST endpoint

- make the duration metric dynamic - add demo dashbard for logs
aklivity · Jul 18, 2024 · 0e877b5 · 0e877b5
1 parent 9b5478b
commit 0e877b5
Show file tree

Hide file tree

Showing 8 changed files with 201 additions and 109 deletions.
diff --git a/.env b/.env
@@ -133,6 +133,12 @@ KAFKA_SERVICE_PORT=9092
 KAFKA_SERVICE_ADDR=kafka:${KAFKA_SERVICE_PORT}
 KAFKA_SERVICE_DOCKERFILE=./src/kafka/Dockerfile
 
+# Zilla
+ZILLA_IMAGE=ghcr.io/aklivity/zilla
+ZILLA_VERSION=0.9.88
+ZILLA_PORT=7114
+ZILLA_ADDR=zilla-proxy:${ZILLA_PORT}
+
 # Valkey
 VALKEY_PORT=6379
 VALKEY_ADDR=valkey-cart:${VALKEY_PORT}

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -74,12 +74,3 @@ jobs:
     - name: run sanitycheck.py
       run: python3 ./internal/tools/sanitycheck.py
 
-  checklicense:
-    runs-on: ubuntu-latest
-    steps:
-    - name: check out code
-      uses: actions/checkout@v4
-    - name: install tools
-      run: make install-tools
-    - name: run checklicense
-      run: make checklicense
diff --git a/README.md b/README.md
@@ -14,26 +14,6 @@ This repository contains the OpenTelemetry Astronomy Shop, a microservice-based
 distributed system intended to illustrate the implementation of OpenTelemetry in
 a near real-world environment.
 
-Our goals are threefold:
-
-- Provide a realistic example of a distributed system that can be used to
-  demonstrate OpenTelemetry instrumentation and observability.
-- Build a base for vendors, tooling authors, and others to extend and
-  demonstrate their OpenTelemetry integrations.
-- Create a living example for OpenTelemetry contributors to use for testing new
-  versions of the API, SDK, and other components or enhancements.
-
-We've already made [huge
-progress](https://github.com/open-telemetry/opentelemetry-demo/blob/main/CHANGELOG.md),
-and development is ongoing. We hope to represent the full feature set of
-OpenTelemetry across its languages in the future.
-
-If you'd like to help (**which we would love**), check out our [contributing
-guidance](./CONTRIBUTING.md).
-
-If you'd like to extend this demo or maintain a fork of it, read our
-[fork guidance](https://opentelemetry.io/docs/demo/forking/).
-
 ## Quick start
 
 You can be up and running with the demo in a few minutes. Check out the docs for
@@ -103,36 +83,3 @@ Emeritus:
 [![contributors](https://contributors-img.web.app/image?repo=open-telemetry/opentelemetry-demo)](https://github.com/open-telemetry/opentelemetry-demo/graphs/contributors)
 
 [docs]: https://opentelemetry.io/docs/demo/
-
-<!-- Links for Demos featuring the Astronomy Shop section -->
-
-[AlibabaCloud]: https://github.com/aliyun-sls/opentelemetry-demo
-[AppDynamics]: https://www.appdynamics.com/blog/cloud/how-to-observe-opentelemetry-demo-app-in-appdynamics-cloud/
-[Aspecto]: https://github.com/aspecto-io/opentelemetry-demo
-[Axiom]: https://play.axiom.co/axiom-play-qf1k/dashboards/otel.traces.otel-demo-traces
-[Axoflow]: https://axoflow.com/opentelemetry-support-in-more-detail-in-axosyslog-and-syslog-ng/
-[Azure]: https://github.com/Azure/Azure-kusto-opentelemetry-demo
-[Coralogix]: https://coralogix.com/blog/configure-otel-demo-send-telemetry-data-coralogix
-[Dash0]: https://github.com/dash0hq/opentelemetry-demo
-[Datadog]: https://docs.datadoghq.com/opentelemetry/guide/otel_demo_to_datadog
-[Dynatrace]: https://www.dynatrace.com/news/blog/opentelemetry-demo-application-with-dynatrace/
-[Elastic]: https://github.com/elastic/opentelemetry-demo
-[GoogleCloud]: https://github.com/GoogleCloudPlatform/opentelemetry-demo
-[GrafanaLabs]: https://github.com/grafana/opentelemetry-demo
-[Guance]: https://github.com/GuanceCloud/opentelemetry-demo
-[Helios]: https://otelsandbox.gethelios.dev
-[Honeycombio]: https://github.com/honeycombio/opentelemetry-demo
-[Instana]: https://github.com/instana/opentelemetry-demo
-[Kloudfuse]: https://github.com/kloudfuse/opentelemetry-demo
-[Liatrio]: https://github.com/liatrio/opentelemetry-demo
-[Logzio]: https://logz.io/learn/how-to-run-opentelemetry-demo-with-logz-io/
-[NewRelic]: https://github.com/newrelic/opentelemetry-demo
-[OpenSearch]: https://github.com/opensearch-project/opentelemetry-demo
-[Sentry]: https://github.com/getsentry/opentelemetry-demo
-[ServiceNowCloudObservability]: https://docs.lightstep.com/otel/quick-start-operator#send-data-from-the-opentelemetry-demo
-[Splunk]: https://github.com/signalfx/opentelemetry-demo
-[SumoLogic]: https://www.sumologic.com/blog/common-opentelemetry-demo-application/
-[TelemetryHub]: https://github.com/TelemetryHub/opentelemetry-demo/tree/telemetryhub-backend
-[Teletrace]: https://github.com/teletrace/opentelemetry-demo
-[Tracetest]: https://github.com/kubeshop/opentelemetry-demo
-[Uptrace]: https://github.com/uptrace/uptrace/tree/master/example/opentelemetry-demo
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -639,6 +639,38 @@ services:
       - "${VALKEY_PORT}"
     logging: *logging
 
+  # Zilla
+  zillaproxy:
+    image: ${ZILLA_IMAGE}:${ZILLA_VERSION}
+    container_name: zilla-proxy
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+    restart: unless-stopped
+    ports:
+      - "${ZILLA_PORT}:${ZILLA_PORT}"
+    environment:
+      - KAFKA_SERVICE_ADDR
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://${OTEL_COLLECTOR_HOST}:${OTEL_COLLECTOR_PORT_HTTP}
+      - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE
+      - OTEL_RESOURCE_ATTRIBUTES
+      - OTEL_SERVICE_NAME=zillaproxy
+    volumes:
+      - ./src/zillaproxy/zilla.yaml:/etc/zilla/zilla.yaml
+    command: start -v -e
+    healthcheck:
+      test: nc -z zilla ${ZILLA_PORT}
+      start_period: 10s
+      interval: 5s
+      timeout: 10s
+      retries: 10
+    depends_on:
+      otelcol:
+        condition: service_started
+      kafka:
+        condition: service_healthy
+    logging: *logging
 
   # ********************
   # Telemetry Components

diff --git a/kubernetes/opentelemetry-demo.yaml b/kubernetes/opentelemetry-demo.yaml
@@ -718,7 +718,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "histogram_quantile(0.50, sum(rate(duration_milliseconds_bucket{service_name=\"${service}\"}[$__rate_interval])) by (le))",
+              "expr": "histogram_quantile(0.50, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=\"${service}\"}[$__rate_interval])) by (le))",
               "legendFormat": "quantile50",
               "range": true,
               "refId": "A"
@@ -730,7 +730,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "histogram_quantile(0.95, sum(rate(duration_milliseconds_bucket{service_name=\"${service}\"}[$__rate_interval])) by (le))",
+              "expr": "histogram_quantile(0.95, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=\"${service}\"}[$__rate_interval])) by (le))",
               "hide": false,
               "legendFormat": "quantile95",
               "range": true,
@@ -743,7 +743,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "histogram_quantile(0.99, sum(rate(duration_milliseconds_bucket{service_name=\"${service}\"}[$__rate_interval])) by (le))",
+              "expr": "histogram_quantile(0.99, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=\"${service}\"}[$__rate_interval])) by (le))",
               "hide": false,
               "legendFormat": "quantile99",
               "range": true,
@@ -756,7 +756,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "histogram_quantile(0.999, sum(rate(duration_milliseconds_bucket{service_name=\"${service}\"}[$__rate_interval])) by (le))",
+              "expr": "histogram_quantile(0.999, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=\"${service}\"}[$__rate_interval])) by (le))",
               "hide": false,
               "legendFormat": "quantile999",
               "range": true,
@@ -946,7 +946,7 @@ data:
                 "uid": "webstore-metrics"
               },
               "editorMode": "code",
-              "expr": "sum by (span_name) (rate(duration_milliseconds_count{service_name=\"${service}\"}[$__rate_interval]))",
+              "expr": "sum by (span_name) (rate({__name__=~\".*duration_milliseconds_count\", {service_name=\"${service}\"}[$__rate_interval]))",
               "legendFormat": "{{ span_name }}",
               "range": true,
               "refId": "A"
@@ -7321,7 +7321,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "topk(7,histogram_quantile(0.50, sum(rate(duration_milliseconds_bucket{service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name)))",
+              "expr": "topk(7,histogram_quantile(0.50, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name)))",
               "format": "time_series",
               "hide": true,
               "instant": false,
@@ -7337,7 +7337,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "topk(7,histogram_quantile(0.95, sum(rate(duration_milliseconds_bucket{service_name=~\"$service\", span_name=~\"$span_name\"}[$__range])) by (le,service_name)))",
+              "expr": "topk(7,histogram_quantile(0.95, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=~\"$service\", span_name=~\"$span_name\"}[$__range])) by (le,service_name)))",
               "hide": false,
               "instant": true,
               "interval": "",
@@ -7352,7 +7352,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "histogram_quantile(0.99, sum(rate(duration_milliseconds_bucket{service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name))",
+              "expr": "histogram_quantile(0.99, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name))",
               "hide": true,
               "interval": "",
               "legendFormat": "quantile99",
@@ -7366,7 +7366,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "histogram_quantile(0.999, sum(rate(duration_milliseconds_bucket{service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name))",
+              "expr": "histogram_quantile(0.999, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name))",
               "hide": true,
               "interval": "",
               "legendFormat": "quantile999",
@@ -7904,7 +7904,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "topk(7,histogram_quantile(0.50, sum(rate(duration_milliseconds_bucket{service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name)))",
+              "expr": "topk(7,histogram_quantile(0.50, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name)))",
               "format": "time_series",
               "hide": true,
               "instant": false,
@@ -7920,7 +7920,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "topk(7,histogram_quantile(0.95, sum(rate(duration_milliseconds_bucket{service_name=~\"$service\", span_name=~\"$span_name\"}[$__range])) by (le,span_name)))",
+              "expr": "topk(7,histogram_quantile(0.95, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=~\"$service\", span_name=~\"$span_name\"}[$__range])) by (le,span_name)))",
               "hide": false,
               "instant": true,
               "interval": "",
@@ -7935,7 +7935,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "histogram_quantile(0.99, sum(rate(duration_milliseconds_bucket{service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name))",
+              "expr": "histogram_quantile(0.99, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name))",
               "hide": true,
               "interval": "",
               "legendFormat": "quantile99",
@@ -7949,7 +7949,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "histogram_quantile(0.999, sum(rate(duration_milliseconds_bucket{service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name))",
+              "expr": "histogram_quantile(0.999, sum(rate({__name__=~\".*duration_.+seconds_bucket\", service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])) by (le,service_name))",
               "hide": true,
               "interval": "",
               "legendFormat": "quantile999",
@@ -8020,7 +8020,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "topk(7, sum by (span_name,service_name)(increase(duration_milliseconds_sum{service_name=~\"${service}\", span_name=~\"$span_name\"}[5m]) / increase(duration_milliseconds_count{service_name=~\"${service}\",span_name=~\"$span_name\"}[5m\n])))",
+              "expr": "topk(7, sum by (span_name,service_name)(increase({__name__=~\".*duration_milliseconds_sum\", service_name=~\"${service}\", span_name=~\"$span_name\"}[5m]) / increase({__name__=~\".*duration_milliseconds_count\", {service_name=~\"${service}\",span_name=~\"$span_name\"}[5m\n])))",
               "instant": true,
               "interval": "",
               "legendFormat": "{{span_name}} [{{service_name}}]",
@@ -8124,7 +8124,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "topk(7,sum by (span_name,service_name)(increase(duration_milliseconds_sum{service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval]) / increase(duration_milliseconds_count{service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])))",
+              "expr": "topk(7,sum by (span_name,service_name)(increase({__name__=~\".*duration_milliseconds_sum\", service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval]) / increase({__name__=~\".*duration_milliseconds_count\", {service_name=~\"$service\", span_name=~\"$span_name\"}[$__rate_interval])))",
               "instant": false,
               "interval": "",
               "legendFormat": "[{{service_name}}]  {{span_name}}",