diff --git a/prometheus/Dockerfile b/prometheus/Dockerfile new file mode 100644 index 00000000..76429ec8 --- /dev/null +++ b/prometheus/Dockerfile @@ -0,0 +1,3 @@ +FROM ghcr.io/deephaven/grpc-api +COPY app.d /app.d +RUN pip3 install -r /app.d/requirements.txt diff --git a/prometheus/README.md b/prometheus/README.md new file mode 100644 index 00000000..cde17765 --- /dev/null +++ b/prometheus/README.md @@ -0,0 +1,70 @@ +# Prometheus + +[Prometheus](https://prometheus.io/) is an open-source systems monitoring and alerting toolkit that collects and stores its metrics as time series data. This sample app shows how to ingest data from Prometheus into [Deephaven](https://deephaven.io/). + +## How it works + +### Deephaven application mode + +This app runs using [Deephaven's application mode](https://deephaven.io/core/docs/how-to-guides/app-mode/). + +### Components + +* `Dockerfile` - The dockerfile for the application. This extends the default Deephaven image to add dependencies. See our guide, [How to install Python packages](https://deephaven.io/core/docs/how-to-guides/install-python-packages/#add-packages-to-a-custom-docker-image), for more information. +* `docker-compose.yml` - The Docker Compose file for the application. This is mostly the same as the [Deephaven docker-compose file](https://raw.githubusercontent.com/deephaven/deephaven-core/main/containers/python-examples/docker-compose.yml) with modifications to run Prometheus, application mode, and the custom dependencies. +* `start.sh` - A simple helper script to launch the application. +* `app.d/app.app` - The Deephaven application mode app file. +* `app.d/requirements.txt` - Python dependencies for the application. +* `app.d/prometheus.py` - The Python script that pulls the data from Prometheus and stores it into Deephaven. + +### High level overview + +This app pulls data from [Prometheus's API](https://prometheus.io/docs/prometheus/latest/querying/api/) through HTTP requests. The API responses are deserialized, and the desired values are extracted and stored into a Deephaven table. + +Once data is collected and tables are created, various [Deephaven queries](https://deephaven.io/core/docs/how-to-guides/simple-python-query/) are then performed on the tables. + +This app writes to Deephaven tables both statically and dynamically. + +## Dependencies + +* The [Deephaven-core dependencies](https://github.com/deephaven/deephaven-core#required-dependencies) are required for this project. +* If you want to use a different Prometheus instance than the default instance, you will need to [install Prometheus](https://prometheus.io/docs/prometheus/latest/installation/) at your desired location. + +## Launch + +Before launching, you can modify the `PROMETHEUS_QUERIES` and `BASE_URL` values in `prometheus.py` to see the results of different queries, and to point the application at different Prometheus instances. + +Once you are set, simply run the following to launch the app: + +``` +sh start.sh +``` + +Go to [http://localhost:10000/ide](http://localhost:10000/ide) to view the tables in the top right **Panels** tab! + +### Ngrok + + +:::note + +If you are running Prometheus locally and seeing errors like: + +``` +HTTPConnectionPool(host='localhost', port=9090): Max retries exceeded with url: /api/v1/query?query=up (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')) +``` + +you may need to use [Ngrok](https://ngrok.com/) to make HTTP requests to your Prometheus instance. + +::: + +[Install Ngrok](https://ngrok.com/download) on the machine that is running your Prometheus instance, then run the following in a new terminal window on that same machine: + +``` +ngrok http 9090 +``` + +Use the URL on the terminal that forwards to to construct the `BASE_URL` value. Edit this value in `prometheus.py` and re-launch the application. For example: + +``` +BASE_URL = "{base}/api/v1/query".format(base="http://c818-2603-6081-2300-2640-50c5-4e0a-6c65-498d.ngrok.io") +``` diff --git a/prometheus/app.d/app.app b/prometheus/app.d/app.app new file mode 100644 index 00000000..4a3091d5 --- /dev/null +++ b/prometheus/app.d/app.app @@ -0,0 +1,6 @@ +type=script +scriptType=python +enabled=true +id=prometheus +name=Prometheus application +file_0=./prometheus.py diff --git a/prometheus/app.d/prometheus.py b/prometheus/app.d/prometheus.py new file mode 100644 index 00000000..3ceaa5dd --- /dev/null +++ b/prometheus/app.d/prometheus.py @@ -0,0 +1,160 @@ +""" +prometheus.py + +A simple python script that pulls data from Prometheus's API, and +stores it in a Deephaven table. + +This is expected to be run within Deephaven's application mode https://deephaven.io/core/docs/how-to-guides/app-mode/. + +After launching, there will be 2 tables within the "Panels" section of the Deephaven UI. +One will be a static table and the other will be continually updating with real data. + +@author Jake Mulford +@copyright Deephaven Data Labs LLC +""" +from deephaven.TableTools import newTable, stringCol, dateTimeCol, doubleCol +from deephaven import DynamicTableWriter +from deephaven.DBTimeUtils import millisToTime +import deephaven.Types as dht +from typing import Callable + +import requests + +import threading +import time + +PROMETHEUS_QUERIES = ["up", "go_memstats_alloc_bytes"] #Edit this and add your queries here +BASE_URL = "{base}/api/v1/query".format(base="http://prometheus:9090") #Edit this to your base URL if you're not using a local Prometheus instance + +ApplicationState = jpy.get_type("io.deephaven.appmode.ApplicationState") + +def make_prometheus_request(prometheus_query, query_url): + """ + A helper method that makes a request on the Prometheus API with the given + query, and returns a list of results containing the timestamp, job, instance, and value for the query. + The data returned by this method will be stored in a Deephaven table. + + This assumes that the query is going to return a "vector" type from the Prometheus API. + https://prometheus.io/docs/prometheus/latest/querying/api/#instant-vectors + + Args: + prometheus_query (str): The Prometheus query to execute with the API request. + query_url (str): The URL of the query endpoint. + Returns: + list[(date-time, str, str, float)]: List of the timestamps, jobs, instances, and values from the API response. + """ + results = [] + query_parameters = { + "query": prometheus_query + } + response = requests.get(query_url, params=query_parameters) + response_json = response.json() + + if "data" in response_json.keys(): + if "resultType" in response_json["data"] and response_json["data"]["resultType"] == "vector": + for result in response_json["data"]["result"]: + #Prometheus timestamps are in seconds. We multiply by 1000 to convert it to + #milliseconds, then cast to an int() to use the millisToTime() method + timestamp = millisToTime(int(result["value"][0] * 1000)) + job = result["metric"]["job"] + instance = result["metric"]["instance"] + value = float(result["value"][1]) + results.append((timestamp, job, instance, value)) + return results + +def start_dynamic(app: ApplicationState): + """ + Deephaven Application Mode method that starts the dynamic data collector. + """ + column_names = ["DateTime", "PrometheusQuery", "Job", "Instance", "Value"] + column_types = [dht.datetime, dht.string, dht.string, dht.string, dht.double] + + table_writer = DynamicTableWriter( + column_names, + column_types + ) + + result = table_writer.getTable() + + def thread_func(): + while True: + for prometheus_query in PROMETHEUS_QUERIES: + values = make_prometheus_request(prometheus_query, BASE_URL) + + for (date_time, job, instance, value) in values: + table_writer.logRow(date_time, prometheus_query, job, instance, value) + time.sleep(2) + + app.setField("result_dynamic", result) + thread = threading.Thread(target = thread_func) + thread.start() + +def start_static(app: ApplicationState, query_count=5): + """ + Deephaven Application Mode method that starts the static data collector. + + query_count sets the number of requests to make. It is recommended to keep this number low, + since it delays how long the Deephaven UI takes to become accessible. + """ + date_time_list = [] + prometheus_query_list = [] + job_list = [] + instance_list = [] + value_list = [] + + for i in range(query_count): + for prometheus_query in PROMETHEUS_QUERIES: + values = make_prometheus_request(prometheus_query, BASE_URL) + + for (date_time, job, instance, value) in values: + date_time_list.append(date_time) + prometheus_query_list.append(prometheus_query) + job_list.append(job) + instance_list.append(instance) + value_list.append(value) + time.sleep(2) + + result = newTable( + dateTimeCol("DateTime", date_time_list), + stringCol("PrometheusQuery", prometheus_query_list), + stringCol("Job", job_list), + stringCol("Instance", instance_list), + doubleCol("Value", value_list) + ) + app.setField("result_static", result) + +def update(app: ApplicationState): + """ + Deephaven Application Mode method that does various updates on the initial tables. + + You can throw any Deehaven Query in here. The ones in here are simply examples. + """ + #Get the tables from the app + result_static = app.getField("result_static").value() + result_dynamic = app.getField("result_dynamic").value() + + #Perform the desired queries, and set the results as new fields + result_static_update = result_static.by("PrometheusQuery") + app.setField("result_static_update", result_static_update) + + result_static_average = result_static.dropColumns("DateTime", "Job", "Instance").avgBy("PrometheusQuery") + app.setField("result_static_average", result_static_average) + + result_dynamic_update = result_dynamic.by("PrometheusQuery") + app.setField("result_dynamic_update", result_dynamic_update) + + result_dynamic_average = result_dynamic.dropColumns("DateTime", "Job", "Instance").avgBy("PrometheusQuery") + app.setField("result_dynamic_average", result_dynamic_average) + +def initialize(func: Callable[[ApplicationState], None]): + """ + Deephaven Application Mode initialization method. + """ + app = jpy.get_type("io.deephaven.appmode.ApplicationContext").get() + func(app) + +#Start the static and dynamic data collectors +initialize(start_static) +initialize(start_dynamic) +#Run the table updates +initialize(update) diff --git a/prometheus/app.d/requirements.txt b/prometheus/app.d/requirements.txt new file mode 100644 index 00000000..0866b439 --- /dev/null +++ b/prometheus/app.d/requirements.txt @@ -0,0 +1,5 @@ +certifi==2021.5.30 +charset-normalizer==2.0.6 +idna==3.2 +requests==2.26.0 +urllib3==1.26.7 diff --git a/prometheus/docker-compose.yml b/prometheus/docker-compose.yml new file mode 100644 index 00000000..5b5fe5d5 --- /dev/null +++ b/prometheus/docker-compose.yml @@ -0,0 +1,47 @@ +version: "3.4" + +services: + grpc-api: + image: prometheus-deephaven/grpc-api:latest + expose: + - '8080' + volumes: + - ./data:/data + - api-cache:/cache + environment: + - JAVA_TOOL_OPTIONS=-Xmx4g -Ddeephaven.console.type=python -Ddeephaven.application.dir=/app.d + + web: + image: ghcr.io/deephaven/web:${VERSION:-latest} + expose: + - '80' + volumes: + - ./data:/data + - web-tmp:/tmp + + grpc-proxy: + image: ghcr.io/deephaven/grpc-proxy:${VERSION:-latest} + environment: + - BACKEND_ADDR=grpc-api:8080 + depends_on: + - grpc-api + expose: + - '8080' + + envoy: + image: ghcr.io/deephaven/envoy:${VERSION:-latest} + depends_on: + - web + - grpc-proxy + - grpc-api + ports: + - "10000:10000" + + prometheus: + image: prom/prometheus + ports: + - "9090:9090" + +volumes: + web-tmp: + api-cache: diff --git a/prometheus/start.sh b/prometheus/start.sh new file mode 100644 index 00000000..8e9f87ab --- /dev/null +++ b/prometheus/start.sh @@ -0,0 +1,2 @@ +docker build --tag prometheus-deephaven/grpc-api . +docker-compose up