Move exometer to prometheus

Dockerfile

@@ -5,10 +5,14 @@ LABEL org.label-schema.name='AMOC Arsenal' \
 
 WORKDIR /amoc_arsenal
 
-COPY ./ ./
-RUN git clean -ffxd
+COPY rebar.lock .
+RUN rebar3 compile --deps_only
+COPY rebar.config .
+COPY src src
+COPY ci ci
+COPY priv priv
 RUN rebar3 release
 
-ENV PATH "/amoc_arsenal/_build/default/rel/amoc_arsenal/bin:${PATH}"
+ENV PATH="/amoc_arsenal/_build/default/rel/amoc_arsenal/bin:${PATH}"
 
 CMD ["amoc_arsenal", "console", "-noshell", "-noinput", "+Bd"]

ci/docker-compose.yml

@@ -1,9 +1,10 @@
 x-amoc-defaults: &amoc-defaults
   image: "amoc-arsenal"
+  pull_policy: never
   networks:
     - amoc-test-network
   environment:
-    AMOC_GRAPHITE_HOST: '"graphite"'
+    AMOC_PROMETHEUS_IP: '{0, 0, 0, 0}'
     AMOC_NODES: "['amoc_arsenal@amoc-master']"
   healthcheck:
     test: "amoc_arsenal status"
@@ -29,14 +30,16 @@ services:
     hostname: "amoc-worker-3"
     ports:
       - "4003:4000"
-  graphite:
-    image: "graphiteapp/graphite-statsd:1.1.10-4"
+  prometheus:
+    image: "prom/prometheus"
     ports:
-      - "8080:80"
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
     networks:
       - amoc-test-network
   grafana:
-    image: "grafana/grafana:8.2.6"
+    image: "grafana/grafana"
     ports:
       - "3000:3000"
     networks:

ci/helper.sh

@@ -81,7 +81,7 @@ function get_status() {
 ## graphite REST API functions ##
 #################################
 function metrics_reported() {
-    curl -s "http://localhost:8080/metrics/find?query=*" | contains "$@"
+    curl -s "http://localhost:9090/api/v1/targets" | contains "$@"
 }
 
 function wait_for_metrics() {

ci/prometheus.yml

@@ -0,0 +1,21 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+scrape_configs:
+  - job_name: prometheus
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+    static_configs:
+      - targets: ["localhost:9090"]
+  - job_name: amoc-master
+    static_configs:
+      - targets: ["amoc-master:9090"]
+  - job_name: amoc-worker-1
+    static_configs:
+      - targets: ["amoc-worker-1:9090"]
+  - job_name: amoc-worker-2
+    static_configs:
+      - targets: ["amoc-worker-2:9090"]
+  - job_name: amoc-worker-3
+    static_configs:
+      - targets: ["amoc-worker-3:9090"]

ci/start_demo_cluster.sh

@@ -3,14 +3,14 @@
 source "$(dirname "$0")/helper.sh"
 enable_strict_mode
 
-docker_compose up --wait --wait-timeout 100 amoc-{master,worker-1,worker-2} graphite grafana
+docker_compose up --wait --wait-timeout 100 amoc-{master,worker-1,worker-2} prometheus grafana
 
 ## configure default grafana datasource
 json=( '{'
-       '"name": "graphite",'
+       '"name": "prometheus",'
        '"access": "proxy",'
-       '"type": "graphite",'
-       '"url": "http://graphite",'
+       '"type": "prometheus",'
+       '"url": "http://prometheus:9090",'
        '"isDefault": true'
        '}' )
 

doc/metrics.md

@@ -1,20 +1,21 @@
 ## Metrics
 
-`amoc_metrics` allow to configure a Graphite reporter using the following environment variables:
+`amoc_metrics` allow to configure a Prometheus exporter using the following environment variables:
 
-* ``graphite_host`` - a graphite host address (string or `undefined`):
-    * default value - `undefined` (amoc_metrics do not try to initialise a metrics reporter)
-    * example: `AMOC_GRAPHITE_HOST='"graphite"'`
+* `prometheus_port` - prometheus port:
+    * default value - `9090`
+    * example: `AMOC_PROMETHEUS_PORT='9090'`
 
-* ``graphite_port`` - graphite port:
-    * default value - `2003`
-    * example: `AMOC_GRAPHITE_PORT='2003'`
+* `prometheus_ip` - prometheus IP:
+    * default value - `"127.0.0.1"`
+    * example: `AMOC_PROMETHEUS_IP='"127.0.0.1"'`
 
-* ``graphite_prefix`` - graphite prefix:
-    * default value - `net_adm:localhost()`
-    * example: `AMOC_GRAPHITE_PREFIX='"amoc"'`
+Note that they are parsed as erlang terms and so the double-quotes inside the single-quotes are
+necessary.
 
-In order to initialise some preconfigured metrics, other applications can declare the `predefined_metrics` environment variable (in their own `*.app.src` file):  
+In order to initialise some preconfigured metrics,
+other applications can declare the `predefined_metrics`
+environment variable (in their own `*.app.src` file):
 ```erl
-{predefined_metrics, [{gauge, some_metric}, {times, another_metric}]}
+{predefined_metrics, [{gauge, some_metric}, {histogram, another_metric}]}
 ```

rebar.config

@@ -7,9 +7,8 @@
 {deps, [
     {amoc, "3.3.0"},
     {telemetry, "1.3.0"},
-    {exometer_core, {git, "https://github.com/esl/exometer_core.git", {branch, "master"}}},
-    {exometer_report_graphite,
-        {git, "https://github.com/esl/exometer_report_graphite.git", {branch, "master"}}},
+    {prometheus, "4.11.0"},
+    {prometheus_cowboy, "0.1.8"},
     %% when updating amoc_rest version, don't forget to update it at ./doc/http-api.md as well.
     {amoc_rest, {git, "https://github.com/esl/amoc_rest.git", {tag, "1.2.2"}}}
 ]}.

rebar.lock

@@ -1,41 +1,42 @@
 {"1.2.0",
-[{<<"amoc">>,{pkg,<<"amoc">>,<<"3.3.0">>},0},
+[{<<"accept">>,{pkg,<<"accept">>,<<"0.3.5">>},2},
+ {<<"amoc">>,{pkg,<<"amoc">>,<<"3.3.0">>},0},
  {<<"amoc_rest">>,
   {git,"https://github.com/esl/amoc_rest.git",
        {ref,"d9f642255f48d14a1ff9601500796f5edb4e11e6"}},
   0},
- {<<"bear">>,{pkg,<<"bear">>,<<"1.0.0">>},1},
  {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.12.0">>},1},
  {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.13.0">>},2},
- {<<"exometer_core">>,
-  {git,"https://github.com/esl/exometer_core.git",
-       {ref,"123daa053a4abb3ff4bdbf52f08344da535294e9"}},
-  0},
- {<<"exometer_report_graphite">>,
-  {git,"https://github.com/esl/exometer_report_graphite.git",
-       {ref,"59e475a094818294443de9dc68e08ee0116a5626"}},
-  0},
  {<<"jesse">>,{pkg,<<"jesse">>,<<"1.8.1">>},1},
- {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.0">>},1},
+ {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.11.0">>},0},
+ {<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.1.8">>},0},
+ {<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.11">>},1},
+ {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},1},
  {<<"ranch">>,{pkg,<<"ranch">>,<<"2.1.0">>},1},
  {<<"telemetry">>,{pkg,<<"telemetry">>,<<"1.3.0">>},0}]}.
 [
 {pkg_hash,[
+ {<<"accept">>, <<"B33B127ABCA7CC948BBE6CAA4C263369ABF1347CFA9D8E699C6D214660F10CD1">>},
  {<<"amoc">>, <<"531B7E8CE39D40B4BF5A819868091C4451DC3D3FDAE753E3E3B1D0E5E8E81CDD">>},
- {<<"bear">>, <<"430419C1126B477686CDE843E88BA0F2C7DC5CDF0881C677500074F704339A99">>},
  {<<"cowboy">>, <<"F276D521A1FF88B2B9B4C54D0E753DA6C66DD7BE6C9FCA3D9418B561828A3731">>},
  {<<"cowlib">>, <<"DB8F7505D8332D98EF50A3EF34B34C1AFDDEC7506E4EE4DD4A3A266285D282CA">>},
  {<<"jesse">>, <<"C9E3670C7EE40F719734E3BC716578143AABA93FC7525A02A7D5CB300B3AD71E">>},
- {<<"parse_trans">>, <<"BB87AC362A03CA674EBB7D9D498F45C03256ADED7214C9101F7035EF44B798C7">>},
+ {<<"prometheus">>, <<"B95F8DE8530F541BD95951E18E355A840003672E5EDA4788C5FA6183406BA29A">>},
+ {<<"prometheus_cowboy">>, <<"CFCE0BC7B668C5096639084FCD873826E6220EA714BF60A716F5BD080EF2A99C">>},
+ {<<"prometheus_httpd">>, <<"F616ED9B85B536B195D94104063025A91F904A4CFC20255363F49A197D96C896">>},
+ {<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>},
  {<<"ranch">>, <<"2261F9ED9574DCFCC444106B9F6DA155E6E540B2F82BA3D42B339B93673B72A3">>},
  {<<"telemetry">>, <<"FEDEBBAE410D715CF8E7062C96A1EF32EC22E764197F70CDA73D82778D61E7A2">>}]},
 {pkg_hash_ext,[
+ {<<"accept">>, <<"11B18C220BCC2EAB63B5470C038EF10EB6783BCB1FCDB11AA4137DEFA5AC1BB8">>},
  {<<"amoc">>, <<"B8DD4F77BB94716ABC64E863158EEF8E1375CECB2F69E57DC4A293B0949D4985">>},
- {<<"bear">>, <<"157B67901ADF84FF0DA6EAE035CA1292A0AC18AA55148154D8C582B2C68959DB">>},
  {<<"cowboy">>, <<"8A7ABE6D183372CEB21CAA2709BEC928AB2B72E18A3911AA1771639BEF82651E">>},
  {<<"cowlib">>, <<"E1E1284DC3FC030A64B1AD0D8382AE7E99DA46C3246B815318A4B848873800A4">>},
  {<<"jesse">>, <<"0EDED3F18623FDA2F25989804A06CF518B4ACF2E9365B18C8E8C013D7E3C906F">>},
- {<<"parse_trans">>, <<"F99E368830BEA44552224E37E04943A54874F08B8590485DE8D13832B63A2DC3">>},
+ {<<"prometheus">>, <<"719862351AABF4DF7079B05DC085D2BBCBE3AC0AC3009E956671B1D5AB88247D">>},
+ {<<"prometheus_cowboy">>, <<"BA286BECA9302618418892D37BCD5DC669A6CC001F4EB6D6AF85FF81F3F4F34C">>},
+ {<<"prometheus_httpd">>, <<"0BBE831452CFDF9588538EB2F570B26F30C348ADAE5E95A7D87F35A5910BCF92">>},
+ {<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>},
  {<<"ranch">>, <<"244EE3FA2A6175270D8E1FC59024FD9DBC76294A321057DE8F803B1479E76916">>},
  {<<"telemetry">>, <<"7015FC8919DBE63764F4B4B87A95B7C0996BD539E0D499BE6EC9D7F3875B79E6">>}]}
 ].

src/amoc_arsenal.app.src

@@ -6,28 +6,11 @@
     {applications, [
         kernel,
         stdlib,
-        exometer_core,
-        exometer_report_graphite,
+        prometheus,
+        prometheus_cowboy,
         amoc_rest,
         amoc
     ]},
-    {env, [
-        {exometer_predefined, [
-            {
-                [erlang, system_info],
-                {function, erlang, system_info, ['$dp'], value, [port_count, process_count]},
-                []
-            },
-            {
-                [erlang, memory],
-                {function, erlang, memory, ['$dp'], value, [
-                    total, processes, processes_used, system, binary, ets
-                ]},
-                []
-            },
-            {[amoc, users], {function, amoc_metrics, user_size, [], proplist, [size]}, []}
-        ]}
-    ]},
     {modules, []},
     {licenses, ["Apache-2.0"]},
     {links, []}

src/amoc_arsenal_app.erl

@@ -10,9 +10,9 @@
 -export([start/2, stop/1]).
 
 start(_StartType, _StartArgs) ->
-    {ok, _} = amoc_api:start(),
-    amoc_metrics:start(),
     amoc_logging:start(),
+    HasMetrics = amoc_metrics:start(),
+    {ok, _} = amoc_api:start(HasMetrics),
     amoc_arsenal_sup:start_link().
 
 stop(_State) ->