Setup New Relic Agent (#299)

* Setup New Relic Agent

* install docker

* increase test time

* remove telegraf

* setup monitoring for github workflows

provisioner/BUILD.bazel

@@ -7,6 +7,7 @@ pyinfra_run(
     name = "provision",
     srcs = [
         "connectors/teleport.py",
+        "deploys/docker/tasks/install_docker.py",
         "deploys/microk8s/tasks/install_microk8s.py",
         "deploys/monitoring/tasks/install_monitoring.py",
         "deploys/network/tasks/install_network.py",
@@ -18,18 +19,18 @@ pyinfra_run(
     ],
     args = [
         "--data install_network=True",
+        "--data install_docker=True",
         "--data install_monitoring=True",
         "--data install_microk8s=True",
         "--data install_teleport=True",
     ],
     data = [
         "deploys/microk8s/files/cmdline.txt",
-        "deploys/monitoring/files/json_file_output.conf",
-        "deploys/monitoring/files/logzio_output.conf.j2",
-        "deploys/monitoring/files/telegraf.conf.j2",
+        "deploys/monitoring/files/docker-compose.yml.j2",
+        "deploys/monitoring/files/newrelic-infra.yml.j2",
+        "deploys/monitoring/files/nri-prometheus-config.yaml",
         "deploys/network/files/99_config.yaml",
         "deploys/teleport/files/teleport.yaml.j2",
-        "deploys/teleport/files/teleport_health_check.conf",
     ],
     deploy = "deploy.py",
     env = {
@@ -139,6 +140,7 @@ task(
 
 task_test(
     name = "test",
+    size = "large",
     cmds = [
         "export CONTAINER_ID=$($run_dev_image)",
         {"defer": "docker rm -f $CONTAINER_ID"},

provisioner/deploy.py

@@ -2,6 +2,7 @@
 from provisioner.deploys.microk8s.tasks.install_microk8s import install_microk8s
 from provisioner.deploys.teleport.tasks.install_teleport import install_teleport
 from provisioner.deploys.monitoring.tasks.install_monitoring import install_monitoring
+from provisioner.deploys.docker.tasks.install_docker import install_docker
 from provisioner.utils import wait_for_reconnect
 
 from pyinfra import host
@@ -24,6 +25,9 @@
 if host.data.get("install_network"):
     install_network()
 
+if host.data.get("install_docker"):
+    install_docker()
+
 if host.data.get("install_monitoring"):
     install_monitoring()
 

provisioner/deploys/docker/tasks/install_docker.py

@@ -0,0 +1,65 @@
+from pyinfra.operations import apt, server, systemd
+from pyinfra import host
+from pyinfra.facts.server import LsbRelease
+from pyinfra.api.deploy import deploy
+from pyinfra.facts.deb import DebArch
+
+
+@deploy("Install Docker")
+def install_docker():
+    """
+    Install Docker on the target machine.
+    """
+
+    # From https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository
+
+    apt.packages(
+        name="Install apt requirements to use HTTPS",
+        packages=["curl", "ca-certificates", "gnupg"],
+        update=True,
+        cache_time=24 * 60 * 60,
+    )
+
+    server.shell(
+        name="Add Docker’s official GPG key",
+        commands=[
+            "install -m 0755 -d /etc/apt/keyrings",
+            "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --yes --dearmor -o /etc/apt/keyrings/docker.gpg",
+            "chmod a+r /etc/apt/keyrings/docker.gpg",
+        ],
+        _sudo=True,
+    )
+
+    lsb = host.get_fact(LsbRelease)
+    version_codename = lsb["codename"]
+    distro = lsb["id"].lower()
+    arch = host.get_fact(DebArch)
+
+    add_apt_repo = apt.repo(
+        name="Add the Docker apt repo",
+        src=(
+            "deb [arch={arch} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/{distro} {version_codename} stable"
+        ).format(arch=arch, distro=distro, version_codename=version_codename),
+        filename="docker-ce-stable",
+    )
+
+    apt.packages(
+        name="Install Docker via apt",
+        packages=[
+            "docker-ce",
+            "docker-ce-cli",
+            "containerd.io",
+            "docker-buildx-plugin",
+            "docker-compose-plugin",
+        ],
+        update=True,
+        cache_time=0 if add_apt_repo.changed else 24 * 60 * 60,
+    )
+
+    systemd.service(
+        name="Enable the docker service",
+        service="docker.service",
+        running=True,
+        enabled=True,
+        _sudo=True,
+    )

provisioner/deploys/monitoring/files/docker-compose.yml.j2

@@ -0,0 +1,24 @@
+name: monitoring
+version: '3.8'
+services:
+  # https://promhippie.github.io/github_exporter/#getting-started
+  github_exporter:
+    image: promhippie/github-exporter:latest@sha256:ad5cfc76d534d4c67ded2042b3ad343a8bcee9fd02f06638b39f74fbee17796e
+    restart: always
+    environment:
+      - GITHUB_EXPORTER_TOKEN={{ github_exporter_token }}
+      - GITHUB_EXPORTER_LOG_PRETTY=true
+      - GITHUB_EXPORTER_ORG=mvgijssel
+      - GITHUB_EXPORTER_REPO=mvgijssel/setup
+      - GITHUB_EXPORTER_COLLECTOR_WORKFLOWS=true
+      - GITHUB_EXPORTER_COLLECTOR_RUNNERS=false
+      - GITHUB_EXPORTER_COLLECTOR_BILLING=false
+      - GITHUB_EXPORTER_COLLECTOR_ADMIN=false
+
+  nri-prometheus:
+    image: newrelic/nri-prometheus:2.18.0@sha256:6b32ce98a098625b980342aae25634c7025d2dac996ac60642ffe0fc47e92bb9
+    restart: always
+    environment:
+      - LICENSE_KEY={{ new_relic_license_key }}
+    volumes:
+      - "/opt/monitoring/nri-prometheus-config.yaml:/config.yaml"

provisioner/deploys/monitoring/files/newrelic-infra.yml.j2

@@ -0,0 +1,5 @@
+license_key: "{{ new_relic_license_key }}"
+display_name: "{{ new_relic_display_name }}"
+custom_attributes:
+  env: "{{ setup_env }}"
+enable_process_metrics: true

provisioner/deploys/monitoring/files/nri-prometheus-config.yaml

@@ -0,0 +1,172 @@
+# The name of your cluster. It's important to match other New Relic products to relate the data.
+cluster_name: "provisioner"
+
+# When standalone is set to false nri-prometheus requires an infrastructure agent to work and send data. Defaults to true
+# standalone: true
+
+# How often the integration should run. Defaults to 30s.
+# scrape_duration: "30s"
+
+# The HTTP client timeout when fetching data from targets. Defaults to 5s.
+# scrape_timeout: "5s"
+
+# How old must the entries used for calculating the counters delta be
+# before the telemetry emitter expires them. Defaults to 5m.
+# telemetry_emitter_delta_expiration_age: "5m"
+
+# How often must the telemetry emitter check for expired delta entries.
+# Defaults to 5m.
+# telemetry_emitter_delta_expiration_check_interval: "5m"
+
+# Wether the integration should run in verbose mode or not. Defaults to false.
+verbose: false
+
+# Whether the integration should run in audit mode or not. Defaults to false.
+# Audit mode logs the uncompressed data sent to New Relic. Use this to log all data sent.
+# It does not include verbose mode. This can lead to a high log volume, use with care.
+audit: false
+
+# Wether the integration should skip TLS verification or not. Defaults to false.
+insecure_skip_verify: false
+
+# The label used to identify scrapable targets. Defaults to "prometheus.io/scrape".
+scrape_enabled_label: "prometheus.io/scrape"
+
+# scrape_services Allows to enable scraping the service and not the endpoints behind.
+# When endpoints are scraped this is no longer needed
+scrape_services: true
+
+# scrape_endpoints Allows to enable scraping directly endpoints instead of services as prometheus service natively does.
+# Please notice that depending on the number of endpoints behind a service the load can increase considerably
+scrape_endpoints: false
+
+# Whether k8s nodes need to be labelled to be scraped or not. Defaults to true.
+require_scrape_enabled_label_for_nodes: true
+# Number of worker threads used for scraping targets.
+# For large clusters with many (>400) targets, slowly increase until scrape
+# time falls between the desired `scrape_duration`.
+# Increasing this value too much will result in huge memory consumption if too
+# many metrics are being scraped.
+# Default: 4
+# worker_threads: 4
+
+# Maximum number of metrics to keep in memory until a report is triggered.
+# Changing this value is not recommended unless instructed by the New Relic support team.
+# max_stored_metrics: 10000
+
+# Minimum amount of time to wait between reports. Cannot be lowered than the default, 200ms.
+# Changing this value is not recommended unless instructed by the New Relic support team.
+# min_emitter_harvest_period: 200ms
+
+targets:
+  - description: github_exporter
+    urls: ["http://github_exporter:9504/metrics"]
+# targets:
+#   - description: Secure etcd example
+#     urls:
+#       [
+#         "https://192.168.3.1:2379",
+#         "https://192.168.3.2:2379",
+#         "https://192.168.3.3:2379",
+#       ]
+#     tls_config:
+#       ca_file_path: "/etc/etcd/etcd-client-ca.crt"
+#       cert_file_path: "/etc/etcd/etcd-client.crt"
+#       key_file_path: "/etc/etcd/etcd-client.key"
+# Proxy to be used by the emitters when submitting metrics. It should be
+# in the format [scheme]://[domain]:[port].
+# The emitter is the component in charge of sending the scraped metrics.
+# This proxy won't be used when scraping metrics from the targets.
+# By default it's empty, meaning that no proxy will be used.
+# emitter_proxy: "http://localhost:8888"
+
+# Certificate to add to the root CA that the emitter will use when
+# verifying server certificates.
+# If left empty, TLS uses the host's root CA set.
+# emitter_ca_file: "/path/to/cert/server.pem"
+
+# Set to true in order to stop autodiscovery in the k8s cluster. It can be useful when running the Pod with a service account
+# having limited privileges. Defaults to false.
+# disable_autodiscovery: false
+
+# Whether the emitter should skip TLS verification when submitting data.
+# Defaults to false.
+# emitter_insecure_skip_verify: false
+
+# Histogram support is based on New Relic's guidelines for higher
+# level metrics abstractions https://github.com/newrelic/newrelic-exporter-specs/blob/master/Guidelines.md.
+# To better support visualization of this data, percentiles are calculated
+# based on the histogram metrics and sent to New Relic.
+# By default, the following percentiles are calculated: 50, 95 and 99.
+#
+# percentiles:
+#   - 50
+#   - 95
+#   - 99
+
+# transformations:
+#   - description: "General processing rules"
+#     rename_attributes:
+#       - metric_prefix: ""
+#         attributes:
+#           container_name: "containerName"
+#           pod_name: "podName"
+#           namespace: "namespaceName"
+#           node: "nodeName"
+#           container: "containerName"
+#           pod: "podName"
+#           deployment: "deploymentName"
+#     ignore_metrics:
+#       # Ignore all the metrics except the ones listed below.
+#       # This is a list that complements the data retrieved by the New
+#       # Relic Kubernetes Integration, that's why Pods and containers are
+#       # not included, because they are already collected by the
+#       # Kubernetes Integration.
+#       - except:
+#         - kube_hpa_
+#         - kube_daemonset_
+#         - kube_statefulset_
+#         - kube_endpoint_
+#         - kube_service_
+#         - kube_limitrange
+#         - kube_node_
+#         - kube_poddisruptionbudget_
+#         - kube_resourcequota
+#         - nr_stats
+#     copy_attributes:
+#       # Copy all the labels from the timeseries with metric name
+#       # `kube_hpa_labels` into every timeseries with a metric name that
+#       # starts with `kube_hpa_` only if they share the same `namespace`
+#       # and `hpa` labels.
+#       - from_metric: "kube_hpa_labels"
+#         to_metrics: "kube_hpa_"
+#         match_by:
+#           - namespace
+#           - hpa
+#       - from_metric: "kube_daemonset_labels"
+#         to_metrics: "kube_daemonset_"
+#         match_by:
+#           - namespace
+#           - daemonset
+#       - from_metric: "kube_statefulset_labels"
+#         to_metrics: "kube_statefulset_"
+#         match_by:
+#           - namespace
+#           - statefulset
+#       - from_metric: "kube_endpoint_labels"
+#         to_metrics: "kube_endpoint_"
+#         match_by:
+#           - namespace
+#           - endpoint
+#       - from_metric: "kube_service_labels"
+#         to_metrics: "kube_service_"
+#         match_by:
+#           - namespace
+#           - service
+#       - from_metric: "kube_node_labels"
+#         to_metrics: "kube_node_"
+#         match_by:
+#           - namespace
+#           - node
+# integration definition files required to map metrics to entities
+# definition_files_path: /etc/newrelic-infra/definition-files