Nomad job example (#4469)

* Add nomad job microservices mode example

* Remove unused tag from query-frontend

* Update nomad example README.md

* Add link to nomad mem oversub in README.md file

* Remove tags from tempo.hcl. Update README.md

example/nomad/tempo-distributed/README.md

@@ -0,0 +1,44 @@
+# Microservices mode
+
+This Nomad job will deploy Tempo in
+[microservices mode](https://grafana.com/docs/tempo/latest/setup/deployment/#microservices-mode) using S3 backend.
+
+## Usage
+
+### Prerequisites
+- S3 compatible storage
+- [Nomad memory oversubscription](https://developer.hashicorp.com/nomad/tutorials/advanced-scheduling/memory-oversubscription). If memory oversubscription is not enabled, remove `memory_max` from tempo.hcl
+
+Have a look at the job file and Tempo configuration file and change it to suite your environment. (e.g. in `config.yml` change s3 endpoint to your s3 compatible storge, prometheus endpoint, etc...)
+
+Variables
+--------------
+
+| Name | Value | Description |
+|---|---|---|
+| version | Default = "2.3.1" | Tempo version |
+| s3_access_key_id | Default = "any" | S3 Access Key ID |
+| s3_secret_access_key | Default = "any" | S3 Secret Access Key |
+
+### Run job
+
+Inside directory with job run:
+
+```shell
+nomad job run tempo.hcl
+```
+
+To deploy a different version change `variable.version` default value or
+specify from command line:
+
+```shell
+nomad job run -var="version=2.6.1" tempo.hcl
+```
+
+### Scale Tempo
+
+Nomad CLI
+
+```shell
+nomad job scale tempo distributor <count>
+```

example/nomad/tempo-distributed/config.yml

@@ -0,0 +1,81 @@
+server:
+  log_level: info
+  http_listen_port: {{ env "NOMAD_PORT_http" }}
+  grpc_listen_port: {{ env "NOMAD_PORT_grpc" }}
+
+distributor:
+  ring:
+    kvstore:
+      store: consul
+      prefix: tempo/
+      consul:
+        host: {{ env "attr.unique.network.ip-address" }}:8500
+
+  receivers: # this configuration will listen on all ports and protocols that tempo is capable of.
+    otlp:
+      protocols:
+        http:
+        grpc:
+
+ingester:
+  max_block_duration: 5m
+  lifecycler:
+    ring:
+      kvstore:
+        store: consul
+        prefix: tempo/
+        consul:
+          host: {{ env "attr.unique.network.ip-address" }}:8500
+      replication_factor: 3
+
+compactor:
+  ring:
+    kvstore:
+      store: consul
+      prefix: tempo/
+      consul:
+        host: {{ env "attr.unique.network.ip-address" }}:8500
+
+  compaction:
+    block_retention: 336h  #Duration to keep blocks.  Default is 14 days (336h).
+
+querier:
+  frontend_worker:
+    frontend_address: tempo-query-frontend-grpc.service.consul:9095
+
+metrics_generator:
+  processor:
+    service_graphs:
+      max_items: 10000
+  ring:
+    kvstore:
+      store: consul
+      prefix: tempo/
+      consul:
+        host: {{ env "attr.unique.network.ip-address" }}:8500
+  storage:
+    path: {{ env "NOMAD_ALLOC_DIR" }}/tempo/wal
+    remote_write:
+      - url: http://prometheus.service.consul/api/v1/write
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: s3
+    wal:
+      path: {{ env "NOMAD_ALLOC_DIR" }}/tempo/wal
+    local:
+      path: {{ env "NOMAD_ALLOC_DIR" }}/tempo/blocks
+    s3:
+      bucket: tempo # how to store data in s3
+      endpoint: seaweedfs-s3.service.consul
+      insecure: true
+      access_key: ${S3_ACCESS_KEY_ID}
+      secret_key: ${S3_SECRET_ACCESS_KEY}
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors:
+        - service-graphs
+        - span-metrics