diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0417505f..31375626 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -136,8 +136,8 @@ jobs:
           TRACE_ID=$(grep "Location: /trace/" curl-output.http | cut -d/ -f3 | tr -d '\r')
 
           mkdir -p output/api/traces
-          for mode in ff{0,1,2,3}{0,1}; do
-            mode_trace=${mode}${TRACE_ID:4}
+          for mode in ff{0,1,2,3}{0,1}00000{0,1}; do
+            mode_trace=${mode}${TRACE_ID:10}
             curl -o output/api/traces/$mode_trace http://localhost:16686/api/traces/$mode_trace
           done
 
diff --git a/.github/workflows/chart-build.yml b/.github/workflows/chart-build.yml
index ddf09760..80296dd0 100644
--- a/.github/workflows/chart-build.yml
+++ b/.github/workflows/chart-build.yml
@@ -23,5 +23,5 @@ jobs:
         id: tag-name
         run: echo "IMAGE_TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
 
-      - run: helm package charts/kelemetry --app-version="${IMAGE_TAG}" --version="${{steps.tag-name.outputs.IMAGE_TAG}}" -d output
+      - run: helm package charts/kelemetry --app-version="${{steps.tag-name.outputs.IMAGE_TAG}}" --version="${{steps.tag-name.outputs.IMAGE_TAG}}" -d output
       - run: helm push output/kelemetry-chart-${{steps.tag-name.outputs.IMAGE_TAG}}.tgz oci://ghcr.io/kubewharf
diff --git a/Dockerfile b/Dockerfile
index 7d34b9c2..dcbb6891 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -17,5 +17,6 @@ COPY --from=build /src/kelemetry /usr/local/bin/kelemetry
 RUN mkdir -p /app/hack
 WORKDIR /app
 ADD hack/tfconfig.yaml hack/tfconfig.yaml
+RUN sed -i 's/127\.0\.0\.1:17272/remote-badger:17271/g' hack/tfconfig.yaml
 
 ENTRYPOINT ["kelemetry"]
diff --git a/Makefile b/Makefile
index 37aec68b..42256f97 100644
--- a/Makefile
+++ b/Makefile
@@ -12,8 +12,8 @@ CLUSTER_NAME ?= tracetest
 KUBECONFIGS ?= $(CLUSTER_NAME)=$(KUBECONFIG)
 
 PORT ?= 8080
-LOG_LEVEL ?= debug
-KLOG_VERBOSITY ?= 5
+LOG_LEVEL ?= info
+KLOG_VERBOSITY ?= 3
 
 RACE_ARG := -race
 ifdef SKIP_DETECT_RACE
@@ -133,13 +133,16 @@ kind:
 	sed "s/host.docker.internal/$$( \
 		docker network inspect kind -f '{{(index .IPAM.Config 0).Gateway}}' \
 	)/g" hack/audit-kubeconfig.yaml >hack/audit-kubeconfig.local.yaml
+	sed "s/host.docker.internal/$$( \
+		docker network inspect kind -f '{{(index .IPAM.Config 0).Gateway}}' \
+	)/g" hack/tracing-config.yaml >hack/tracing-config.local.yaml
 	cd hack && kind create cluster --config kind-cluster.yaml
 
 COMPOSE_COMMAND ?= up --build -d --remove-orphans
 
 stack:
-	docker-compose -f dev.docker-compose.yaml up --no-recreate --no-start # create network only
-	docker-compose \
+	docker compose -f dev.docker-compose.yaml up --no-recreate --no-start # create network only
+	docker compose \
 		-f dev.docker-compose.yaml \
 		-f <(jq -n \
 			--arg GATEWAY_ADDR $$(docker network inspect kelemetry_default -f '{{(index .IPAM.Config 0).Gateway}}') \
@@ -155,13 +158,13 @@ endef
 
 export QUICKSTART_JQ_PATCH
 quickstart:
-	docker-compose -f quickstart.docker-compose.yaml \
+	docker compose -f quickstart.docker-compose.yaml \
 		-f <(jq -n --arg KELEMETRY_IMAGE "$(KELEMETRY_IMAGE)" "$$QUICKSTART_JQ_PATCH") \
 		up --no-recreate --no-start
 	kubectl config view --raw --minify --flatten --merge >hack/client-kubeconfig.local.yaml
 	sed -i "s/0\.0\.0\.0/$$(docker network inspect kelemetry_default -f '{{(index .IPAM.Config 0).Gateway}}')/g" hack/client-kubeconfig.local.yaml
 	sed -i 's/certificate-authority-data: .*$$/insecure-skip-tls-verify: true/' hack/client-kubeconfig.local.yaml
-	docker-compose -f quickstart.docker-compose.yaml \
+	docker compose -f quickstart.docker-compose.yaml \
 		-f <(jq -n --arg KELEMETRY_IMAGE "$(KELEMETRY_IMAGE)" "$$QUICKSTART_JQ_PATCH") \
 		$(COMPOSE_COMMAND)
 
diff --git a/charts/kelemetry/Chart.yaml b/charts/kelemetry/Chart.yaml
index bf4fbaf5..80070bc4 100644
--- a/charts/kelemetry/Chart.yaml
+++ b/charts/kelemetry/Chart.yaml
@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.1.0"
+appVersion: "0.2.1"
diff --git a/charts/kelemetry/templates/_helpers.yaml b/charts/kelemetry/templates/_helpers.yaml
index 1a54ee37..2524c062 100644
--- a/charts/kelemetry/templates/_helpers.yaml
+++ b/charts/kelemetry/templates/_helpers.yaml
@@ -116,6 +116,7 @@ owner-linker-enable: {{ .Values.linkers.ownerReference }}
 
 {{/* TRACER */}}
 tracer-otel-endpoint: {{.Release.Name}}-collector.{{.Release.Namespace}}.svc:4317
+tracer-otel-insecure: {{ .Values.collector.insecure }}
 {{- end }}
 
 {{- define "kelemetry.object-cache-options" }}
@@ -177,13 +178,17 @@ kube-other-rest-burst: {{.otherClusterBurst}}
 {{- define "kelemetry.diff-cache-options-raw" }}
 diff-cache-wrapper-enable: {{.Values.diffCache.memoryWrapper}}
 
-{{- if .Values.diffCache.resourceVersionIndex | eq "Before" }}
-diff-cache-use-old-rv: true
-{{- else if .Values.diffCache.resourceVersionIndex | eq "After" }}
-diff-cache-use-old-rv: false
-{{- else }}
-{{ printf "Unsupported resource version index type %q" .Values.diffCache.resourceVersionIndex }}
-{{- end }}
+kube-use-old-resource-version-clusters: [
+  {{- range .Values.multiCluster.clusters }}
+  {{- if .resourceVersionIndex | eq "Before" }}
+  {{toJson .name}},
+  {{- else if .resourceVersionIndex | eq "After" }}
+  # not {{toJson .name}},
+  {{- else }}
+  {{ printf "Unsupported resource version index type %q" .resourceVersionIndex | fail }}
+  {{- end }}
+  {{- end }}
+]
 
 {{- if .Values.diffCache.type | eq "etcd" }}
 diff-cache: etcd
@@ -218,6 +223,16 @@ diff-cache-patch-ttl: {{toJson .Values.informers.diff.persistDuration.patch}}
 diff-cache-snapshot-ttl: {{toJson .Values.informers.diff.persistDuration.snapshot}}
 {{- end }}
 
+{{- define "kelemetry.diff-decorator-options" }}
+{{- include "kelemetry.diff-decorator-options-raw" . | include "kelemetry.yaml-to-args" }}
+{{- end }}
+{{- define "kelemetry.diff-decorator-options-raw" }}
+diff-decorator-enable: {{ .Values.consumer.diff.enable | toJson }}
+diff-decorator-fetch-backoff: {{ .Values.consumer.diff.backoff | toJson }}
+diff-decorator-fetch-event-timeout: {{ .Values.consumer.diff.fetchEventTimeout | toJson }}
+diff-decorator-fetch-total-timeout: {{ .Values.consumer.diff.fetchTotalTimeout | toJson }}
+{{- end }}
+
 {{- define "kelemetry.event-informer-options" }}
 {{- include "kelemetry.event-informer-options-raw" . | include "kelemetry.yaml-to-args" }}
 {{- end }}
@@ -291,14 +306,14 @@ jaeger-trace-cache-etcd-prefix: {{ .Values.frontend.traceCache.etcd.prefix | toJ
 {{- if list "badger" "memory" | has .Values.storageBackend.type }}
 {{- include "kelemetry.storage-options-stateful-grpc" . }}
 {{- else }}
-{{- include "kelemetry.storage-options-raw-stateless" . }}
+{{- include "kelemetry.storage-options-stateless-raw" . }}
 {{- end }}
 {{- end }}
 {{- define "kelemetry.storage-options-stateful-grpc" }}
 span-storage.type: grpc-plugin
 grpc-storage.server: {{.Release.Name}}-storage.{{.Release.Namespace}}.svc:17271
 {{- end }}
-{{- define "kelemetry.storage-options-stateless" }}
+{{- define "kelemetry.storage-options-stateless-raw" }}
 span-storage.type: {{toJson .Values.storageBackend.type}}
 {{- range $key, $value := .Values.storageBackend.options }}
 {{ toJson $key }}: {{ toJson $value }}
diff --git a/charts/kelemetry/templates/collector.deployment.yaml b/charts/kelemetry/templates/collector.deployment.yaml
index 8fa7b5f9..46f89869 100644
--- a/charts/kelemetry/templates/collector.deployment.yaml
+++ b/charts/kelemetry/templates/collector.deployment.yaml
@@ -24,6 +24,9 @@ spec:
           args: [
             {{- include "kelemetry.storage-options-raw" . | include "kelemetry.yaml-to-args" }}
           ]
+          env:
+            - name: COLLECTOR_OTLP_ENABLED
+              value: "true"
           image: {{ printf "%s:%s" .Values.jaegerImages.collector.repository .Values.jaegerImages.collector.tag | toJson }}
           imagePullPolicy: {{ toJson .Values.jaegerImages.pullPolicy }}
           livenessProbe:
diff --git a/charts/kelemetry/templates/consumer.deployment.yaml b/charts/kelemetry/templates/consumer.deployment.yaml
index 1c1bf9fb..5c6f8fd2 100644
--- a/charts/kelemetry/templates/consumer.deployment.yaml
+++ b/charts/kelemetry/templates/consumer.deployment.yaml
@@ -33,6 +33,10 @@ spec:
             {{ include "kelemetry.logging-options" .Values.consumer }}
             {{ include "kelemetry.kube-options" .Values.consumer }}
             {{ include "kelemetry.audit-options" . }}
+            {{- if .Values.informers.diff.enable }}
+            {{ include "kelemetry.diff-cache-options" . }}
+            {{ include "kelemetry.diff-decorator-options" . }}
+            {{- end }}
           ]
           ports: [
             {{- if .Values.consumer.pprof }}
diff --git a/charts/kelemetry/templates/informers.rbac.yaml b/charts/kelemetry/templates/informers.rbac.yaml
index f2d26186..56d0a961 100644
--- a/charts/kelemetry/templates/informers.rbac.yaml
+++ b/charts/kelemetry/templates/informers.rbac.yaml
@@ -34,3 +34,6 @@ rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["*"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["*"]
diff --git a/charts/kelemetry/templates/storage.deployment.yaml b/charts/kelemetry/templates/storage.deployment.yaml
index 0d297408..4f54741c 100644
--- a/charts/kelemetry/templates/storage.deployment.yaml
+++ b/charts/kelemetry/templates/storage.deployment.yaml
@@ -27,7 +27,7 @@ spec:
           imagePullPolicy: {{ toJson .Values.jaegerImages.pullPolicy }}
           args: [
             # Don't include storage-options-raw directly here; otherwise the traffic would be infinite recursion.
-            {{- include "kelemetry.storage-options-stateless" . | include "kelemetry.yaml-to-args" }}
+            {{- include "kelemetry.storage-options-stateless-raw" . | include "kelemetry.yaml-to-args" }}
           ]
           livenessProbe:
             httpGet:
diff --git a/charts/kelemetry/values.yaml b/charts/kelemetry/values.yaml
index f1bdd40b..c29405f5 100644
--- a/charts/kelemetry/values.yaml
+++ b/charts/kelemetry/values.yaml
@@ -139,6 +139,24 @@ consumer:
         otherConfig: {}
           # clusterIP: xxx, externalIP: xxx, etc.
 
+  # Decorate audit logs with diff info from the diff cache.
+  diff:
+    enable: true
+    # The backoff duration between attempts to fetch diff from the diff cache.
+    backoff: 1s
+    # If this duration has elapsed since kube-apiserver sends the ResponseComplete audit stage
+    # but the diff cache still returns NotFound for the requested diff,
+    # consumer gives up retrying to fetch diff for this audit event.
+    # Consumer always tries to fetch at least once even if this duration has elapsed.
+    fetchEventTimeout: 15s
+    # The total duration that consumer takes to retry and fetch diff for each audit event.
+    # Comparing fetchEventTimeout vs fetchTotalTimeout:
+    # fetchEventTimeout is non-accumulative, e.g. if consumer is lagging behind for more than fetchEventTimeout,
+    # the consumer give up after the first attempt;
+    # meanwhile, fetchTotalTimeout is accumulative, e.g. in the worst case,
+    # the consumer takes fetchTotalTimeout to process each audit event before processing the next one.
+    fetchTotalTimeout: 10s
+
   # The audit consumer is primarily CPU-bound.
   resources: {}
     # limits:
@@ -168,6 +186,8 @@ consumer:
 
 # Jaeger collector collects otel tracing data and dispatches them to Jaeger storage.
 collector:
+  insecure: true
+
   replicaCount: 3
   resources: {}
     # limits:
@@ -270,8 +290,7 @@ storageBackend:
 aggregator:
   globalTags:
     # Tags applied to all object spans (i.e. not actual events, just the parent placeholder)
-    pseudoSpan:
-      cluster: foo # one recommended use is to write the cluster name here.
+    pseudoSpan: {}
     # Tags applied to all actual spans (e.g. events, audit logs)
     eventSpan: {}
 
@@ -322,14 +341,6 @@ objectCache:
 
 # Diff cache stores the object diff from informers so that audit consumer can use it.
 diffCache:
-  # Whether to index by the resourceVersion `Before` or `After` a diff.
-  # `After` is more accurate, but it requires audit logs to be sent at RequestResponse level,
-  # which may result in more expensive audit logging costs.
-  # Use `Before` if you cannot use RequestResponse-level audit logging,
-  # but it may suffer from accuracy bugs such as duplicate events.
-  # See <https://github.com/kubernetes/kubernetes/issues/115791>.
-  resourceVersionIndex: Before
-
   # Whether to persist a layer of read cache in memory to reduce etcd load.
   memoryWrapper: true
 
@@ -370,9 +381,17 @@ multiCluster:
       # This is the list of addresses that audit webhook requests from the apiserver may be sent from.
       peerAddresses: [127.0.0.1]
 
+      # Whether to index object diff in this cluster by the resourceVersion `Before` or `After` a diff.
+      # `After` is more accurate, but it requires audit logs to be sent at RequestResponse level,
+      # which may result in more expensive audit logging costs.
+      # Use `Before` if you cannot use RequestResponse-level audit logging,
+      # but it may suffer from accuracy bugs such as duplicate events.
+      # See <https://github.com/kubernetes/kubernetes/issues/115791>.
+      resourceVersionIndex: After
+
 kelemetryImage:
   repository: ghcr.io/kubewharf/kelemetry
-  pullPolicy: IfNotPresent
+  pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
   tag: ""
   pullSecrets: []
diff --git a/dev.docker-compose.yaml b/dev.docker-compose.yaml
index 87bef0d0..8f66a2b3 100644
--- a/dev.docker-compose.yaml
+++ b/dev.docker-compose.yaml
@@ -26,8 +26,8 @@ services:
   jaeger-query:
     image: jaegertracing/jaeger-query:1.42
     environment:
-      GRPC_STORAGE_SERVER: host.docker.internal:17271 # run on host directly
       SPAN_STORAGE_TYPE: grpc-plugin
+      GRPC_STORAGE_SERVER: host.docker.internal:17271 # run on host directly
     ports:
       - 0.0.0.0:16686:16686
     restart: always
@@ -39,7 +39,7 @@ services:
       SPAN_STORAGE_TYPE: grpc-plugin
       GRPC_STORAGE_SERVER: remote-badger:17271
     ports:
-      - 127.0.0.1:4317:4317
+      - 0.0.0.0:4317:4317
     restart: always
   # Backend badger storage
   # Feel free to override environment.SPAN_STORAGE_TYPE to other storages given the proper configuration.
@@ -55,6 +55,16 @@ services:
     volumes:
       - badger:/mnt/badger
 
+  # Web frontend for raw trace database view.
+  jaeger-query-raw:
+    image: jaegertracing/jaeger-query:1.42
+    environment:
+      SPAN_STORAGE_TYPE: grpc-plugin
+      GRPC_STORAGE_SERVER: remote-badger:17271
+    ports:
+      - 0.0.0.0:26686:16686
+    restart: always
+
 volumes:
   etcd: {}
   badger: {}
diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md
index 6167217f..62db0425 100644
--- a/docs/DEPLOY.md
+++ b/docs/DEPLOY.md
@@ -1,5 +1,11 @@
 # Deploying Kelemetry for Production Clusters
 
+> Note: Due to the variety of cloud providers and cluster management solutions,
+> deploying Kelemetry for production might be tricky.
+> If you just want to try out the features of Kelemetry,
+> follow the [quick start guide](QUICK_START.md) instead,
+> which sets up a basic stack locally using Docker.
+
 To minimize data loss and latency and ensure high availability,
 we recommend deploying Kelemetry in 3 separate components:
 consumers, informers and storage plugin.
@@ -61,8 +67,11 @@ This setup is bundled into a Helm chart.
 
 ## Steps
 
-1. Download [`values.yaml`](charts/kelemetry/values.yaml) and configure the settings.
-2. Install the chart: `helm install kelemetry kelemetry oci://ghcr.io/kubewharf/kelemetry-chart --values values.yaml`
+1. Download [`values.yaml`](/charts/kelemetry/values.yaml) and configure the settings.
+2. Install the chart: `helm install kelemetry oci://ghcr.io/kubewharf/kelemetry-chart --values values.yaml`
+3. If you use an audit webhook directly, remember to
+   [configure the apiserver](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#webhook-backend)
+   to send audit logs to the webhook:
 
 The default configuration is designed for single-cluster deployment.
 For multi-cluster deployment, configure the `sharedEtcd` and `storageBackend` to use a common database.
diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md
index 3a6860b5..ea1417fd 100644
--- a/docs/USER_GUIDE.md
+++ b/docs/USER_GUIDE.md
@@ -40,12 +40,10 @@ Currently, the following tags are supported:
 
 ### Time
 
-Currently, all traces are **rounded down** to the **newest half-hour before the event**.
-That is, if you want to look for an event that happened at 12:34,
-you should search for the trace at 12:30 instead.
-Searching between 12:33 and 12:35 will yield **no search results**.
-
-Each trace lasts for exactly 30 minutes, so the max/min duration fields are unsupported.
+Kelemetry merges and truncates traces based on the time range given in the user input.
+Only spans and events within this range are displayed.
+Some display modes further truncate the time range to the duration from the earliest to the latest event,
+so refer to the "Trace start" timestamp indicated in the trace view page.
 
 ## Trace view
 
@@ -54,7 +52,7 @@ If the current half-hour is still in progress, reload the page to load the new d
 
 For the recommended `tracing` display mode,
 
-- The timestamps are relative to the trace start, which is either `:00` or `:30` of an hour.
+- The timestamps are relative to the trace start.
 - Click on the arrow button on the left to collapse/expand a span.
 - Click on the empty space on a span row to reveal details of the span.
 - Hover cursor over a black vertical line on the span to reveal the events.
diff --git a/go.mod b/go.mod
index 4261fb95..0186acaf 100644
--- a/go.mod
+++ b/go.mod
@@ -4,10 +4,11 @@ go 1.20
 
 require (
 	github.com/coocood/freecache v1.2.3
+	github.com/dlclark/regexp2 v1.10.0
 	github.com/gin-gonic/gin v1.9.1
 	github.com/go-logr/logr v1.2.4
-	github.com/jaegertracing/jaeger v1.46.0
-	github.com/pelletier/go-toml/v2 v2.0.8
+	github.com/jaegertracing/jaeger v1.48.0
+	github.com/pelletier/go-toml/v2 v2.0.9
 	github.com/prometheus/client_golang v1.16.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/pflag v1.0.5
@@ -19,14 +20,14 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0
 	go.opentelemetry.io/otel/sdk v1.16.0
 	go.opentelemetry.io/otel/trace v1.16.0
-	go.uber.org/zap v1.24.0
-	google.golang.org/grpc v1.56.1
-	k8s.io/api v0.27.3
-	k8s.io/apimachinery v0.27.3
-	k8s.io/apiserver v0.27.3
-	k8s.io/client-go v0.27.3
+	go.uber.org/zap v1.25.0
+	google.golang.org/grpc v1.57.0
+	k8s.io/api v0.28.1
+	k8s.io/apimachinery v0.28.1
+	k8s.io/apiserver v0.28.1
+	k8s.io/client-go v0.28.1
 	k8s.io/klog/v2 v2.100.1
-	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
 )
 
 require (
@@ -48,7 +49,7 @@ require (
 	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/fatih/color v1.14.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
@@ -69,17 +70,16 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v23.1.21+incompatible // indirect
-	github.com/google/gnostic v0.6.9 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
-	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-hclog v1.4.0 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-plugin v1.4.8 // indirect
+	github.com/hashicorp/go-plugin v1.4.10 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/yamux v0.1.1 // indirect
@@ -91,7 +91,7 @@ require (
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
@@ -100,7 +100,7 @@ require (
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -111,7 +111,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/common v0.43.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
@@ -128,27 +128,30 @@ require (
 	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
 	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.1-0.20230612162650-64be7e574a17 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.7.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/term v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/crypto v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
+	golang.org/x/oauth2 v0.8.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/term v0.11.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
diff --git a/go.sum b/go.sum
index 13f3275a..f9ff64bd 100644
--- a/go.sum
+++ b/go.sum
@@ -17,12 +17,15 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb
 cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
 cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
 cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
+cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY=
+cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
@@ -55,7 +58,6 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE
 github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
 github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
@@ -83,6 +85,7 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
 github.com/coocood/freecache v1.2.3 h1:lcBwpZrwBZRZyLk/8EMyQVXRiFl663cCuMOrjCALeto=
 github.com/coocood/freecache v1.2.3/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -104,7 +107,8 @@ github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWa
 github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
@@ -124,12 +128,12 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
-github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8=
+github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
+github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
 github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
-github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -164,7 +168,7 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
 github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/gocql/gocql v1.3.2 h1:ox3T+R7VFibHSIGxRkuUi1uIvAv8jBHCWxc+9aFQ/LA=
@@ -216,8 +220,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/flatbuffers v23.1.21+incompatible h1:bUqzx/MXCDxuS0hRJL2EfjyZL3uQrPbMocUa8zGqsTA=
 github.com/google/flatbuffers v23.1.21+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
-github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0=
-github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E=
+github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
+github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -262,19 +266,17 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08=
-github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
-github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-hclog v1.4.0 h1:ctuWFGrhFha8BnnzxqeRGidlEcQkDyL5u8J8t5eA11I=
-github.com/hashicorp/go-hclog v1.4.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
+github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-plugin v1.4.8 h1:CHGwpxYDOttQOY7HOWgETU9dyVjOXzniXDqJcYJE1zM=
-github.com/hashicorp/go-plugin v1.4.8/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s=
+github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk=
+github.com/hashicorp/go-plugin v1.4.10/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -290,8 +292,8 @@ github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
-github.com/jaegertracing/jaeger v1.46.0 h1:lXr8LTbj0kWjTAhR2FXTnMvVC3N93NRdaBEnqlu096Q=
-github.com/jaegertracing/jaeger v1.46.0/go.mod h1:MCjqgGvkrjG9P0zD7gc8TwRpqZMEdwhjogyywHNJ+FU=
+github.com/jaegertracing/jaeger v1.48.0 h1:YuKooQ7qJsjgxws9xuf8C/BLNTPx8qTAJz4wv7IHhSc=
+github.com/jaegertracing/jaeger v1.48.0/go.mod h1:BoAPkdCAIEuLsVz/EDhjXd+GSVpHtJhiGqWoFEvBCKg=
 github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
 github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
 github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
@@ -314,14 +316,13 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
-github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
-github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
 github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -350,8 +351,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
 github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
-github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY=
+github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -364,13 +365,13 @@ github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DV
 github.com/olivere/elastic v6.2.37+incompatible h1:UfSGJem5czY+x/LqxgeCBgjDn6St+z8OnsCuxwD3L0U=
 github.com/olivere/elastic v6.2.37+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
 github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
-github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk=
-github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E=
+github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
+github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
-github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
+github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0=
+github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
 github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -383,8 +384,8 @@ github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lF
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
 github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
-github.com/prometheus/common v0.43.0 h1:iq+BVjvYLei5f27wiuNiB1DN6DYQkp1c8Bx0Vykh5us=
-github.com/prometheus/common v0.43.0/go.mod h1:NCvr5cQIh3Y/gy73/RdVtC9r8xxrxwJnB+2lB3BxrFc=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
@@ -415,7 +416,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc=
 github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
@@ -430,7 +430,6 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
@@ -450,9 +449,6 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -473,6 +469,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.1-0.20230612162650-64be7e574a17 h1:mdcNStUIXngF/mH3xxAo4nbR4g65IXqLL1SvYMjz7JQ=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.1-0.20230612162650-64be7e574a17/go.mod h1:N2Nw/UmmvQn0yCnaUzvsWzTWIeffYIdFteg6mxqCWII=
 go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
 go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
 go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0=
@@ -495,8 +493,8 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
-go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
+go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
+go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
 golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
@@ -510,8 +508,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
-golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -579,13 +577,12 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -596,8 +593,8 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
-golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
+golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
+golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -661,13 +658,13 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -679,8 +676,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -736,7 +733,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
+golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -806,9 +803,12 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
+google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M=
+google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk=
+google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -829,8 +829,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
-google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
+google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -844,8 +844,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -862,7 +862,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -873,20 +872,20 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.27.3 h1:yR6oQXXnUEBWEWcvPWS0jQL575KoAboQPfJAuKNrw5Y=
-k8s.io/api v0.27.3/go.mod h1:C4BNvZnQOF7JA/0Xed2S+aUyJSfTGkGFxLXz9MnpIpg=
-k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM=
-k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E=
-k8s.io/apiserver v0.27.3 h1:AxLvq9JYtveYWK+D/Dz/uoPCfz8JC9asR5z7+I/bbQ4=
-k8s.io/apiserver v0.27.3/go.mod h1:Y61+EaBMVWUBJtxD5//cZ48cHZbQD+yIyV/4iEBhhNA=
-k8s.io/client-go v0.27.3 h1:7dnEGHZEJld3lYwxvLl7WoehK6lAq7GvgjxpA3nv1E8=
-k8s.io/client-go v0.27.3/go.mod h1:2MBEKuTo6V1lbKy3z1euEGnhPfGZLKTS9tiJ2xodM48=
+k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108=
+k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg=
+k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY=
+k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw=
+k8s.io/apiserver v0.28.1 h1:dw2/NKauDZCnOUAzIo2hFhtBRUo6gQK832NV8kuDbGM=
+k8s.io/apiserver v0.28.1/go.mod h1:d8aizlSRB6yRgJ6PKfDkdwCy2DXt/d1FDR6iJN9kY1w=
+k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8=
+k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
 k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg=
-k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg=
-k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk=
-k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
+k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
+k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
+k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
diff --git a/hack/kind-cluster.yaml b/hack/kind-cluster.yaml
index 1ac0a20f..67783811 100644
--- a/hack/kind-cluster.yaml
+++ b/hack/kind-cluster.yaml
@@ -7,15 +7,15 @@ networking:
   apiServerAddress: "0.0.0.0"
 nodes:
   - role: control-plane
-    image: kindest/node:v1.25.3
+    image: kindest/node:v1.27.3
     kubeadmConfigPatches:
       - |
         kind: ClusterConfiguration
         apiServer:
           extraArgs:
             audit-webhook-config-file: /mnt/audit/audit-kubeconfig.local.yaml
+            tracing-config-file: /mnt/audit/tracing-config.local.yaml
             audit-policy-file: /mnt/audit/audit-policy.yaml
-            feature-gates: ServerSideApply=false
           extraVolumes:
             - name: audit-config
               hostPath: /mnt/audit
@@ -27,4 +27,4 @@ nodes:
         containerPath: /mnt/audit
         readOnly: true
   - role: worker
-    image: kindest/node:v1.25.3
+    image: kindest/node:v1.27.3
diff --git a/hack/tfconfig.yaml b/hack/tfconfig.yaml
index 0684f998..2be80224 100644
--- a/hack/tfconfig.yaml
+++ b/hack/tfconfig.yaml
@@ -1,3 +1,4 @@
+defaultConfig: "20000000"
 configs:
   - id: "00000000"
     name: "tree"
@@ -12,7 +13,7 @@ configs:
       - kind: Batch
         batchName: initial
       - kind: ExtractNestingVisitor
-        batchName:
+        matchesNestLevel:
           oneOf: []
           negate: true
       - kind: Batch
@@ -51,7 +52,53 @@ configs:
         batchName: final
 
 modifiers:
-  exclusive: "01000000"
+  "01000000":
+    displayName: exclusive
+    modifierName: exclusive
+
+# Uncomment to enable extension trace from apiserver
+#   "00000001":
+#     # We want to run extension modifiers after exclusive modifier to avoid fetching unused traces
+#     displayName: apiserver trace
+#     modifierName: extension
+#     args:
+#       kind: JaegerStorage
+#       storageArgs:
+#         span-storage.type: grpc-plugin
+#         grpc-storage.server: 127.0.0.1:17272
+#       service: apiserver
+#       operation: Update
+#       tagTemplates:
+#         audit-id: "{{.auditId}}"
+#       numTracesLimit: 5
+#
+#       forAuditEvent: true
+#
+#       # Restrict the total wall time duration for performing queries for this extension on each trace
+#       totalTimeout: 15s
+#       # Restrict the number of concurrent queries
+#       maxConcurrency: 4
+
+# http trace extension demo
+#   "00000002":
+#     displayName: http trace extension
+#     modifierName: extension
+#     args:
+#       kind: HTTPTrace
+#       traceBackends:
+#       - tagFilters:
+#           resource: pods
+#           nodes: ".+"
+#         argsTemplates:
+#           node: "{{.nodes}}"
+#           pod: "{{.namespace}}/{{.name}}"
+#           start: "{{unixMicro .start}}"
+#           end: "{{unixMicro .end}}"
+#           limit: "100"
+#         urlTemplate: "http://test-domain/api/traces"
+#         forObject: true
+#       maxConcurrency: 100
+#       totalTimeout: 60s
 
 batches:
   - name: initial
@@ -102,6 +149,10 @@ batches:
               fields:
                 - "metadata.resourceVersion"
                 - "metadata.generation"
+            - class:
+                shouldDisplay: false
+              fields:
+                - "metadata.managedFields"
         logTypeMapping:
           event/message: "message"
           audit/objectSnapshot: "snapshot"
diff --git a/hack/tracing-config.yaml b/hack/tracing-config.yaml
new file mode 100644
index 00000000..d340e39f
--- /dev/null
+++ b/hack/tracing-config.yaml
@@ -0,0 +1,4 @@
+apiVersion: apiserver.config.k8s.io/v1beta1
+kind: TracingConfiguration
+endpoint: host.docker.internal:4317
+samplingRatePerMillion: 1000000
diff --git a/pkg/aggregator/aggregator.go b/pkg/aggregator/aggregator.go
index 38bcae24..4c7c976c 100644
--- a/pkg/aggregator/aggregator.go
+++ b/pkg/aggregator/aggregator.go
@@ -33,7 +33,7 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/aggregator/tracer"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 	"github.com/kubewharf/kelemetry/pkg/util/zconstants"
 )
 
@@ -109,7 +109,7 @@ type Aggregator interface {
 	// it waits for the primary event to be created and takes it as the parent.
 	// If the primary event does not get created after options.subObjectPrimaryBackoff, this event is promoted as primary.
 	// If multiple primary events are sent, the slower one (by SpanCache-authoritative timing) is demoted.
-	Send(ctx context.Context, object util.ObjectRef, event *aggregatorevent.Event, subObjectId *SubObjectId) error
+	Send(ctx context.Context, object utilobject.Rich, event *aggregatorevent.Event, subObjectId *SubObjectId) error
 }
 
 type SubObjectId struct {
@@ -183,7 +183,7 @@ func (aggregator *aggregator) Close(ctx context.Context) error { return nil }
 
 func (aggregator *aggregator) Send(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Rich,
 	event *aggregatorevent.Event,
 	subObjectId *SubObjectId,
 ) (err error) {
@@ -370,22 +370,16 @@ func (aggregator *aggregator) Send(
 
 func (aggregator *aggregator) ensureObjectSpan(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Rich,
 	eventTime time.Time,
 ) (tracer.SpanContext, error) {
 	return aggregator.getOrCreateSpan(ctx, object, eventTime, func() (_ tracer.SpanContext, err error) {
 		// try to associate a parent object
-		var parent *util.ObjectRef
+		var parent *utilobject.Rich
 
 		for _, linker := range aggregator.Linkers.Impls {
-			thisParent, err := linker.Lookup(ctx, object)
-			if err != nil {
-				aggregator.Logger.WithField("linker", fmt.Sprintf("%T", linker)).WithError(err).Error("linker error")
-				continue
-			}
-
-			if thisParent != nil {
-				parent = thisParent
+			parent = linker.Lookup(ctx, object)
+			if parent != nil {
 				break
 			}
 		}
@@ -401,7 +395,7 @@ func (aggregator *aggregator) ensureObjectSpan(
 
 func (aggregator *aggregator) getOrCreateSpan(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Rich,
 	eventTime time.Time,
 	parentGetter func() (tracer.SpanContext, error),
 ) (tracer.SpanContext, error) {
@@ -536,7 +530,7 @@ func (aggregator *aggregator) getOrCreateSpan(
 
 func (aggregator *aggregator) createSpan(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Rich,
 	nestLevel string,
 	eventTime time.Time,
 	parent tracer.SpanContext,
@@ -586,11 +580,11 @@ func (aggregator *aggregator) createSpan(
 	return spanContext, nil
 }
 
-func (aggregator *aggregator) expiringSpanCacheKey(object util.ObjectRef, timestamp time.Time) string {
+func (aggregator *aggregator) expiringSpanCacheKey(object utilobject.Rich, timestamp time.Time) string {
 	expiringWindow := timestamp.Unix() / int64(aggregator.options.spanTtl.Seconds())
 	return aggregator.spanCacheKey(object, fmt.Sprintf("field=object,window=%d", expiringWindow))
 }
 
-func (aggregator *aggregator) spanCacheKey(object util.ObjectRef, subObjectId string) string {
+func (aggregator *aggregator) spanCacheKey(object utilobject.Rich, subObjectId string) string {
 	return fmt.Sprintf("%s/%s", object.String(), subObjectId)
 }
diff --git a/pkg/aggregator/aggregatorevent/event.go b/pkg/aggregator/aggregatorevent/event.go
index 15e1c77a..1845ee4b 100644
--- a/pkg/aggregator/aggregatorevent/event.go
+++ b/pkg/aggregator/aggregatorevent/event.go
@@ -48,16 +48,16 @@ func (event *Event) GetEndTime() time.Time {
 	return event.Time.Add(zconstants.DummyDuration)
 }
 
-func (event *Event) WithEndTime(when time.Time) *Event {
+func (event *Event) SetEndTime(when time.Time) *Event {
 	event.EndTime = &when
 	return event
 }
 
-func (event *Event) WithDuration(duration time.Duration) *Event {
-	return event.WithEndTime(event.Time.Add(duration))
+func (event *Event) SetDuration(duration time.Duration) *Event {
+	return event.SetEndTime(event.Time.Add(duration))
 }
 
-func (event *Event) WithTag(key string, value any) *Event {
+func (event *Event) SetTag(key string, value any) *Event {
 	event.Tags[key] = value
 	return event
 }
diff --git a/pkg/aggregator/eventdecorator/decorator.go b/pkg/aggregator/eventdecorator/decorator.go
index b5d627d9..fadbe09d 100644
--- a/pkg/aggregator/eventdecorator/decorator.go
+++ b/pkg/aggregator/eventdecorator/decorator.go
@@ -18,9 +18,9 @@ import (
 	"context"
 
 	"github.com/kubewharf/kelemetry/pkg/aggregator/aggregatorevent"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 type Decorator interface {
-	Decorate(ctx context.Context, object util.ObjectRef, event *aggregatorevent.Event)
+	Decorate(ctx context.Context, object utilobject.Rich, event *aggregatorevent.Event)
 }
diff --git a/pkg/aggregator/eventdecorator/eventtagger/eventtagger.go b/pkg/aggregator/eventdecorator/eventtagger/eventtagger.go
index fd89376a..f6a97ad5 100644
--- a/pkg/aggregator/eventdecorator/eventtagger/eventtagger.go
+++ b/pkg/aggregator/eventdecorator/eventtagger/eventtagger.go
@@ -24,7 +24,7 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/aggregator/eventdecorator"
 	"github.com/kubewharf/kelemetry/pkg/aggregator/resourcetagger"
 	"github.com/kubewharf/kelemetry/pkg/manager"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func init() {
@@ -71,7 +71,7 @@ func (d *eventTagDecorator) Init() error {
 func (d *eventTagDecorator) Start(ctx context.Context) error { return nil }
 func (d *eventTagDecorator) Close(ctx context.Context) error { return nil }
 
-func (d *eventTagDecorator) Decorate(ctx context.Context, object util.ObjectRef, event *aggregatorevent.Event) {
+func (d *eventTagDecorator) Decorate(ctx context.Context, object utilobject.Rich, event *aggregatorevent.Event) {
 	if event == nil {
 		return
 	}
diff --git a/pkg/aggregator/linker/linker.go b/pkg/aggregator/linker/linker.go
index f8e43742..f648be25 100644
--- a/pkg/aggregator/linker/linker.go
+++ b/pkg/aggregator/linker/linker.go
@@ -17,9 +17,9 @@ package linker
 import (
 	"context"
 
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 type Linker interface {
-	Lookup(ctx context.Context, object util.ObjectRef) (*util.ObjectRef, error)
+	Lookup(ctx context.Context, object utilobject.Rich) *utilobject.Rich
 }
diff --git a/pkg/aggregator/objectspandecorator/decorator.go b/pkg/aggregator/objectspandecorator/decorator.go
index 3443a96d..3c6662e9 100644
--- a/pkg/aggregator/objectspandecorator/decorator.go
+++ b/pkg/aggregator/objectspandecorator/decorator.go
@@ -17,9 +17,9 @@ package objectspandecorator
 import (
 	"context"
 
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 type Decorator interface {
-	Decorate(ctx context.Context, object util.ObjectRef, traceSource string, tags map[string]string)
+	Decorate(ctx context.Context, object utilobject.Rich, traceSource string, tags map[string]string)
 }
diff --git a/pkg/aggregator/objectspandecorator/resourcetagger/objecttagger.go b/pkg/aggregator/objectspandecorator/resourcetagger/objecttagger.go
index 0cf723d7..ee4a00b7 100644
--- a/pkg/aggregator/objectspandecorator/resourcetagger/objecttagger.go
+++ b/pkg/aggregator/objectspandecorator/resourcetagger/objecttagger.go
@@ -23,7 +23,7 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/aggregator/objectspandecorator"
 	"github.com/kubewharf/kelemetry/pkg/aggregator/resourcetagger"
 	"github.com/kubewharf/kelemetry/pkg/manager"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func init() {
@@ -54,7 +54,7 @@ func (d *ObjectSpanTag) Init() error                     { return nil }
 func (d *ObjectSpanTag) Start(ctx context.Context) error { return nil }
 func (d *ObjectSpanTag) Close(ctx context.Context) error { return nil }
 
-func (d *ObjectSpanTag) Decorate(ctx context.Context, object util.ObjectRef, traceSource string, tags map[string]string) {
+func (d *ObjectSpanTag) Decorate(ctx context.Context, object utilobject.Rich, traceSource string, tags map[string]string) {
 	if tags == nil {
 		return
 	}
diff --git a/pkg/aggregator/resourcetagger/resource_tagger.go b/pkg/aggregator/resourcetagger/resource_tagger.go
index b83e4722..0646494e 100644
--- a/pkg/aggregator/resourcetagger/resource_tagger.go
+++ b/pkg/aggregator/resourcetagger/resource_tagger.go
@@ -31,7 +31,7 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/k8s/objectcache"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func init() {
@@ -133,7 +133,7 @@ func (d *ResourceTagger) registerResource(gr schema.GroupResource, tagPathMappin
 	}
 }
 
-func (d *ResourceTagger) DecorateTag(ctx context.Context, object util.ObjectRef, traceSource string, tags map[string]any) {
+func (d *ResourceTagger) DecorateTag(ctx context.Context, object utilobject.Rich, traceSource string, tags map[string]any) {
 	if tags == nil {
 		return
 	}
@@ -153,7 +153,7 @@ func (d *ResourceTagger) DecorateTag(ctx context.Context, object util.ObjectRef,
 		logger.Debug("Fetching dynamic object for tag decorator")
 
 		var err error
-		raw, err = d.ObjectCache.Get(ctx, object)
+		raw, err = d.ObjectCache.Get(ctx, object.VersionedKey)
 		if err != nil {
 			tagMetric.Result = "FetchErr"
 			logger.WithError(err).Error("cannot fetch object value")
diff --git a/pkg/annotationlinker/linker.go b/pkg/annotationlinker/linker.go
index 729d2e59..46841d36 100644
--- a/pkg/annotationlinker/linker.go
+++ b/pkg/annotationlinker/linker.go
@@ -17,18 +17,16 @@ package annotationlinker
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 
 	"github.com/kubewharf/kelemetry/pkg/aggregator/linker"
 	"github.com/kubewharf/kelemetry/pkg/k8s"
 	"github.com/kubewharf/kelemetry/pkg/k8s/discovery"
 	"github.com/kubewharf/kelemetry/pkg/k8s/objectcache"
 	"github.com/kubewharf/kelemetry/pkg/manager"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func init() {
@@ -60,20 +58,25 @@ func (ctrl *controller) Init() error                     { return nil }
 func (ctrl *controller) Start(ctx context.Context) error { return nil }
 func (ctrl *controller) Close(ctx context.Context) error { return nil }
 
-func (ctrl *controller) Lookup(ctx context.Context, object util.ObjectRef) (*util.ObjectRef, error) {
+func (ctrl *controller) Lookup(ctx context.Context, object utilobject.Rich) *utilobject.Rich {
+	raw := object.Raw
+
 	logger := ctrl.Logger.WithFields(object.AsFields("object"))
 
-	raw := object.Raw
 	if raw == nil {
 		logger.Debug("Fetching dynamic object")
 
 		var err error
-		raw, err = ctrl.ObjectCache.Get(ctx, object)
+		raw, err = ctrl.ObjectCache.Get(ctx, object.VersionedKey)
+
 		if err != nil {
-			return nil, fmt.Errorf("cannot fetch object value: %w", err)
+			logger.WithError(err).Error("cannot fetch object value")
+			return nil
 		}
+
 		if raw == nil {
-			return nil, fmt.Errorf("object does not exist")
+			logger.Debug("object no longer exists")
+			return nil
 		}
 	}
 
@@ -81,28 +84,19 @@ func (ctrl *controller) Lookup(ctx context.Context, object util.ObjectRef) (*uti
 		ref := &ParentLink{}
 		err := json.Unmarshal([]byte(ann), ref)
 		if err != nil {
-			return nil, fmt.Errorf("cannot parse ParentLink annotation: %w", err)
+			logger.WithError(err).Error("cannot parse ParentLink annotation")
+			return nil
 		}
 
 		if ref.Cluster == "" {
 			ref.Cluster = object.Cluster
 		}
 
-		objectRef := &util.ObjectRef{
-			Cluster: ref.Cluster,
-			GroupVersionResource: schema.GroupVersionResource{
-				Group:    ref.GroupVersionResource.Group,
-				Version:  ref.GroupVersionResource.Version,
-				Resource: ref.GroupVersionResource.Resource,
-			},
-			Namespace: ref.Namespace,
-			Name:      ref.Name,
-			Uid:       ref.Uid,
-		}
+		objectRef := ref.ToRich()
 		logger.WithField("parent", objectRef).Debug("Resolved parent")
 
-		return objectRef, nil
+		return &objectRef
 	}
 
-	return nil, nil
+	return nil
 }
diff --git a/pkg/annotationlinker/schema.go b/pkg/annotationlinker/schema.go
index c9517d67..ce13026d 100644
--- a/pkg/annotationlinker/schema.go
+++ b/pkg/annotationlinker/schema.go
@@ -17,6 +17,8 @@ package annotationlinker
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 const LinkAnnotation = "kelemetry.kubewharf.io/parent-link"
@@ -31,3 +33,19 @@ type ParentLink struct {
 
 	Uid types.UID `json:"uid"`
 }
+
+func (ln ParentLink) ToRich() utilobject.Rich {
+	return utilobject.Rich{
+		VersionedKey: utilobject.VersionedKey{
+			Key: utilobject.Key{
+				Cluster:   ln.Cluster,
+				Group:     ln.Group,
+				Resource:  ln.Resource,
+				Namespace: ln.Namespace,
+				Name:      ln.Name,
+			},
+			Version: ln.Version,
+		},
+		Uid: ln.Uid,
+	}
+}
diff --git a/pkg/audit/consumer/consumer.go b/pkg/audit/consumer/consumer.go
index dafdcfac..a937cb70 100644
--- a/pkg/audit/consumer/consumer.go
+++ b/pkg/audit/consumer/consumer.go
@@ -38,7 +38,7 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/k8s/discovery"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 	"github.com/kubewharf/kelemetry/pkg/util/shutdown"
 	"github.com/kubewharf/kelemetry/pkg/util/zconstants"
 )
@@ -210,17 +210,7 @@ func (recv *receiver) handleItem(
 		}
 	}
 
-	objectRef := util.ObjectRef{
-		Cluster: message.Cluster,
-		GroupVersionResource: schema.GroupVersionResource{
-			Group:    message.ObjectRef.APIGroup,
-			Version:  message.ObjectRef.APIVersion,
-			Resource: message.ObjectRef.Resource,
-		},
-		Namespace: message.ObjectRef.Namespace,
-		Name:      message.ObjectRef.Name,
-		Uid:       message.ObjectRef.UID,
-	}
+	objectRef := utilobject.RichFromAudit(message.ObjectRef, message.Cluster)
 
 	if message.ResponseObject != nil {
 		objectRef.Raw = &unstructured.Unstructured{
@@ -257,19 +247,20 @@ func (recv *receiver) handleItem(
 	}
 
 	event := aggregatorevent.NewEvent(title, message.RequestReceivedTimestamp.Time, zconstants.TraceSourceAudit).
-		WithEndTime(message.StageTimestamp.Time).
-		WithTag("username", username).
-		WithTag("userAgent", message.UserAgent).
-		WithTag("responseCode", message.ResponseStatus.Code).
-		WithTag("resourceVersion", message.ObjectRef.ResourceVersion).
-		WithTag("apiserver", message.ApiserverAddr).
-		WithTag("tag", message.Verb)
+		SetEndTime(message.StageTimestamp.Time).
+		SetTag("auditId", message.AuditID).
+		SetTag("username", username).
+		SetTag("userAgent", message.UserAgent).
+		SetTag("responseCode", message.ResponseStatus.Code).
+		SetTag("resourceVersion", message.ObjectRef.ResourceVersion).
+		SetTag("apiserver", message.ApiserverAddr).
+		SetTag("tag", message.Verb)
 
 	if len(message.SourceIPs) > 0 {
-		event = event.WithTag("sourceIP", message.SourceIPs[0])
+		event = event.SetTag("sourceIP", message.SourceIPs[0])
 
 		if len(message.SourceIPs) > 1 {
-			event = event.WithTag("proxy", message.SourceIPs[1:])
+			event = event.SetTag("proxy", message.SourceIPs[1:])
 		}
 	}
 
diff --git a/pkg/crds/apis/v1alpha1/util/util.go b/pkg/crds/apis/v1alpha1/util/util.go
index f154fe04..01f2f9a1 100644
--- a/pkg/crds/apis/v1alpha1/util/util.go
+++ b/pkg/crds/apis/v1alpha1/util/util.go
@@ -22,14 +22,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/rest"
 
 	kelemetryv1a1 "github.com/kubewharf/kelemetry/pkg/crds/apis/v1alpha1"
 	kelemetryv1a1client "github.com/kubewharf/kelemetry/pkg/crds/client/clientset/versioned/typed/apis/v1alpha1"
 	"github.com/kubewharf/kelemetry/pkg/manager"
-	"github.com/kubewharf/kelemetry/pkg/util"
 	globalinformer "github.com/kubewharf/kelemetry/pkg/util/informer/global"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 type (
@@ -53,7 +52,7 @@ func init() {
 	}))
 }
 
-func TestObject(objectRef util.ObjectRef, object metav1.Object, rule *kelemetryv1a1.LinkRuleFilter) (bool, error) {
+func TestObject(objectRef utilobject.Key, object metav1.Object, rule *kelemetryv1a1.LinkRuleFilter) (bool, error) {
 	objectGr := metav1.GroupResource{Group: objectRef.Group, Resource: objectRef.Resource}
 
 	matchedGr := false
@@ -79,15 +78,15 @@ func TestObject(objectRef util.ObjectRef, object metav1.Object, rule *kelemetryv
 
 func GenerateParent(
 	rule *kelemetryv1a1.LinkRuleParent,
-	childObjectRef util.ObjectRef,
+	childObjectRef utilobject.Rich,
 	childObject *unstructured.Unstructured,
-) (_r util.ObjectRef, _ error) {
+) (_parent utilobject.VersionedKey, _ error) {
 	cluster := childObjectRef.Cluster
 	if rule.ClusterTemplate != "" {
 		var err error
 		cluster, err = executeTemplate("clusterTemplate", rule.ClusterTemplate, childObject)
 		if err != nil {
-			return _r, err
+			return _parent, err
 		}
 	}
 
@@ -98,18 +97,18 @@ func GenerateParent(
 
 	name, err := executeTemplate("nameTemplate", rule.NameTemplate, childObject)
 	if err != nil {
-		return _r, err
+		return _parent, err
 	}
 
-	return util.ObjectRef{
-		Cluster: cluster,
-		GroupVersionResource: schema.GroupVersionResource{
-			Group:    rule.Group,
-			Version:  rule.Version,
+	return utilobject.VersionedKey{
+		Key: utilobject.Key{
+			Cluster: cluster,
+			Group: rule.Group,
 			Resource: rule.Resource,
+			Namespace: namespace,
+			Name:      name,
 		},
-		Namespace: namespace,
-		Name:      name,
+		Version: rule.Version,
 	}, nil
 }
 
diff --git a/pkg/diff/api/api.go b/pkg/diff/api/api.go
index fbb117d2..9423facf 100644
--- a/pkg/diff/api/api.go
+++ b/pkg/diff/api/api.go
@@ -31,7 +31,7 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/k8s/objectcache"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 	"github.com/kubewharf/kelemetry/pkg/util/shutdown"
 )
 
@@ -114,15 +114,15 @@ func (api *api) handleGet(ctx *gin.Context) error {
 		cluster = clusterQuery
 	}
 
-	raw, err := api.ObjectCache.Get(ctx, util.ObjectRef{
-		Cluster: cluster,
-		GroupVersionResource: schema.GroupVersionResource{
-			Group:    group,
-			Version:  version,
-			Resource: resource,
+	raw, err := api.ObjectCache.Get(ctx, utilobject.VersionedKey{
+		Key: utilobject.Key{
+			Cluster:   cluster,
+			Group:     group,
+			Resource:  resource,
+			Namespace: namespace,
+			Name:      name,
 		},
-		Namespace: namespace,
-		Name:      name,
+		Version: version,
 	})
 	if err != nil {
 		return err
@@ -131,21 +131,13 @@ func (api *api) handleGet(ctx *gin.Context) error {
 		return ctx.AbortWithError(404, fmt.Errorf("object does not exist"))
 	}
 
-	object := util.ObjectRefFromUnstructured(raw, cluster, schema.GroupVersionResource{
+	object := utilobject.RichFromUnstructured(raw, cluster, schema.GroupVersionResource{
 		Group:    group,
 		Version:  version,
 		Resource: resource,
 	})
 
-	var oldRv string
-	var newRv *string
-	if api.DiffCache.GetCommonOptions().UseOldResourceVersion {
-		oldRv = rv
-	} else {
-		newRv = &rv
-	}
-
-	patch, err := api.DiffCache.Fetch(ctx, object, oldRv, newRv)
+	patch, err := api.DiffCache.Fetch(ctx, object.Key, rv, &rv)
 	if err != nil || patch == nil {
 		return ctx.AbortWithError(404, fmt.Errorf("patch not found for rv: %w", err))
 	}
@@ -173,15 +165,15 @@ func (api *api) handleScan(ctx *gin.Context) error {
 		limit = parsedLimit
 	}
 
-	raw, err := api.ObjectCache.Get(ctx, util.ObjectRef{
-		Cluster: cluster,
-		GroupVersionResource: schema.GroupVersionResource{
-			Group:    group,
-			Version:  version,
-			Resource: resource,
+	raw, err := api.ObjectCache.Get(ctx, utilobject.VersionedKey{
+		Key: utilobject.Key{
+			Cluster:   cluster,
+			Group:     group,
+			Resource:  resource,
+			Namespace: namespace,
+			Name:      name,
 		},
-		Namespace: namespace,
-		Name:      name,
+		Version: version,
 	})
 	if err != nil {
 		return err
@@ -190,12 +182,12 @@ func (api *api) handleScan(ctx *gin.Context) error {
 		return ctx.AbortWithError(404, fmt.Errorf("object does not exist"))
 	}
 
-	object := util.ObjectRefFromUnstructured(raw, cluster, schema.GroupVersionResource{
+	object := utilobject.RichFromUnstructured(raw, cluster, schema.GroupVersionResource{
 		Group:    group,
 		Version:  version,
 		Resource: resource,
 	})
-	list, err := api.DiffCache.List(ctx, object, limit)
+	list, err := api.DiffCache.List(ctx, object.Key, limit)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/diff/cache/etcd/etcd.go b/pkg/diff/cache/etcd/etcd.go
index 8e4b66ab..2c6d229d 100644
--- a/pkg/diff/cache/etcd/etcd.go
+++ b/pkg/diff/cache/etcd/etcd.go
@@ -26,9 +26,10 @@ import (
 	etcdv3 "go.etcd.io/etcd/client/v3"
 
 	diffcache "github.com/kubewharf/kelemetry/pkg/diff/cache"
+	k8sconfig "github.com/kubewharf/kelemetry/pkg/k8s/config"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 	"github.com/kubewharf/kelemetry/pkg/util/shutdown"
 )
 
@@ -60,8 +61,9 @@ func (options *etcdOptions) EnableFlag() *bool { return nil }
 type Etcd struct {
 	manager.MuxImplBase
 
-	options etcdOptions
-	Logger  logrus.FieldLogger
+	options        etcdOptions
+	Logger         logrus.FieldLogger
+	ClusterConfigs k8sconfig.Config
 
 	client    *etcdv3.Client
 	deferList *shutdown.DeferList
@@ -108,7 +110,7 @@ func (cache *Etcd) GetCommonOptions() *diffcache.CommonOptions {
 	return cache.GetAdditionalOptions().(*diffcache.CommonOptions)
 }
 
-func (cache *Etcd) Store(ctx context.Context, object util.ObjectRef, patch *diffcache.Patch) {
+func (cache *Etcd) Store(ctx context.Context, object utilobject.Key, patch *diffcache.Patch) {
 	patchJson, err := json.Marshal(patch)
 	if err != nil {
 		cache.Logger.WithError(err).Error("cannot marshal patch")
@@ -121,7 +123,7 @@ func (cache *Etcd) Store(ctx context.Context, object util.ObjectRef, patch *diff
 		return
 	}
 
-	keyRv, _ := cache.GetCommonOptions().ChooseResourceVersion(patch.OldResourceVersion, &patch.NewResourceVersion)
+	keyRv, _ := cache.ClusterConfigs.Provide(object.Cluster).ChooseResourceVersion(patch.OldResourceVersion, &patch.NewResourceVersion)
 	_, err = cache.client.KV.Put(ctx, cache.cacheKey(object, keyRv), string(patchJson), etcdv3.WithLease(lease.ID))
 	if err != nil {
 		cache.Logger.WithError(err).Error("cannot write cache")
@@ -131,11 +133,11 @@ func (cache *Etcd) Store(ctx context.Context, object util.ObjectRef, patch *diff
 
 func (cache *Etcd) Fetch(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Key,
 	oldResourceVersion string,
 	newResourceVersion *string,
 ) (*diffcache.Patch, error) {
-	keyRv, err := cache.GetCommonOptions().ChooseResourceVersion(oldResourceVersion, newResourceVersion)
+	keyRv, err := cache.ClusterConfigs.Provide(object.Cluster).ChooseResourceVersion(oldResourceVersion, newResourceVersion)
 	if err != nil {
 		return nil, err
 	}
@@ -162,7 +164,7 @@ func (cache *Etcd) Fetch(
 	return patch, nil
 }
 
-func (cache *Etcd) StoreSnapshot(ctx context.Context, object util.ObjectRef, snapshotName string, snapshot *diffcache.Snapshot) {
+func (cache *Etcd) StoreSnapshot(ctx context.Context, object utilobject.Key, snapshotName string, snapshot *diffcache.Snapshot) {
 	snapshotJson, err := json.Marshal(snapshot)
 	if err != nil {
 		cache.Logger.WithError(err).Error("cannot marshal snapshot")
@@ -183,7 +185,7 @@ func (cache *Etcd) StoreSnapshot(ctx context.Context, object util.ObjectRef, sna
 	}
 }
 
-func (cache *Etcd) FetchSnapshot(ctx context.Context, object util.ObjectRef, snapshotName string) (*diffcache.Snapshot, error) {
+func (cache *Etcd) FetchSnapshot(ctx context.Context, object utilobject.Key, snapshotName string) (*diffcache.Snapshot, error) {
 	key := cache.snapshotKey(object, snapshotName)
 	resp, err := cache.client.KV.Get(ctx, key)
 	if err != nil {
@@ -205,7 +207,7 @@ func (cache *Etcd) FetchSnapshot(ctx context.Context, object util.ObjectRef, sna
 	return snapshot, nil
 }
 
-func (cache *Etcd) List(ctx context.Context, object util.ObjectRef, limit int) ([]string, error) {
+func (cache *Etcd) List(ctx context.Context, object utilobject.Key, limit int) ([]string, error) {
 	resp, err := cache.client.KV.Get(
 		ctx,
 		cache.cacheKeyPrefix(object),
@@ -230,19 +232,19 @@ func (cache *Etcd) List(ctx context.Context, object util.ObjectRef, limit int) (
 	return keys, nil
 }
 
-func (cache *Etcd) cacheKeyPrefix(object util.ObjectRef) string {
+func (cache *Etcd) cacheKeyPrefix(object utilobject.Key) string {
 	return fmt.Sprintf("%s%s/", cache.options.prefix, object.String())
 }
 
-func (cache *Etcd) cacheKey(object util.ObjectRef, keyRv string) string {
+func (cache *Etcd) cacheKey(object utilobject.Key, keyRv string) string {
 	whichRv := "newRv"
-	if cache.GetCommonOptions().UseOldResourceVersion {
+	if cluster := cache.ClusterConfigs.Provide(object.Cluster); cluster != nil && cluster.UseOldResourceVersion {
 		whichRv = "oldRv"
 	}
 
 	return cache.cacheKeyPrefix(object) + fmt.Sprintf("%s/%s", whichRv, keyRv)
 }
 
-func (cache *Etcd) snapshotKey(object util.ObjectRef, snapshotName string) string {
+func (cache *Etcd) snapshotKey(object utilobject.Key, snapshotName string) string {
 	return cache.cacheKeyPrefix(object) + snapshotName
 }
diff --git a/pkg/diff/cache/interface.go b/pkg/diff/cache/interface.go
index 1ffb1f4e..be7d0464 100644
--- a/pkg/diff/cache/interface.go
+++ b/pkg/diff/cache/interface.go
@@ -24,9 +24,10 @@ import (
 	"k8s.io/utils/clock"
 
 	diffcmp "github.com/kubewharf/kelemetry/pkg/diff/cmp"
+	k8sconfig "github.com/kubewharf/kelemetry/pkg/k8s/config"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func init() {
@@ -48,20 +49,9 @@ type Snapshot struct {
 }
 
 type CommonOptions struct {
-	PatchTtl              time.Duration
-	SnapshotTtl           time.Duration
-	EnableCacheWrapper    bool
-	UseOldResourceVersion bool
-}
-
-func (options *CommonOptions) ChooseResourceVersion(oldRv string, newRv *string) (string, error) {
-	if options.UseOldResourceVersion {
-		return oldRv, nil
-	} else if newRv != nil {
-		return *newRv, nil
-	} else {
-		return "", metrics.MakeLabeledError("NoNewRv")
-	}
+	PatchTtl           time.Duration
+	SnapshotTtl        time.Duration
+	EnableCacheWrapper bool
 }
 
 func (options *CommonOptions) Setup(fs *pflag.FlagSet) {
@@ -73,33 +63,27 @@ func (options *CommonOptions) Setup(fs *pflag.FlagSet) {
 		"duration for which snapshot cache remains (0 to disable TTL)",
 	)
 	fs.BoolVar(&options.EnableCacheWrapper, "diff-cache-wrapper-enable", false, "enable an intermediate layer of cache in memory")
-	fs.BoolVar(
-		&options.UseOldResourceVersion,
-		"diff-cache-use-old-rv",
-		false,
-		"index diff entries with the resource version before update "+
-			"(inaccurate, but works with Metadata-level audit policy)",
-	)
 }
 
 type Cache interface {
 	GetCommonOptions() *CommonOptions
 
-	Store(ctx context.Context, object util.ObjectRef, patch *Patch)
-	Fetch(ctx context.Context, object util.ObjectRef, oldResourceVersion string, newResourceVersion *string) (*Patch, error)
+	Store(ctx context.Context, object utilobject.Key, patch *Patch)
+	Fetch(ctx context.Context, object utilobject.Key, oldResourceVersion string, newResourceVersion *string) (*Patch, error)
 
-	StoreSnapshot(ctx context.Context, object util.ObjectRef, snapshotName string, snapshot *Snapshot)
-	FetchSnapshot(ctx context.Context, object util.ObjectRef, snapshotName string) (*Snapshot, error)
+	StoreSnapshot(ctx context.Context, object utilobject.Key, snapshotName string, snapshot *Snapshot)
+	FetchSnapshot(ctx context.Context, object utilobject.Key, snapshotName string) (*Snapshot, error)
 
-	List(ctx context.Context, object util.ObjectRef, limit int) ([]string, error)
+	List(ctx context.Context, object utilobject.Key, limit int) ([]string, error)
 }
 
 type mux struct {
 	options *CommonOptions
 	*manager.Mux
-	Metrics metrics.Client
-	Logger  logrus.FieldLogger
-	Clock   clock.Clock
+	Metrics        metrics.Client
+	Logger         logrus.FieldLogger
+	Clock          clock.Clock
+	ClusterConfigs k8sconfig.Config
 
 	impl Cache
 
@@ -108,6 +92,7 @@ type mux struct {
 	StoreSnapshotMetric *metrics.Metric[*storeSnapshotMetric]
 	FetchSnapshotMetric *metrics.Metric[*fetchSnapshotMetric]
 	ListMetric          *metrics.Metric[*listMetric]
+	PenetrateMetric     *metrics.Metric[*penetrateMetric]
 }
 
 func newCache() Cache {
@@ -155,7 +140,7 @@ func (mux *mux) Init() error {
 
 	mux.impl = mux.Impl().(Cache)
 	if mux.options.EnableCacheWrapper {
-		wrapper := newCacheWrapper(mux.options, mux.impl, mux.Clock)
+		wrapper := newCacheWrapper(mux.options, mux.impl, mux.Clock, mux.ClusterConfigs, mux.PenetrateMetric)
 		wrapper.initMetricsLoop(mux.Metrics)
 		mux.impl = wrapper
 	}
@@ -175,12 +160,12 @@ func (mux *mux) GetCommonOptions() *CommonOptions {
 	return mux.options
 }
 
-func (mux *mux) Store(ctx context.Context, object util.ObjectRef, patch *Patch) {
+func (mux *mux) Store(ctx context.Context, object utilobject.Key, patch *Patch) {
 	defer mux.StoreDiffMetric.DeferCount(mux.Clock.Now(), &storeDiffMetric{Redacted: patch.Redacted})
 	mux.Impl().(Cache).Store(ctx, object, patch)
 }
 
-func (mux *mux) Fetch(ctx context.Context, object util.ObjectRef, oldResourceVersion string, newResourceVersion *string) (*Patch, error) {
+func (mux *mux) Fetch(ctx context.Context, object utilobject.Key, oldResourceVersion string, newResourceVersion *string) (*Patch, error) {
 	metric := &fetchDiffMetric{}
 	defer mux.FetchDiffMetric.DeferCount(mux.Clock.Now(), metric)
 
@@ -194,12 +179,12 @@ func (mux *mux) Fetch(ctx context.Context, object util.ObjectRef, oldResourceVer
 	return patch, nil
 }
 
-func (mux *mux) StoreSnapshot(ctx context.Context, object util.ObjectRef, snapshotName string, snapshot *Snapshot) {
+func (mux *mux) StoreSnapshot(ctx context.Context, object utilobject.Key, snapshotName string, snapshot *Snapshot) {
 	defer mux.StoreSnapshotMetric.DeferCount(mux.Clock.Now(), &storeSnapshotMetric{Redacted: snapshot.Redacted})
 	mux.Impl().(Cache).StoreSnapshot(ctx, object, snapshotName, snapshot)
 }
 
-func (mux *mux) FetchSnapshot(ctx context.Context, object util.ObjectRef, snapshotName string) (*Snapshot, error) {
+func (mux *mux) FetchSnapshot(ctx context.Context, object utilobject.Key, snapshotName string) (*Snapshot, error) {
 	metric := &fetchSnapshotMetric{}
 	defer mux.FetchSnapshotMetric.DeferCount(mux.Clock.Now(), metric)
 
@@ -213,7 +198,7 @@ func (mux *mux) FetchSnapshot(ctx context.Context, object util.ObjectRef, snapsh
 	return snapshot, nil
 }
 
-func (mux *mux) List(ctx context.Context, object util.ObjectRef, limit int) ([]string, error) {
+func (mux *mux) List(ctx context.Context, object utilobject.Key, limit int) ([]string, error) {
 	defer mux.ListMetric.DeferCount(mux.Clock.Now(), &listMetric{})
 	return mux.Impl().(Cache).List(ctx, object, limit)
 }
diff --git a/pkg/diff/cache/local/local.go b/pkg/diff/cache/local/local.go
index 13dee0b2..bd30518b 100644
--- a/pkg/diff/cache/local/local.go
+++ b/pkg/diff/cache/local/local.go
@@ -24,9 +24,10 @@ import (
 	"k8s.io/utils/clock"
 
 	diffcache "github.com/kubewharf/kelemetry/pkg/diff/cache"
+	k8sconfig "github.com/kubewharf/kelemetry/pkg/k8s/config"
 	"github.com/kubewharf/kelemetry/pkg/manager"
-	"github.com/kubewharf/kelemetry/pkg/util"
 	"github.com/kubewharf/kelemetry/pkg/util/cache"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 	"github.com/kubewharf/kelemetry/pkg/util/shutdown"
 )
 
@@ -39,8 +40,9 @@ func init() {
 type localCache struct {
 	manager.MuxImplBase
 
-	Logger logrus.FieldLogger
-	Clock  clock.Clock
+	Logger         logrus.FieldLogger
+	Clock          clock.Clock
+	ClusterConfigs k8sconfig.Config
 
 	data     map[string]*history
 	dataLock sync.RWMutex
@@ -109,7 +111,7 @@ func (cache *localCache) GetCommonOptions() *diffcache.CommonOptions {
 	return cache.GetAdditionalOptions().(*diffcache.CommonOptions)
 }
 
-func (cache *localCache) Store(ctx context.Context, object util.ObjectRef, patch *diffcache.Patch) {
+func (cache *localCache) Store(ctx context.Context, object utilobject.Key, patch *diffcache.Patch) {
 	cache.dataLock.Lock()
 	defer cache.dataLock.Unlock()
 
@@ -120,20 +122,20 @@ func (cache *localCache) Store(ctx context.Context, object util.ObjectRef, patch
 	patches := cache.data[object.String()]
 	patches.lastModify = cache.Clock.Now()
 
-	keyRv, _ := cache.GetCommonOptions().ChooseResourceVersion(patch.OldResourceVersion, &patch.NewResourceVersion)
+	keyRv, _ := cache.ClusterConfigs.Provide(object.Cluster).ChooseResourceVersion(patch.OldResourceVersion, &patch.NewResourceVersion)
 	patches.patches[keyRv] = patch
 }
 
 func (cache *localCache) Fetch(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Key,
 	oldResourceVersion string,
 	newResourceVersion *string,
 ) (*diffcache.Patch, error) {
 	cache.dataLock.RLock()
 	defer cache.dataLock.RUnlock()
 
-	keyRv, err := cache.GetCommonOptions().ChooseResourceVersion(oldResourceVersion, newResourceVersion)
+	keyRv, err := cache.ClusterConfigs.Provide(object.Cluster).ChooseResourceVersion(oldResourceVersion, newResourceVersion)
 	if err != nil {
 		return nil, err
 	}
@@ -158,13 +160,13 @@ func (cache *localCache) Fetch(
 	return nil, nil
 }
 
-func (cache *localCache) StoreSnapshot(ctx context.Context, object util.ObjectRef, snapshotName string, value *diffcache.Snapshot) {
+func (cache *localCache) StoreSnapshot(ctx context.Context, object utilobject.Key, snapshotName string, value *diffcache.Snapshot) {
 	cache.snapshotCache.Add(fmt.Sprintf("%v/%s", object, snapshotName), value)
 }
 
 func (cache *localCache) FetchSnapshot(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Key,
 	snapshotName string,
 ) (*diffcache.Snapshot, error) {
 	if value, ok := cache.snapshotCache.Get(fmt.Sprintf("%v/%s", object, snapshotName)); ok {
@@ -174,7 +176,7 @@ func (cache *localCache) FetchSnapshot(
 	return nil, nil
 }
 
-func (cache *localCache) List(ctx context.Context, object util.ObjectRef, limit int) ([]string, error) {
+func (cache *localCache) List(ctx context.Context, object utilobject.Key, limit int) ([]string, error) {
 	cache.dataLock.RLock()
 	defer cache.dataLock.RUnlock()
 
diff --git a/pkg/diff/cache/memory_wrapper.go b/pkg/diff/cache/memory_wrapper.go
index 0582b5fb..cc872d24 100644
--- a/pkg/diff/cache/memory_wrapper.go
+++ b/pkg/diff/cache/memory_wrapper.go
@@ -20,15 +20,17 @@ import (
 
 	"k8s.io/utils/clock"
 
+	k8sconfig "github.com/kubewharf/kelemetry/pkg/k8s/config"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
 	"github.com/kubewharf/kelemetry/pkg/util/cache"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 type CacheWrapper struct {
 	delegate        Cache
 	patchCache      *cache.TtlOnce
-	PenetrateMetric *metrics.Metric[*penetrateMetric]
+	penetrateMetric *metrics.Metric[*penetrateMetric]
+	clusterConfigs  k8sconfig.Config
 	snapshotCache   *cache.TtlOnce
 	options         *CommonOptions
 	clock           clock.Clock
@@ -38,13 +40,17 @@ func newCacheWrapper(
 	options *CommonOptions,
 	delegate Cache,
 	clock clock.Clock,
+	clusterConfigs k8sconfig.Config,
+	penetrateMetric *metrics.Metric[*penetrateMetric],
 ) *CacheWrapper {
 	return &CacheWrapper{
-		delegate:      delegate,
-		patchCache:    cache.NewTtlOnce(options.PatchTtl, clock),
-		snapshotCache: cache.NewTtlOnce(options.SnapshotTtl, clock),
-		options:       options,
-		clock:         clock,
+		delegate:        delegate,
+		patchCache:      cache.NewTtlOnce(options.PatchTtl, clock),
+		penetrateMetric: penetrateMetric,
+		clusterConfigs:  clusterConfigs,
+		snapshotCache:   cache.NewTtlOnce(options.SnapshotTtl, clock),
+		options:         options,
+		clock:           clock,
 	}
 }
 
@@ -71,7 +77,7 @@ func (wrapper *CacheWrapper) GetCommonOptions() *CommonOptions {
 	return wrapper.options
 }
 
-func (wrapper *CacheWrapper) Store(ctx context.Context, object util.ObjectRef, patch *Patch) {
+func (wrapper *CacheWrapper) Store(ctx context.Context, object utilobject.Key, patch *Patch) {
 	wrapper.delegate.Store(ctx, object, patch)
 
 	wrapper.patchCache.Add(cacheWrapperKey(object, patch.NewResourceVersion), patch)
@@ -79,14 +85,14 @@ func (wrapper *CacheWrapper) Store(ctx context.Context, object util.ObjectRef, p
 
 func (wrapper *CacheWrapper) Fetch(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Key,
 	oldResourceVersion string,
 	newResourceVersion *string,
 ) (*Patch, error) {
 	penetrateMetric := &penetrateMetric{Type: "diff"}
-	defer wrapper.PenetrateMetric.DeferCount(wrapper.clock.Now(), penetrateMetric)
+	defer wrapper.penetrateMetric.DeferCount(wrapper.clock.Now(), penetrateMetric)
 
-	keyRv, err := wrapper.options.ChooseResourceVersion(oldResourceVersion, newResourceVersion)
+	keyRv, err := wrapper.clusterConfigs.Provide(object.Cluster).ChooseResourceVersion(oldResourceVersion, newResourceVersion)
 	if err != nil {
 		return nil, err
 	}
@@ -107,7 +113,7 @@ func (wrapper *CacheWrapper) Fetch(
 
 func (wrapper *CacheWrapper) StoreSnapshot(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Key,
 	snapshotName string,
 	snapshot *Snapshot,
 ) {
@@ -117,11 +123,11 @@ func (wrapper *CacheWrapper) StoreSnapshot(
 
 func (wrapper *CacheWrapper) FetchSnapshot(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Key,
 	snapshotName string,
 ) (*Snapshot, error) {
 	penetrateMetric := &penetrateMetric{Type: fmt.Sprintf("snapshot/%s", snapshotName)}
-	defer wrapper.PenetrateMetric.DeferCount(wrapper.clock.Now(), penetrateMetric)
+	defer wrapper.penetrateMetric.DeferCount(wrapper.clock.Now(), penetrateMetric)
 
 	if value, ok := wrapper.patchCache.Get(cacheWrapperKey(object, snapshotName)); ok {
 		return value.(*Snapshot), nil
@@ -134,10 +140,10 @@ func (wrapper *CacheWrapper) FetchSnapshot(
 }
 
 // List always penetrates the cache because we cannot get notified of new keys
-func (wrapper *CacheWrapper) List(ctx context.Context, object util.ObjectRef, limit int) ([]string, error) {
+func (wrapper *CacheWrapper) List(ctx context.Context, object utilobject.Key, limit int) ([]string, error) {
 	return wrapper.delegate.List(ctx, object, limit)
 }
 
-func cacheWrapperKey(object util.ObjectRef, subkey string) string {
+func cacheWrapperKey(object utilobject.Key, subkey string) string {
 	return fmt.Sprintf("%s/%s", object.String(), subkey)
 }
diff --git a/pkg/diff/controller/controller.go b/pkg/diff/controller/controller.go
index 6cf66e1b..8dc8d78b 100644
--- a/pkg/diff/controller/controller.go
+++ b/pkg/diff/controller/controller.go
@@ -43,9 +43,9 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/k8s/multileader"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
 	"github.com/kubewharf/kelemetry/pkg/util/channel"
 	informerutil "github.com/kubewharf/kelemetry/pkg/util/informer"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 	"github.com/kubewharf/kelemetry/pkg/util/shutdown"
 )
 
@@ -585,7 +585,7 @@ func (monitor *monitor) onUpdate(
 		patch.DiffList = diffcmp.Compare(oldObj.Object, newObj.Object)
 	}
 
-	objectRef := util.ObjectRefFromUnstructured(newObj, monitor.ctrl.Clients.TargetCluster().ClusterName(), monitor.gvr).Clone()
+	objectRef := utilobject.RichFromUnstructured(newObj, monitor.ctrl.Clients.TargetCluster().ClusterName(), monitor.gvr).Clone()
 
 	return func(ctx context.Context) {
 		ctx, cancelFunc := context.WithTimeout(ctx, monitor.ctrl.options.storeTimeout)
@@ -593,7 +593,7 @@ func (monitor *monitor) onUpdate(
 
 		monitor.ctrl.Cache.Store(
 			ctx,
-			objectRef,
+			objectRef.Key,
 			patch,
 		)
 	}
@@ -617,7 +617,7 @@ func (monitor *monitor) onNeedSnapshot(
 			Error("cannot re-marshal unstructured object")
 	}
 
-	objectRef := util.ObjectRefFromUnstructured(obj, monitor.ctrl.Clients.TargetCluster().ClusterName(), monitor.gvr).Clone()
+	objectRef := utilobject.RichFromUnstructured(obj, monitor.ctrl.Clients.TargetCluster().ClusterName(), monitor.gvr).Clone()
 	snapshotRv := obj.GetResourceVersion()
 
 	return func(ctx context.Context) {
@@ -626,7 +626,7 @@ func (monitor *monitor) onNeedSnapshot(
 
 		monitor.ctrl.Cache.StoreSnapshot(
 			ctx,
-			objectRef,
+			objectRef.Key,
 			snapshotName,
 			&diffcache.Snapshot{
 				ResourceVersion: snapshotRv,
diff --git a/pkg/diff/decorator/decorator.go b/pkg/diff/decorator/decorator.go
index 4070e647..7537438f 100644
--- a/pkg/diff/decorator/decorator.go
+++ b/pkg/diff/decorator/decorator.go
@@ -33,7 +33,7 @@ import (
 	diffcache "github.com/kubewharf/kelemetry/pkg/diff/cache"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 	"github.com/kubewharf/kelemetry/pkg/util/zconstants"
 )
 
@@ -211,7 +211,7 @@ func (decorator *decorator) tryDecorate(
 	}
 
 	// NOTE: UID may be empty, but we don't use it anyway
-	object := util.ObjectRefFromAudit(message.ObjectRef, message.Cluster, message.ObjectRef.UID)
+	object := utilobject.RichFromAudit(message.ObjectRef, message.Cluster)
 
 	var tryOnce func(context.Context) (bool, error)
 
@@ -229,7 +229,14 @@ func (decorator *decorator) tryDecorate(
 			return cacheHitTypeSameRv, nil
 		}
 
-		logger = logger.WithField("oldRv", oldRv).WithField("newRv", newRv)
+		if newRv != nil {
+			event.SetTag("newResourceVersion", *newRv)
+		}
+
+		logger = logger.WithField("oldRv", oldRv)
+		if newRv != nil {
+			logger = logger.WithField("newRv", *newRv)
+		}
 
 		tryOnce = func(ctx context.Context) (bool, error) {
 			return decorator.tryUpdateOnce(ctx, object, oldRv, newRv, event, message)
@@ -294,19 +301,19 @@ func (decorator *decorator) tryDecorate(
 }
 
 func decoratesResource(message *audit.Message) bool {
-	return message.Verb != audit.VerbPatch
+	return true
 }
 
 func (decorator *decorator) tryUpdateOnce(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Rich,
 	oldRv string,
 	newRv *string,
 	event *aggregatorevent.Event,
 	message *audit.Message,
 ) (bool, error) {
 	var err error
-	patch, err := decorator.Cache.Fetch(ctx, object, oldRv, newRv)
+	patch, err := decorator.Cache.Fetch(ctx, object.Key, oldRv, newRv)
 	if err != nil || patch == nil {
 		return false, err
 	}
@@ -333,18 +340,18 @@ func (decorator *decorator) tryUpdateOnce(
 		},
 		Resource: message.ObjectRef.Resource,
 	}).Histogram(float64(informerLatency.Nanoseconds()))
-	event.WithTag("informer latency", informerLatency)
+	event.SetTag("informer latency", informerLatency)
 
 	return true, nil
 }
 
 func (decorator *decorator) tryCreateDeleteOnce(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.Rich,
 	snapshotName string,
 	event *aggregatorevent.Event,
 ) (bool, error) {
-	snapshot, err := decorator.Cache.FetchSnapshot(ctx, object, snapshotName)
+	snapshot, err := decorator.Cache.FetchSnapshot(ctx, object.Key, snapshotName)
 	if err != nil || snapshot == nil {
 		return false, err
 	}
diff --git a/pkg/event/controller.go b/pkg/event/controller.go
index 314f74b1..e77fd5b6 100644
--- a/pkg/event/controller.go
+++ b/pkg/event/controller.go
@@ -40,8 +40,8 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/k8s/multileader"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
 	informerutil "github.com/kubewharf/kelemetry/pkg/util/informer"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 	"github.com/kubewharf/kelemetry/pkg/util/shutdown"
 	"github.com/kubewharf/kelemetry/pkg/util/zconstants"
 )
@@ -287,8 +287,8 @@ func (ctrl *controller) handleEvent(ctx context.Context, event *corev1.Event) {
 	}
 
 	aggregatorEvent := aggregatorevent.NewEvent(event.Reason, eventTime, zconstants.TraceSourceEvent).
-		WithTag("source", event.Source.Component).
-		WithTag("action", event.Action).
+		SetTag("source", event.Source.Component).
+		SetTag("action", event.Action).
 		Log(zconstants.LogTypeEventMessage, event.Message)
 
 	cdc, err := ctrl.DiscoveryCache.ForCluster(clusterName)
@@ -312,12 +312,18 @@ func (ctrl *controller) handleEvent(ctx context.Context, event *corev1.Event) {
 		Resource: gvr.Resource,
 	}).Summary(float64(ctrl.Clock.Since(eventTime).Nanoseconds()))
 
-	if err := ctrl.Aggregator.Send(ctx, util.ObjectRef{
-		Cluster:              clusterName,
-		GroupVersionResource: gvr,
-		Namespace:            event.InvolvedObject.Namespace,
-		Name:                 event.InvolvedObject.Name,
-		Uid:                  event.InvolvedObject.UID,
+	if err := ctrl.Aggregator.Send(ctx, utilobject.Rich{
+		VersionedKey: utilobject.VersionedKey{
+			Key: utilobject.Key{
+				Cluster:   clusterName,
+				Group:     gvr.Group,
+				Resource:  gvr.Resource,
+				Namespace: event.InvolvedObject.Namespace,
+				Name:      event.InvolvedObject.Name,
+			},
+			Version: gvr.Version,
+		},
+		Uid: event.InvolvedObject.UID,
 	}, aggregatorEvent, nil); err != nil {
 		logger.WithError(err).Error("Cannot send trace")
 		metric.Error = metrics.LabelError(err, "SendTrace")
diff --git a/pkg/frontend/backend/jaeger-storage/backend.go b/pkg/frontend/backend/jaeger-storage/backend.go
index cf6dcf37..8cea1ee2 100644
--- a/pkg/frontend/backend/jaeger-storage/backend.go
+++ b/pkg/frontend/backend/jaeger-storage/backend.go
@@ -36,6 +36,8 @@ import (
 	jaegerbackend "github.com/kubewharf/kelemetry/pkg/frontend/backend"
 	tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
 	"github.com/kubewharf/kelemetry/pkg/manager"
+	utiljaeger "github.com/kubewharf/kelemetry/pkg/util/jaeger"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 	"github.com/kubewharf/kelemetry/pkg/util/zconstants"
 )
 
@@ -45,14 +47,6 @@ func init() {
 	}), jaegerbackend.Backend.List)
 }
 
-var spanStorageTypesToAddFlag = []string{
-	"cassandra",
-	"elasticsearch",
-	"kafka",
-	"grpc-plugin",
-	"badger",
-}
-
 type Backend struct {
 	manager.MuxImplBase
 	Logger logrus.FieldLogger
@@ -72,7 +66,7 @@ var _ jaegerbackend.Backend = &Backend{}
 func (backend *Backend) Setup(fs *pflag.FlagSet) {
 	// SpanWriterTypes here is used to add flags only, the actual storageFactory is initialized in Backend.Init function
 	factoryConfig := jaegerstorage.FactoryConfig{
-		SpanWriterTypes:         spanStorageTypesToAddFlag,
+		SpanWriterTypes:         utiljaeger.SpanStorageTypesToAddFlag,
 		SpanReaderType:          "elasticsearch",
 		DependenciesStorageType: "elasticsearch",
 	}
@@ -201,11 +195,11 @@ func (backend *Backend) List(
 		// exclusive mode, each object under trace should have a list entry
 		type objectInTrace struct {
 			traceId model.TraceID
-			key     tftree.GroupingKey
+			key     utilobject.Key
 		}
 		seenObjects := sets.New[objectInTrace]()
 		deduplicator = func(span *model.Span) bool {
-			key, hasKey := tftree.GroupingKeyFromSpan(span)
+			key, hasKey := utilobject.FromSpan(span)
 			if !hasKey {
 				return false // not a root
 			}
@@ -257,7 +251,7 @@ func (backend *Backend) List(
 
 		for _, span := range trace.Spans {
 			if deduplicator(span) {
-				tree := tftree.NewSpanTree(trace)
+				tree := tftree.NewSpanTree(trace.Spans)
 				if err := tree.SetRoot(span.SpanID); err != nil {
 					return nil, fmt.Errorf("unexpected SetRoot error for span ID from trace: %w", err)
 				}
@@ -301,7 +295,7 @@ func (backend *Backend) Get(
 		return nil, fmt.Errorf("jaeger storage backend returned empty trace")
 	}
 
-	tree := tftree.NewSpanTree(trace)
+	tree := tftree.NewSpanTree(trace.Spans)
 	if err := tree.SetRoot(ident.SpanId); err != nil {
 		if errors.Is(err, tftree.ErrRootDoesNotExist) {
 			return nil, fmt.Errorf("cached span does not exist in refreshed trace result: %w", err)
diff --git a/pkg/frontend/extension/httptrace/httptrace.go b/pkg/frontend/extension/httptrace/httptrace.go
new file mode 100644
index 00000000..4fd68f7a
--- /dev/null
+++ b/pkg/frontend/extension/httptrace/httptrace.go
@@ -0,0 +1,352 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httptrace
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"text/template"
+	"time"
+
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/sirupsen/logrus"
+
+	"github.com/kubewharf/kelemetry/pkg/frontend/extension"
+	"github.com/kubewharf/kelemetry/pkg/manager"
+	filterutil "github.com/kubewharf/kelemetry/pkg/util/filter"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+)
+
+func init() {
+	manager.Global.ProvideListImpl("frontend-extension/http-trace",
+		manager.Ptr(&NodeTraceFactory{}), &manager.List[extension.ProviderFactory]{})
+}
+
+const extensionKind = "HTTPTrace"
+
+type NodeTraceFactory struct {
+	manager.BaseComponent
+	Logger logrus.FieldLogger
+}
+
+func (*NodeTraceFactory) ListIndex() string { return extensionKind }
+
+func (s *NodeTraceFactory) Configure(jsonBuf []byte) (extension.Provider, error) {
+	providerArgs := &ProviderArgs{}
+	if err := json.Unmarshal(jsonBuf, providerArgs); err != nil {
+		return nil, fmt.Errorf("cannot parse httptrace config: %w", err)
+	}
+
+	for i, traceArg := range providerArgs.TraceBackends {
+		argsTemplates := map[string]*template.Template{}
+		for tagKey, templateString := range traceArg.ArgsTemplates {
+			t, err := template.New(tagKey).Funcs(templateFuncs()).Parse(templateString)
+			if err != nil {
+				return nil, fmt.Errorf("invalid tag template %q: %w", tagKey, err)
+			}
+			argsTemplates[tagKey] = t
+		}
+		urlTemplate, err := template.New("url").Funcs(templateFuncs()).Parse(traceArg.URLTemplate)
+		if err != nil {
+			return nil, fmt.Errorf("invalid url template %q: %w", traceArg.URLTemplate, err)
+		}
+		providerArgs.TraceBackends[i].urlTemplate = urlTemplate
+		providerArgs.TraceBackends[i].argsTemplates = argsTemplates
+	}
+
+	totalTimeout, err := time.ParseDuration(providerArgs.TotalTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("parse totalTimeout error: %w", err)
+	}
+
+	return &Provider{
+		logger:         s.Logger,
+		service:        providerArgs.Service,
+		operation:      providerArgs.Operation,
+		traceBackends:  providerArgs.TraceBackends,
+		totalTimeout:   totalTimeout,
+		maxConcurrency: providerArgs.MaxConcurrency,
+		rawConfig:      jsonBuf,
+	}, nil
+}
+
+type ProviderArgs struct {
+	Service   string `json:"service"`
+	Operation string `json:"operation"`
+
+	TraceBackends []TraceBackend `json:"traceBackends"`
+
+	TotalTimeout   string `json:"totalTimeout"`
+	MaxConcurrency int    `json:"maxConcurrency"`
+}
+
+type TraceBackend struct {
+	TagFilters    filterutil.TagFilters `json:"tagFilters"`
+	ArgsTemplates map[string]string     `json:"argsTemplates"`
+	URLTemplate   string                `json:"urlTemplate"`
+
+	ForObject     bool `json:"forObject"`
+	ForAuditEvent bool `json:"forAuditEvent"`
+
+	urlTemplate   *template.Template
+	argsTemplates map[string]*template.Template
+}
+
+type Provider struct {
+	logger         logrus.FieldLogger
+	service        string
+	operation      string
+	traceBackends  []TraceBackend
+	totalTimeout   time.Duration
+	maxConcurrency int
+	rawConfig      []byte
+}
+
+func (provider *Provider) Kind() string { return extensionKind }
+
+func (provider *Provider) RawConfig() []byte { return provider.rawConfig }
+
+func (provider *Provider) TotalTimeout() time.Duration { return provider.totalTimeout }
+func (provider *Provider) MaxConcurrency() int         { return provider.maxConcurrency }
+
+func (provider *Provider) FetchForObject(
+	ctx context.Context,
+	object utilobject.Rich,
+	mainTags model.KeyValues,
+	start, end time.Time,
+) (*extension.FetchResult, error) {
+	var backends []TraceBackend
+	for _, b := range provider.traceBackends {
+		if b.ForObject {
+			backends = append(backends, b)
+		}
+	}
+	if len(backends) == 0 {
+		return nil, nil
+	}
+
+	return provider.fetch(
+		ctx, object, mainTags, start, end,
+		objectTemplateArgs(object),
+		backends,
+	)
+}
+
+func (provider *Provider) FetchForVersion(
+	ctx context.Context,
+	object utilobject.Rich,
+	resourceVersion string,
+	mainTags model.KeyValues,
+	start, end time.Time,
+) (*extension.FetchResult, error) {
+	var backends []TraceBackend
+	for _, b := range provider.traceBackends {
+		if b.ForAuditEvent {
+			backends = append(backends, b)
+		}
+	}
+	if len(backends) == 0 {
+		return nil, nil
+	}
+
+	templateArgs := objectTemplateArgs(object)
+	templateArgs["resourceVersion"] = resourceVersion
+
+	return provider.fetch(
+		ctx, object, mainTags, start, end,
+		templateArgs,
+		backends,
+	)
+}
+
+type identifier struct {
+	TraceIds []model.TraceID `json:"traceIds"`
+	Start    time.Time       `json:"start"`
+	End      time.Time       `json:"end"`
+}
+
+func (provider *Provider) LoadCache(ctx context.Context, jsonBuf []byte) ([]*model.Span, error) {
+	var ident identifier
+	if err := json.Unmarshal(jsonBuf, &ident); err != nil {
+		return nil, fmt.Errorf("cannot unmarshal cached identifier: %w", err)
+	}
+
+	var spans []*model.Span
+
+	for _, traceId := range ident.TraceIds {
+		trace, err := provider.getTrace(ctx, traceId)
+		if err != nil {
+			return nil, fmt.Errorf("cannot get trace from extension storage: %w", err)
+		}
+
+		spans = append(spans, trace.Spans...)
+	}
+
+	return spans, nil
+}
+
+func objectTemplateArgs(object utilobject.Rich) map[string]any {
+	return map[string]any{
+		"cluster":       object.Cluster,
+		"group":         object.Group,
+		"version":       object.Version,
+		"resource":      object.Resource,
+		"groupVersion":  object.GroupVersion().String(),
+		"groupResource": object.GroupResource().String(),
+		"namespace":     object.Namespace,
+		"name":          object.Name,
+	}
+}
+
+// fetch returns traces search from first successful traceBackend
+func (provider *Provider) fetch(
+	ctx context.Context,
+	_ utilobject.Rich,
+	mainTags model.KeyValues,
+	start, end time.Time,
+	templateArgs map[string]any,
+	traceBackends []TraceBackend,
+) (*extension.FetchResult, error) {
+	for _, mainTag := range mainTags {
+		templateArgs[mainTag.Key] = mainTag.Value()
+	}
+
+	templateArgs["start"] = start
+	templateArgs["end"] = end
+
+	stringTags := map[string]string{}
+	for key, val := range templateArgs {
+		if strVal, ok := val.(string); ok {
+			stringTags[key] = strVal
+		}
+	}
+
+	for _, traceBackend := range traceBackends {
+		if !traceBackend.TagFilters.Check(stringTags) {
+			continue
+		}
+
+		queryArgs := map[string]string{}
+		for tagKey, tagTemplate := range traceBackend.argsTemplates {
+			buf := new(bytes.Buffer)
+			err := tagTemplate.Execute(buf, templateArgs)
+			if err != nil {
+				return nil, fmt.Errorf("cannot execute tag template: %w", err)
+			}
+
+			queryArgs[tagKey] = buf.String()
+		}
+
+		buf := new(bytes.Buffer)
+		err := traceBackend.urlTemplate.Execute(buf, templateArgs)
+		if err != nil {
+			return nil, fmt.Errorf("cannot execute tag template: %w", err)
+		}
+		traceUrl := buf.String()
+
+		traces, err := provider.findTraces(ctx, traceUrl, queryArgs)
+		provider.logger.
+			WithField("url", traceUrl).
+			WithField("args", queryArgs).
+			WithField("startTime", start).
+			WithField("endTime", end).
+			WithError(err).
+			Info("search node traces")
+		if err != nil {
+			continue
+		}
+
+		var traceIds []model.TraceID
+		var spans []*model.Span
+		for _, trace := range traces {
+			if len(trace.Spans) == 0 {
+				continue
+			}
+
+			traceIds = append(traceIds, trace.Spans[0].TraceID)
+			spans = append(spans, trace.Spans...)
+		}
+
+		return &extension.FetchResult{
+			Identifier: identifier{
+				TraceIds: traceIds,
+				Start:    start,
+				End:      end,
+			},
+			Spans: spans,
+		}, nil
+	}
+
+	return nil, nil
+}
+
+func (provider *Provider) findTraces(ctx context.Context, traceUrl string, queryArgs map[string]string) ([]*model.Trace, error) {
+	query := url.Values{}
+	for k, v := range queryArgs {
+		query.Add(k, v)
+	}
+	traceUrl = fmt.Sprintf("%s?%s", traceUrl, query.Encode())
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, traceUrl, nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	body, err := io.ReadAll(res.Body)
+	defer res.Body.Close()
+	if res.StatusCode > 299 {
+		return nil, fmt.Errorf("response failed with status code: %d and body: %s", res.StatusCode, string(body))
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read response error: %w", err)
+	}
+
+	var traces TraceResponse
+	if err := json.Unmarshal(body, &traces); err != nil {
+		return nil, fmt.Errorf("cannot parse HTTP extension response: %w", err)
+	}
+
+	return traces.Data, nil
+}
+
+type TraceResponse struct {
+	Data []*model.Trace
+}
+
+// TODO
+func (provider *Provider) getTrace(ctx context.Context, traceId model.TraceID) (*model.Trace, error) {
+	return nil, fmt.Errorf("trace cache for HTTP extension is unimplemented")
+}
+
+func templateFuncs() template.FuncMap {
+	return template.FuncMap{
+		"unixMicro": unixMicro,
+	}
+}
+
+func unixMicro(t interface{}) string {
+	if tt, ok := t.(time.Time); ok {
+		return fmt.Sprintf("%v", tt.UnixMicro())
+	}
+	return ""
+}
diff --git a/pkg/frontend/extension/jaeger-storage/storage.go b/pkg/frontend/extension/jaeger-storage/storage.go
new file mode 100644
index 00000000..3fecb2a3
--- /dev/null
+++ b/pkg/frontend/extension/jaeger-storage/storage.go
@@ -0,0 +1,347 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extensionjaeger
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"text/template"
+	"time"
+
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/jaegertracing/jaeger/pkg/metrics"
+	jaegerstorage "github.com/jaegertracing/jaeger/plugin/storage"
+	"github.com/jaegertracing/jaeger/storage/spanstore"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+	"go.uber.org/zap"
+
+	"github.com/kubewharf/kelemetry/pkg/frontend/extension"
+	"github.com/kubewharf/kelemetry/pkg/manager"
+	utiljaeger "github.com/kubewharf/kelemetry/pkg/util/jaeger"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+)
+
+func init() {
+	manager.Global.ProvideListImpl("frontend-extension/jaeger-storage", manager.Ptr(&Storage{}), &manager.List[extension.ProviderFactory]{})
+}
+
+const jaegerStorageKind = "JaegerStorage"
+
+type Storage struct {
+	manager.BaseComponent
+	Logger logrus.FieldLogger
+}
+
+func (*Storage) ListIndex() string { return jaegerStorageKind }
+
+func getViper() (*viper.Viper, error) {
+	factoryConfig := jaegerstorage.FactoryConfig{
+		SpanWriterTypes:         utiljaeger.SpanStorageTypesToAddFlag,
+		SpanReaderType:          "elasticsearch",
+		DependenciesStorageType: "elasticsearch",
+	}
+	storageFactory, err := jaegerstorage.NewFactory(factoryConfig)
+	if err != nil {
+		return nil, fmt.Errorf("Cannot initialize storage factory: %w", err)
+	}
+
+	viper := viper.New()
+
+	goFs := &flag.FlagSet{}
+	storageFactory.AddFlags(goFs)
+	pfs := &pflag.FlagSet{}
+	pfs.AddGoFlagSet(goFs)
+	if err := viper.BindPFlags(pfs); err != nil {
+		return nil, fmt.Errorf("cannot bind extension storage factory flags: %w", err)
+	}
+
+	return viper, nil
+}
+
+const DefaultNumTracesLimit = 5
+
+func (s *Storage) Configure(jsonBuf []byte) (extension.Provider, error) {
+	providerArgs := &ProviderArgs{NumTracesLimit: DefaultNumTracesLimit}
+	if err := json.Unmarshal(jsonBuf, providerArgs); err != nil {
+		return nil, fmt.Errorf("cannot parse jaeger-storage config: %w", err)
+	}
+
+	viper, err := getViper()
+	if err != nil {
+		return nil, err
+	}
+
+	for argKey, argValue := range providerArgs.StorageArgs {
+		viper.Set(argKey, argValue)
+	}
+
+	storageType := viper.GetString("span-storage.type")
+	if len(storageType) == 0 {
+		return nil, fmt.Errorf("span-storage.type must be specified for jaeger-storage extension")
+	}
+
+	factoryConfig := jaegerstorage.FactoryConfig{
+		SpanReaderType:          storageType,
+		SpanWriterTypes:         []string{storageType},
+		DependenciesStorageType: storageType,
+	}
+
+	storageFactory, err := jaegerstorage.NewFactory(factoryConfig)
+	if err != nil {
+		return nil, fmt.Errorf("cannot instantiate extension storage factory: %w", err)
+	}
+
+	zapLogger := zap.NewExample()
+
+	storageFactory.InitFromViper(viper, zapLogger)
+	if err := storageFactory.Initialize(metrics.NullFactory, zapLogger); err != nil {
+		return nil, fmt.Errorf("cannot initialize extension storage: %w", err)
+	}
+
+	reader, err := storageFactory.CreateSpanReader()
+	if err != nil {
+		return nil, fmt.Errorf("cannot create span reader for extension storage: %w", err)
+	}
+
+	tagTemplates := map[string]*template.Template{}
+	for tagKey, templateString := range providerArgs.TagTemplates {
+		t, err := template.New(tagKey).Parse(templateString)
+		if err != nil {
+			return nil, fmt.Errorf("invalid tag template %q: %w", tagKey, err)
+		}
+		tagTemplates[tagKey] = t
+	}
+
+	totalTimeout, err := time.ParseDuration(providerArgs.TotalTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("parse totalTimeout error: %w", err)
+	}
+
+	return &Provider{
+		reader: reader,
+		logger: s.Logger,
+
+		service:      providerArgs.Service,
+		operation:    providerArgs.Operation,
+		tagTemplates: tagTemplates,
+		numTraces:    providerArgs.NumTracesLimit,
+
+		forObject:     providerArgs.ForObject,
+		forAuditEvent: providerArgs.ForAuditEvent,
+
+		totalTimeout:   totalTimeout,
+		maxConcurrency: providerArgs.MaxConcurrency,
+
+		rawConfig: jsonBuf,
+	}, nil
+}
+
+type ProviderArgs struct {
+	StorageArgs map[string]string `json:"storageArgs"`
+
+	Service        string            `json:"service"`
+	Operation      string            `json:"operation"`
+	TagTemplates   map[string]string `json:"tagTemplates"`
+	NumTracesLimit int               `json:"numTracesLimit"`
+
+	ForObject     bool `json:"forObject"`
+	ForAuditEvent bool `json:"forAuditEvent"`
+
+	TotalTimeout   string `json:"totalTimeout"`
+	MaxConcurrency int    `json:"maxConcurrency"`
+}
+
+type Provider struct {
+	reader spanstore.Reader
+	logger logrus.FieldLogger
+
+	service      string
+	operation    string
+	tagTemplates map[string]*template.Template
+	numTraces    int
+
+	forObject     bool
+	forAuditEvent bool
+
+	totalTimeout   time.Duration
+	maxConcurrency int
+
+	rawConfig []byte
+}
+
+func (provider *Provider) Kind() string { return jaegerStorageKind }
+
+func (provider *Provider) RawConfig() []byte { return provider.rawConfig }
+
+func (provider *Provider) TotalTimeout() time.Duration { return provider.totalTimeout }
+func (provider *Provider) MaxConcurrency() int         { return provider.maxConcurrency }
+
+func (provider *Provider) FetchForObject(
+	ctx context.Context,
+	object utilobject.Rich,
+	mainTags model.KeyValues,
+	start, end time.Time,
+) (*extension.FetchResult, error) {
+	if !provider.forObject {
+		return nil, nil
+	}
+
+	return provider.fetch(
+		ctx, mainTags, start, end,
+		objectTemplateArgs(object),
+	)
+}
+
+func (provider *Provider) FetchForVersion(
+	ctx context.Context,
+	object utilobject.Rich,
+	resourceVersion string,
+	mainTags model.KeyValues,
+	start, end time.Time,
+) (*extension.FetchResult, error) {
+	if !provider.forAuditEvent {
+		return nil, nil
+	}
+
+	templateArgs := objectTemplateArgs(object)
+	templateArgs["resourceVersion"] = resourceVersion
+
+	return provider.fetch(
+		ctx, mainTags, start, end,
+		templateArgs,
+	)
+}
+
+func objectTemplateArgs(object utilobject.Rich) map[string]any {
+	var objectApiPath string
+	if object.Group == "" {
+		objectApiPath = fmt.Sprintf("/api/%s", object.Version)
+	} else {
+		objectApiPath = fmt.Sprintf("/apis/%s/%s", object.Group, object.Version)
+	}
+
+	if object.Namespace != "" && object.Resource != "namespaces" {
+		objectApiPath += fmt.Sprintf("/namespaces/%s", object.Namespace)
+	}
+
+	objectApiPath += fmt.Sprintf("/%s/%s", object.Resource, object.Name)
+
+	return map[string]any{
+		"cluster":       object.Cluster,
+		"group":         object.Group,
+		"version":       object.Version,
+		"resource":      object.Resource,
+		"groupVersion":  object.GroupVersion().String(),
+		"groupResource": object.GroupResource().String(),
+		"namespace":     object.Namespace,
+		"name":          object.Name,
+		"objectApiPath": objectApiPath,
+	}
+}
+
+func (provider *Provider) fetch(
+	ctx context.Context,
+	mainTags model.KeyValues,
+	start, end time.Time,
+	templateArgs map[string]any,
+) (*extension.FetchResult, error) {
+	queryTags := map[string]string{}
+
+	for _, mainTag := range mainTags {
+		templateArgs[mainTag.Key] = mainTag.Value()
+	}
+
+	for tagKey, tagTemplate := range provider.tagTemplates {
+		buf := new(bytes.Buffer)
+		err := tagTemplate.Execute(buf, templateArgs)
+		if err != nil {
+			return nil, fmt.Errorf("cannot execute tag template: %w", err)
+		}
+
+		queryTags[tagKey] = buf.String()
+	}
+
+	provider.logger.
+		WithField("service", provider.service).
+		WithField("operation", provider.operation).
+		WithField("tags", queryTags).
+		WithField("startTime", start).
+		WithField("endTime", end).
+		Info("search extension traces")
+
+	traces, err := provider.reader.FindTraces(ctx, &spanstore.TraceQueryParameters{
+		ServiceName:   provider.service,
+		OperationName: provider.operation,
+		Tags:          queryTags,
+		StartTimeMin:  start,
+		StartTimeMax:  end,
+		NumTraces:     provider.numTraces,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("cannot find traces: %w", err)
+	}
+
+	traceIds := []model.TraceID{}
+	spans := []*model.Span{}
+
+	for _, trace := range traces {
+		if len(trace.Spans) == 0 {
+			continue
+		}
+
+		traceIds = append(traceIds, trace.Spans[0].TraceID)
+		spans = append(spans, trace.Spans...)
+	}
+
+	return &extension.FetchResult{
+		Identifier: identifier{
+			TraceIds: traceIds,
+			Start:    start,
+			End:      end,
+		},
+		Spans: spans,
+	}, nil
+}
+
+type identifier struct {
+	TraceIds []model.TraceID `json:"traceIds"`
+	Start    time.Time       `json:"start"`
+	End      time.Time       `json:"end"`
+}
+
+func (provider *Provider) LoadCache(ctx context.Context, jsonBuf []byte) ([]*model.Span, error) {
+	var ident identifier
+	if err := json.Unmarshal(jsonBuf, &ident); err != nil {
+		return nil, fmt.Errorf("cannot unmarshal cached identifier: %w", err)
+	}
+
+	spans := []*model.Span{}
+
+	for _, traceId := range ident.TraceIds {
+		trace, err := provider.reader.GetTrace(ctx, traceId)
+		if err != nil {
+			return nil, fmt.Errorf("cannot get trace from extension storage: %w", err)
+		}
+
+		spans = append(spans, trace.Spans...)
+	}
+
+	return spans, nil
+}
diff --git a/pkg/frontend/extension/provider.go b/pkg/frontend/extension/provider.go
new file mode 100644
index 00000000..865a6ec4
--- /dev/null
+++ b/pkg/frontend/extension/provider.go
@@ -0,0 +1,95 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extension
+
+import (
+	"context"
+	"time"
+
+	"github.com/jaegertracing/jaeger/model"
+
+	"github.com/kubewharf/kelemetry/pkg/manager"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+)
+
+// Produces new extension providers based on the configuration.
+//
+// This interface is used for manager.List.
+type ProviderFactory interface {
+	manager.IndexedListImpl
+
+	Configure(jsonBuf []byte) (Provider, error)
+}
+
+// An instance of extension provider as specified in the config.
+// Loads spans from a specific source, typically from one specific component.
+type Provider interface {
+	// Kind of this extension provider, same as the one from `ProviderFactory.Kind()`
+	Kind() string
+
+	// The raw JSON config buffer used to configure this provider.
+	RawConfig() []byte
+
+	// Maximum wall time to execute all fetch calls.
+	// Only successuful queries before this timestamp will be included in the output.
+	TotalTimeout() time.Duration
+	// Maximum concurrent fetch calls to execute for the same trace.
+	MaxConcurrency() int
+
+	// Searches for spans associated with an object.
+	//
+	// This method is invoked during the first time a main trace is loaded.
+	// Subsequent invocations will call `LoadCache` instead.
+	//
+	// Returned traces are attached as subtrees under the object pseudospan.
+	//
+	// `tags` contains the tags in the object pseudospan in the main trace.
+	FetchForObject(
+		ctx context.Context,
+		object utilobject.Rich,
+		tags model.KeyValues,
+		start, end time.Time,
+	) (*FetchResult, error)
+
+	// Searches for spans associated with an object version.
+	//
+	// This method is invoked during the first time a main trace is loaded.
+	// Subsequent invocations will call `LoadCache` instead.
+	//
+	// Returned traces are attached as subtrees under the audit span.
+	//
+	// `tags` contains the tags in the object pseudospan in the main trace.
+	FetchForVersion(
+		ctx context.Context,
+		object utilobject.Rich,
+		resourceVersion string,
+		tags model.KeyValues,
+		start, end time.Time,
+	) (*FetchResult, error)
+
+	// Restores the FetchResult from the identifier
+	// persisted from the result of a previous call to `Fetch`.
+	LoadCache(ctx context.Context, identifier []byte) ([]*model.Span, error)
+}
+
+type FetchResult struct {
+	// A JSON-serializable object used to persist the parameters of this fetch,
+	// such as the extension trace ID.
+	Identifier any
+	// The spans in the extension trace.
+	//
+	// Orphan spans (with no or invalid parent reference) are treated as root spans.
+	Spans []*model.Span
+}
diff --git a/pkg/frontend/http/trace/server.go b/pkg/frontend/http/trace/server.go
index c7799f90..2f5466d2 100644
--- a/pkg/frontend/http/trace/server.go
+++ b/pkg/frontend/http/trace/server.go
@@ -105,7 +105,11 @@ func (server *server) handleTrace(ctx *gin.Context, metric *requestMetric) (code
 		return 400, fmt.Errorf("invalid param %w", err)
 	}
 
-	trace, code, err := server.findTrace(metric, "tracing (exclusive)", query)
+	if query.DisplayMode == "" {
+		query.DisplayMode = "tracing [exclusive]"
+	}
+
+	trace, code, err := server.findTrace(metric, query.DisplayMode, query)
 	if err != nil {
 		return code, err
 	}
@@ -149,12 +153,14 @@ func pruneTrace(trace *model.Trace, spanType string) {
 }
 
 type traceQuery struct {
-	Cluster   string `form:"cluster"`
-	Resource  string `form:"resource"`
-	Namespace string `form:"namespace"`
-	Name      string `form:"name"`
-	Ts        string `form:"ts"`
-	SpanType  string `form:"span_type"`
+	Cluster     string `form:"cluster"`
+	Resource    string `form:"resource"`
+	Namespace   string `form:"namespace"`
+	Name        string `form:"name"`
+	Start       string `form:"start"`
+	End         string `form:"end"`
+	SpanType    string `form:"span_type"`
+	DisplayMode string `form:"displayMode"`
 }
 
 func (server *server) findTrace(metric *requestMetric, serviceName string, query traceQuery) (trace *model.Trace, code int, err error) {
@@ -179,10 +185,16 @@ func (server *server) findTrace(metric *requestMetric, serviceName string, query
 		return nil, 404, fmt.Errorf("cluster %s not supported now", cluster)
 	}
 
-	timestamp, err := time.Parse(time.RFC3339, query.Ts)
+	startTimestamp, err := time.Parse(time.RFC3339, query.Start)
+	if err != nil {
+		metric.Error = metrics.MakeLabeledError("InvalidTimestamp")
+		return nil, 400, fmt.Errorf("invalid timestamp for start param %w", err)
+	}
+
+	endTimestamp, err := time.Parse(time.RFC3339, query.End)
 	if err != nil {
 		metric.Error = metrics.MakeLabeledError("InvalidTimestamp")
-		return nil, 400, fmt.Errorf("invalid timestamp for ts param %w", err)
+		return nil, 400, fmt.Errorf("invalid timestamp for end param %w", err)
 	}
 
 	tags := map[string]string{
@@ -197,8 +209,9 @@ func (server *server) findTrace(metric *requestMetric, serviceName string, query
 		ServiceName:   serviceName,
 		OperationName: cluster,
 		Tags:          tags,
-		StartTimeMin:  timestamp.Truncate(time.Minute * 30),
-		StartTimeMax:  timestamp.Truncate(time.Minute * 30).Add(time.Minute * 30),
+		StartTimeMin:  startTimestamp,
+		StartTimeMax:  endTimestamp,
+		NumTraces:     20,
 	}
 	traces, err := server.SpanReader.FindTraces(context.Background(), parameters)
 	if err != nil {
diff --git a/pkg/frontend/reader/merge.go b/pkg/frontend/reader/merge.go
index 3a95e551..d35d5c38 100644
--- a/pkg/frontend/reader/merge.go
+++ b/pkg/frontend/reader/merge.go
@@ -19,21 +19,21 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 
 	jaegerbackend "github.com/kubewharf/kelemetry/pkg/frontend/backend"
-	tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 type mergeMap struct {
 	ptrSet   sets.Set[*mergeEntry]
-	fromKeys map[tftree.GroupingKey]*mergeEntry
+	fromKeys map[utilobject.Key]*mergeEntry
 }
 
 type mergeEntry struct {
-	keys        sets.Set[tftree.GroupingKey]
+	keys        sets.Set[utilobject.Key]
 	identifiers []any
 	spans       []*model.Span
 }
 
-func singletonMerged(keys sets.Set[tftree.GroupingKey], thumbnail *jaegerbackend.TraceThumbnail) *mergeEntry {
+func singletonMerged(keys sets.Set[utilobject.Key], thumbnail *jaegerbackend.TraceThumbnail) *mergeEntry {
 	return &mergeEntry{
 		keys:        keys,
 		identifiers: []any{thumbnail.Identifier},
@@ -51,7 +51,7 @@ func (entry *mergeEntry) join(other *mergeEntry) {
 }
 
 // add a thumbnail with a preferred root key.
-func (m *mergeMap) add(keys sets.Set[tftree.GroupingKey], thumbnail *jaegerbackend.TraceThumbnail) {
+func (m *mergeMap) add(keys sets.Set[utilobject.Key], thumbnail *jaegerbackend.TraceThumbnail) {
 	entry := singletonMerged(keys.Clone(), thumbnail)
 	m.ptrSet.Insert(entry)
 
@@ -77,11 +77,11 @@ func (m *mergeMap) add(keys sets.Set[tftree.GroupingKey], thumbnail *jaegerbacke
 func mergeSegments(thumbnails []*jaegerbackend.TraceThumbnail) []*mergeEntry {
 	m := mergeMap{
 		ptrSet:   sets.New[*mergeEntry](),
-		fromKeys: map[tftree.GroupingKey]*mergeEntry{},
+		fromKeys: map[utilobject.Key]*mergeEntry{},
 	}
 
 	for _, thumbnail := range thumbnails {
-		keys := tftree.GroupingKeysFromSpans(thumbnail.Spans)
+		keys := utilobject.FromSpans(thumbnail.Spans)
 		m.add(keys, thumbnail)
 	}
 
diff --git a/pkg/frontend/reader/reader.go b/pkg/frontend/reader/reader.go
index 38930e34..08b86ebf 100644
--- a/pkg/frontend/reader/reader.go
+++ b/pkg/frontend/reader/reader.go
@@ -31,9 +31,9 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/frontend/clusterlist"
 	transform "github.com/kubewharf/kelemetry/pkg/frontend/tf"
 	tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
-	tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
 	"github.com/kubewharf/kelemetry/pkg/frontend/tracecache"
 	"github.com/kubewharf/kelemetry/pkg/manager"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 	"github.com/kubewharf/kelemetry/pkg/util/zconstants"
 )
 
@@ -45,9 +45,17 @@ type Interface interface {
 	spanstore.Reader
 }
 
-type options struct{}
+type options struct {
+	cacheExtensions bool
+}
 
 func (options *options) Setup(fs *pflag.FlagSet) {
+	fs.BoolVar(
+		&options.cacheExtensions,
+		"jaeger-extension-cache",
+		false,
+		"cache extension trace search result, otherwise trace is searched again every time result is reloaded",
+	)
 }
 
 func (options *options) EnableFlag() *bool { return nil }
@@ -127,8 +135,8 @@ func (reader *spanReader) FindTraces(ctx context.Context, query *spanstore.Trace
 		return nil, err
 	}
 
-	var rootKey *tftree.GroupingKey
-	if rootKeyValue, ok := tftree.GroupingKeyFromMap(query.Tags); ok {
+	var rootKey *utilobject.Key
+	if rootKeyValue, ok := utilobject.FromMap(query.Tags); ok {
 		rootKey = &rootKeyValue
 	}
 
@@ -139,26 +147,6 @@ func (reader *spanReader) FindTraces(ctx context.Context, query *spanstore.Trace
 	for _, entry := range mergedEntries {
 		cacheId := generateCacheId(config.Id)
 
-		identifiers := make([]json.RawMessage, len(entry.identifiers))
-		for i, identifier := range entry.identifiers {
-			idJson, err := json.Marshal(identifier)
-			if err != nil {
-				return nil, fmt.Errorf("thumbnail identifier marshal: %w", err)
-			}
-
-			identifiers[i] = json.RawMessage(idJson)
-		}
-
-		cacheEntries = append(cacheEntries, tracecache.Entry{
-			LowId: cacheId.Low,
-			Value: tracecache.EntryValue{
-				Identifiers: identifiers,
-				StartTime:   query.StartTimeMin,
-				EndTime:     query.StartTimeMax,
-				RootObject:  rootKey,
-			},
-		})
-
 		for _, span := range entry.spans {
 			span.TraceID = cacheId
 			for i := range span.References {
@@ -178,10 +166,40 @@ func (reader *spanReader) FindTraces(ctx context.Context, query *spanstore.Trace
 
 		displayMode := extractDisplayMode(cacheId)
 
-		if err := reader.Transformer.Transform(trace, rootKey, displayMode); err != nil {
+		extensions := &transform.FetchExtensionsAndStoreCache{}
+
+		if err := reader.Transformer.Transform(
+			ctx, trace, rootKey, displayMode,
+			extensions,
+			query.StartTimeMin, query.StartTimeMax,
+		); err != nil {
 			return nil, fmt.Errorf("trace transformation failed: %w", err)
 		}
 		traces = append(traces, trace)
+
+		identifiers := make([]json.RawMessage, len(entry.identifiers))
+		for i, identifier := range entry.identifiers {
+			idJson, err := json.Marshal(identifier)
+			if err != nil {
+				return nil, fmt.Errorf("thumbnail identifier marshal: %w", err)
+			}
+
+			identifiers[i] = json.RawMessage(idJson)
+		}
+
+		cacheEntry := tracecache.Entry{
+			LowId: cacheId.Low,
+			Value: tracecache.EntryValue{
+				Identifiers: identifiers,
+				StartTime:   query.StartTimeMin,
+				EndTime:     query.StartTimeMax,
+				RootObject:  rootKey,
+			},
+		}
+		if reader.options.cacheExtensions {
+			cacheEntry.Value.Extensions = extensions.Cache
+		}
+		cacheEntries = append(cacheEntries, cacheEntry)
 	}
 
 	if len(cacheEntries) > 0 {
@@ -221,8 +239,17 @@ func (reader *spanReader) GetTrace(ctx context.Context, cacheId model.TraceID) (
 		aggTrace.Spans = append(aggTrace.Spans, clipped...)
 	}
 
+	var extensions transform.ExtensionProcessor = &transform.FetchExtensionsAndStoreCache{}
+	if reader.options.cacheExtensions && len(entry.Extensions) > 0 {
+		extensions = &transform.LoadExtensionCache{Cache: entry.Extensions}
+	}
+
 	displayMode := extractDisplayMode(cacheId)
-	if err := reader.Transformer.Transform(aggTrace, entry.RootObject, displayMode); err != nil {
+	if err := reader.Transformer.Transform(
+		ctx, aggTrace, entry.RootObject, displayMode,
+		extensions,
+		entry.StartTime, entry.EndTime,
+	); err != nil {
 		return nil, fmt.Errorf("trace transformation failed: %w", err)
 	}
 
diff --git a/pkg/frontend/tf/config/config.go b/pkg/frontend/tf/config/config.go
index 53460c85..b5bd4849 100644
--- a/pkg/frontend/tf/config/config.go
+++ b/pkg/frontend/tf/config/config.go
@@ -17,6 +17,7 @@ package tfconfig
 import (
 	"strconv"
 
+	"github.com/kubewharf/kelemetry/pkg/frontend/extension"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 )
 
@@ -53,6 +54,8 @@ type Config struct {
 	// If true, only displays the spans below the matched span.
 	// If false, displays the whole trace including parent and sibling spans.
 	UseSubtree bool
+	// The extension traces for this config.
+	Extensions []extension.Provider
 	// The steps to transform the tree
 	Steps []Step
 }
diff --git a/pkg/frontend/tf/config/file/file.go b/pkg/frontend/tf/config/file/file.go
index 410f9634..af77b9c9 100644
--- a/pkg/frontend/tf/config/file/file.go
+++ b/pkg/frontend/tf/config/file/file.go
@@ -20,8 +20,10 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"sort"
 
 	"github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/yaml"
 
 	tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
@@ -30,7 +32,7 @@ import (
 
 func init() {
 	manager.Global.ProvideMuxImpl("jaeger-transform-config/file", manager.Ptr(&FileProvider{
-		configs:        make(map[tfconfig.Id]*tfconfig.Config),
+		configs:        make(map[tfconfig.Id]registeredConfig),
 		nameToConfigId: make(map[string]tfconfig.Id),
 	}), tfconfig.Provider.DefaultId)
 }
@@ -49,15 +51,20 @@ type FileProvider struct {
 	manager.MuxImplBase
 
 	RegisteredSteps     *manager.List[tfconfig.RegisteredStep]
-	RegisteredModifiers *manager.List[tfconfig.Modifier]
+	RegisteredModifiers *manager.List[tfconfig.ModifierFactory]
 
 	options        options
 	names          []string
-	configs        map[tfconfig.Id]*tfconfig.Config
+	configs        map[tfconfig.Id]registeredConfig
 	nameToConfigId map[string]tfconfig.Id
 	defaultConfig  tfconfig.Id
 }
 
+type registeredConfig struct {
+	modifierClasses sets.Set[string]
+	config          *tfconfig.Config
+}
+
 var (
 	_ manager.Component = &FileProvider{}
 	_ tfconfig.Provider = &FileProvider{}
@@ -97,47 +104,71 @@ func (p *FileProvider) loadJsonBytes(jsonBytes []byte) error {
 		Steps json.RawMessage `json:"steps"`
 	}
 
+	type modifierConfig struct {
+		Priority     int64           `json:"priority"`
+		DisplayName  string          `json:"displayName"`
+		ModifierName string          `json:"modifierName"`
+		Args         json.RawMessage `json:"args"`
+	}
+
 	var file struct {
-		Modifiers map[string]tfconfig.Id `json:"modifiers"`
-		Batches   []Batch                `json:"batches"`
+		Modifiers map[tfconfig.Id]modifierConfig `json:"modifiers"`
+		Batches   []Batch                        `json:"batches"`
 		Configs   []struct {
 			Id         tfconfig.Id     `json:"id"`
 			Name       string          `json:"name"`
 			UseSubtree bool            `json:"useSubtree"`
 			Steps      json.RawMessage `json:"steps"`
-		}
+		} `json:"configs"`
+		DefaultConfig tfconfig.Id `json:"defaultConfig"`
 	}
 	if err := json.Unmarshal(jsonBytes, &file); err != nil {
 		return fmt.Errorf("parse tfconfig error: %w", err)
 	}
 
-	modifiers := make([]func(*tfconfig.Config), 0, len(file.Modifiers))
-	for modifierName, bitmask := range file.Modifiers {
-		modifierName := modifierName
+	p.defaultConfig = file.DefaultConfig
+
+	type knownModifier struct {
+		class    string
+		priority int64
+		fn       func(config *tfconfig.Config)
+	}
+
+	modifiers := make([]knownModifier, 0, len(file.Modifiers))
+
+	for bitmask, modifierConfig := range file.Modifiers {
 		bitmask := bitmask
+		modifierConfig := modifierConfig
 
-		var matched tfconfig.Modifier
-		for _, modifier := range p.RegisteredModifiers.Impls {
-			if modifierName == modifier.ModifierName() {
-				matched = modifier
-				break
-			}
+		displayName := modifierConfig.DisplayName
+
+		factory, hasFactory := p.RegisteredModifiers.Indexed[modifierConfig.ModifierName]
+		if !hasFactory {
+			return fmt.Errorf("parse tfconfig modifier error: unknown modifier name %q", modifierConfig.ModifierName)
 		}
 
-		if matched == nil {
-			return fmt.Errorf("parse tfconfig modifier error: unknown modifier name %q", modifierName)
+		modifier, err := factory.Build([]byte(modifierConfig.Args))
+		if err != nil {
+			return fmt.Errorf("parse tfconfig modifier error: invalid modifier args: %w", err)
 		}
 
-		modifiers = append(modifiers, func(config *tfconfig.Config) {
-			config.Id |= bitmask
-			config.Name += fmt.Sprintf(" [%s]", modifierName)
-			matched.Modify(config)
+		modifiers = append(modifiers, knownModifier{
+			class:    modifier.ModifierClass(),
+			priority: modifierConfig.Priority,
+			fn: func(config *tfconfig.Config) {
+				config.Id |= bitmask
+				config.Name += fmt.Sprintf(" [%s]", displayName)
+				modifier.Modify(config)
+			},
 		})
 	}
 
+	// Sort modifiers by the ascending order of priorities
+	sort.Slice(modifiers, func(i, j int) bool { return modifiers[i].priority < modifiers[j].priority })
+
 	batches := map[string][]tfconfig.Step{}
 	for _, batch := range file.Batches {
-		steps, err := tfconfig.ParseSteps(batch.Steps, batches, p.RegisteredSteps.Impls)
+		steps, err := tfconfig.ParseSteps(batch.Steps, batches, p.RegisteredSteps.Indexed)
 		if err != nil {
 			return fmt.Errorf("parse tfconfig batch error: %w", err)
 		}
@@ -146,7 +177,7 @@ func (p *FileProvider) loadJsonBytes(jsonBytes []byte) error {
 	}
 
 	for _, raw := range file.Configs {
-		steps, err := tfconfig.ParseSteps(raw.Steps, batches, p.RegisteredSteps.Impls)
+		steps, err := tfconfig.ParseSteps(raw.Steps, batches, p.RegisteredSteps.Indexed)
 		if err != nil {
 			return fmt.Errorf("parse tfconfig step error: %w", err)
 		}
@@ -158,16 +189,28 @@ func (p *FileProvider) loadJsonBytes(jsonBytes []byte) error {
 			Steps:      steps,
 		}
 
-		p.register(config)
+		p.register(registeredConfig{config: config, modifierClasses: sets.New[string]()})
 	}
 
 	for _, modifier := range modifiers {
-		var newEntries []*tfconfig.Config
+		var newEntries []registeredConfig
 
 		for _, config := range p.configs {
-			newConfig := config.Clone()
-			modifier(newConfig)
-			newEntries = append(newEntries, newConfig)
+			newConfig := config.config.Clone()
+			modifier.fn(newConfig)
+
+			if config.modifierClasses.Has(modifier.class) {
+				// incompatible combination
+				continue
+			}
+
+			classes := config.modifierClasses.Clone()
+			classes.Insert(modifier.class)
+
+			newEntries = append(newEntries, registeredConfig{
+				config:          newConfig,
+				modifierClasses: classes,
+			})
 		}
 
 		for _, newEntry := range newEntries {
@@ -178,9 +221,11 @@ func (p *FileProvider) loadJsonBytes(jsonBytes []byte) error {
 	return nil
 }
 
-func (p *FileProvider) register(config *tfconfig.Config) {
+func (p *FileProvider) register(regConfig registeredConfig) {
+	config := regConfig.config
+
 	p.names = append(p.names, config.Name)
-	p.configs[config.Id] = config
+	p.configs[config.Id] = regConfig
 	p.nameToConfigId[config.Name] = config.Id
 }
 
@@ -191,7 +236,7 @@ func (p *FileProvider) Names() []string {
 	return p.names
 }
 
-func (p *FileProvider) DefaultName() string { return p.configs[p.defaultConfig].Name }
+func (p *FileProvider) DefaultName() string { return p.configs[p.defaultConfig].config.Name }
 
 func (p *FileProvider) DefaultId() tfconfig.Id { return p.defaultConfig }
 
@@ -200,7 +245,7 @@ func (p *FileProvider) GetByName(name string) *tfconfig.Config {
 	if !exists {
 		return nil
 	}
-	return p.configs[id]
+	return p.configs[id].config
 }
 
-func (p *FileProvider) GetById(id tfconfig.Id) *tfconfig.Config { return p.configs[id] }
+func (p *FileProvider) GetById(id tfconfig.Id) *tfconfig.Config { return p.configs[id].config }
diff --git a/pkg/frontend/tf/config/modifier.go b/pkg/frontend/tf/config/modifier.go
index 36107f63..958efcbd 100644
--- a/pkg/frontend/tf/config/modifier.go
+++ b/pkg/frontend/tf/config/modifier.go
@@ -14,8 +14,18 @@
 
 package tfconfig
 
+import "github.com/kubewharf/kelemetry/pkg/manager"
+
+type ModifierFactory interface {
+	manager.IndexedListImpl
+
+	Build(jsonBuf []byte) (Modifier, error)
+}
+
 type Modifier interface {
-	ModifierName() string
+	// Modifiers with the same class are incompatible with one another.
+	// Used to reduce cardinality of available display modes in frontend.
+	ModifierClass() string
 
 	Modify(config *Config)
 }
diff --git a/pkg/frontend/tf/config/step.go b/pkg/frontend/tf/config/step.go
index 8a955d2c..913401c7 100644
--- a/pkg/frontend/tf/config/step.go
+++ b/pkg/frontend/tf/config/step.go
@@ -28,6 +28,7 @@ type Step interface {
 
 type RegisteredStep interface {
 	Step
+	manager.IndexedListImpl
 
 	Kind() string
 
@@ -56,7 +57,8 @@ func (step *VisitorStep[V]) Run(tree *tftree.SpanTree) {
 	tree.Visit(step.Visitor)
 }
 
-func (step *VisitorStep[V]) Kind() string { return step.Visitor.Kind() }
+func (step *VisitorStep[V]) ListIndex() string { return step.Visitor.Kind() }
+func (step *VisitorStep[V]) Kind() string      { return step.Visitor.Kind() }
 
 func (*VisitorStep[V]) UnmarshalNewJSON(buf []byte) (Step, error) {
 	step := &VisitorStep[V]{}
@@ -76,7 +78,7 @@ func (step *BatchStep) Run(tree *tftree.SpanTree) {
 	}
 }
 
-func ParseSteps(buf []byte, batches map[string][]Step, registeredSteps []RegisteredStep) ([]Step, error) {
+func ParseSteps(buf []byte, batches map[string][]Step, registeredSteps map[string]RegisteredStep) ([]Step, error) {
 	raw := []json.RawMessage{}
 
 	if err := json.Unmarshal(buf, &raw); err != nil {
@@ -96,7 +98,7 @@ func ParseSteps(buf []byte, batches map[string][]Step, registeredSteps []Registe
 	return steps, nil
 }
 
-func UnmarshalStep(batches map[string][]Step, buf []byte, registeredSteps []RegisteredStep) (Step, error) {
+func UnmarshalStep(batches map[string][]Step, buf []byte, registeredSteps map[string]RegisteredStep) (Step, error) {
 	var hasKind struct {
 		Kind string `json:"kind"`
 	}
@@ -120,10 +122,8 @@ func UnmarshalStep(batches map[string][]Step, buf []byte, registeredSteps []Regi
 		return &BatchStep{Steps: batch}, nil
 	}
 
-	for _, step := range registeredSteps {
-		if step.Kind() == hasKind.Kind {
-			return step.UnmarshalNewJSON(buf)
-		}
+	if step, hasStep := registeredSteps[hasKind.Kind]; hasStep {
+		return step.UnmarshalNewJSON(buf)
 	}
 
 	return nil, fmt.Errorf("unknown kind %q", hasKind.Kind)
diff --git a/pkg/frontend/tf/defaults/modifier/exclusive.go b/pkg/frontend/tf/defaults/modifier/exclusive.go
index a7cfc1dc..7b141675 100644
--- a/pkg/frontend/tf/defaults/modifier/exclusive.go
+++ b/pkg/frontend/tf/defaults/modifier/exclusive.go
@@ -24,7 +24,11 @@ import (
 )
 
 func init() {
-	manager.Global.ProvideListImpl("tf-modifier/exclusive", manager.Ptr(&ExclusiveModifier{}), &manager.List[tfconfig.Modifier]{})
+	manager.Global.ProvideListImpl(
+		"tf-modifier/exclusive",
+		manager.Ptr(&ExclusiveModifierFactory{}),
+		&manager.List[tfconfig.ModifierFactory]{},
+	)
 }
 
 type ExclusiveModifierOptions struct {
@@ -37,18 +41,26 @@ func (options *ExclusiveModifierOptions) Setup(fs *pflag.FlagSet) {
 
 func (options *ExclusiveModifierOptions) EnableFlag() *bool { return &options.enable }
 
-type ExclusiveModifier struct {
+type ExclusiveModifierFactory struct {
 	options ExclusiveModifierOptions
 }
 
-var _ manager.Component = &ExclusiveModifier{}
+var _ manager.Component = &ExclusiveModifierFactory{}
 
-func (m *ExclusiveModifier) Options() manager.Options        { return &m.options }
-func (m *ExclusiveModifier) Init() error                     { return nil }
-func (m *ExclusiveModifier) Start(ctx context.Context) error { return nil }
-func (m *ExclusiveModifier) Close(ctx context.Context) error { return nil }
+func (m *ExclusiveModifierFactory) Options() manager.Options        { return &m.options }
+func (m *ExclusiveModifierFactory) Init() error                     { return nil }
+func (m *ExclusiveModifierFactory) Start(ctx context.Context) error { return nil }
+func (m *ExclusiveModifierFactory) Close(ctx context.Context) error { return nil }
 
-func (*ExclusiveModifier) ModifierName() string { return "exclusive" }
+func (*ExclusiveModifierFactory) ListIndex() string { return "exclusive" }
+
+func (*ExclusiveModifierFactory) Build(jsonBuf []byte) (tfconfig.Modifier, error) {
+	return &ExclusiveModifier{}, nil
+}
+
+type ExclusiveModifier struct{}
+
+func (*ExclusiveModifier) ModifierClass() string { return "kelemetry.kubewharf.io/exclusive" }
 
 func (*ExclusiveModifier) Modify(config *tfconfig.Config) {
 	config.UseSubtree = true
diff --git a/pkg/frontend/tf/defaults/modifier/extension.go b/pkg/frontend/tf/defaults/modifier/extension.go
new file mode 100644
index 00000000..4a69d60c
--- /dev/null
+++ b/pkg/frontend/tf/defaults/modifier/extension.go
@@ -0,0 +1,98 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tfmodifier
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/spf13/pflag"
+
+	"github.com/kubewharf/kelemetry/pkg/frontend/extension"
+	tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
+	"github.com/kubewharf/kelemetry/pkg/manager"
+)
+
+func init() {
+	manager.Global.ProvideListImpl(
+		"tf-modifier/extension",
+		manager.Ptr(&ExtensionModifierFactory{}),
+		&manager.List[tfconfig.ModifierFactory]{},
+	)
+}
+
+type ExtensionModifierOptions struct {
+	enable bool
+}
+
+func (options *ExtensionModifierOptions) Setup(fs *pflag.FlagSet) {
+	fs.BoolVar(&options.enable, "jaeger-tf-extension-modifier-enable", true, "enable extension modifier")
+}
+
+func (options *ExtensionModifierOptions) EnableFlag() *bool { return &options.enable }
+
+type ExtensionModifierFactory struct {
+	options      ExtensionModifierOptions
+	ProviderList *manager.List[extension.ProviderFactory]
+}
+
+var _ manager.Component = &ExtensionModifierFactory{}
+
+func (m *ExtensionModifierFactory) Options() manager.Options        { return &m.options }
+func (m *ExtensionModifierFactory) Init() error                     { return nil }
+func (m *ExtensionModifierFactory) Start(ctx context.Context) error { return nil }
+func (m *ExtensionModifierFactory) Close(ctx context.Context) error { return nil }
+
+func (*ExtensionModifierFactory) ListIndex() string { return "extension" }
+
+func (m *ExtensionModifierFactory) Build(jsonBuf []byte) (tfconfig.Modifier, error) {
+	var hasKind struct {
+		Kind  string `json:"kind"`
+		Class string `json:"modifierClass"`
+	}
+
+	if err := json.Unmarshal(jsonBuf, &hasKind); err != nil || hasKind.Kind == "" {
+		return nil, fmt.Errorf("no extension kind specified: %w", err)
+	}
+
+	matchedFactory, hasMatchedFactory := m.ProviderList.Indexed[hasKind.Kind]
+	if !hasMatchedFactory {
+		return nil, fmt.Errorf("no extension provider with kind %q", hasKind.Kind)
+	}
+
+	provider, err := matchedFactory.Configure(jsonBuf)
+	if err != nil {
+		return nil, fmt.Errorf("parse extension provider config error: %w", err)
+	}
+
+	return &ExtensionModifier{
+		provider: provider,
+		class:    hasKind.Class,
+	}, nil
+}
+
+type ExtensionModifier struct {
+	class    string
+	provider extension.Provider
+}
+
+func (modifier *ExtensionModifier) ModifierClass() string {
+	return fmt.Sprintf("kelemetry.kubewharf.io/extension/%s", modifier.class)
+}
+
+func (modifier *ExtensionModifier) Modify(config *tfconfig.Config) {
+	config.Extensions = append(config.Extensions, modifier.provider)
+}
diff --git a/pkg/frontend/tf/defaults/step/collapse_nesting.go b/pkg/frontend/tf/defaults/step/collapse_nesting.go
index c028a46a..9d6bfb46 100644
--- a/pkg/frontend/tf/defaults/step/collapse_nesting.go
+++ b/pkg/frontend/tf/defaults/step/collapse_nesting.go
@@ -94,8 +94,10 @@ type AuditDiffClass struct {
 }
 
 func (classes *AuditDiffClassification) Get(prefix string) *AuditDiffClass {
-	if class, hasSpecific := classes.SpecificFields.fieldMap[prefix]; hasSpecific {
-		return class
+	for ptr := len(prefix); ptr > 0; ptr = strings.LastIndex(prefix[:ptr], ".") { // remove the last `.`-delimited substring if not found
+		if class, hasSpecific := classes.SpecificFields.fieldMap[prefix[:ptr]]; hasSpecific {
+			return class
+		}
 	}
 
 	return &classes.DefaultClass
diff --git a/pkg/frontend/tf/extension.go b/pkg/frontend/tf/extension.go
new file mode 100644
index 00000000..f3a0e252
--- /dev/null
+++ b/pkg/frontend/tf/extension.go
@@ -0,0 +1,255 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transform
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/jaegertracing/jaeger/model"
+
+	"github.com/kubewharf/kelemetry/pkg/frontend/extension"
+	"github.com/kubewharf/kelemetry/pkg/frontend/tracecache"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+	"github.com/kubewharf/kelemetry/pkg/util/semaphore"
+	"github.com/kubewharf/kelemetry/pkg/util/zconstants"
+)
+
+type ExtensionProcessor interface {
+	ProcessExtensions(
+		ctx context.Context,
+		transformer *Transformer,
+		extensions []extension.Provider,
+		spans []*model.Span,
+		start, end time.Time,
+	) ([]*model.Span, error)
+}
+
+type FetchExtensionsAndStoreCache struct {
+	Cache []tracecache.ExtensionCache
+}
+
+var _ ExtensionProcessor = &FetchExtensionsAndStoreCache{}
+
+func (x *FetchExtensionsAndStoreCache) ProcessExtensions(
+	ctx context.Context,
+	transformer *Transformer,
+	extensions []extension.Provider,
+	spans []*model.Span,
+	start, end time.Time,
+) ([]*model.Span, error) {
+	var mutex sync.Mutex
+
+	newSpans := []*model.Span{}
+
+	addNewSpans := func(extension extension.Provider, result *extension.FetchResult, under *model.Span) error {
+		if result == nil || len(result.Spans) == 0 {
+			return nil
+		}
+
+		identifierJson, err := json.Marshal(result.Identifier)
+		if err != nil {
+			return fmt.Errorf("encode extension trace cache identifier: %w", err)
+		}
+
+		mutex.Lock()
+		defer mutex.Unlock()
+
+		x.Cache = append(x.Cache, tracecache.ExtensionCache{
+			ParentTrace:      under.TraceID,
+			ParentSpan:       under.SpanID,
+			ProviderKind:     extension.Kind(),
+			ProviderConfig:   json.RawMessage(extension.RawConfig()),
+			CachedIdentifier: json.RawMessage(identifierJson),
+		})
+
+		setRootParent(result.Spans, under.TraceID, under.SpanID)
+		setTraceId(result.Spans, under.TraceID)
+
+		newSpans = append(newSpans, result.Spans...)
+
+		return nil
+	}
+
+	extensionSemaphores := make([]*semaphore.Semaphore, len(extensions))
+	for extId, extension := range extensions {
+		extensionSemaphores[extId] = semaphore.New(extension.MaxConcurrency())
+	}
+
+	for _, span := range spans {
+		span := span
+
+		tags := model.KeyValues(span.Tags)
+		if tag, exists := tags.FindByKey(zconstants.NestLevel); exists && tag.VStr == zconstants.NestLevelObject {
+			for extId, ext := range extensions {
+				ext := ext
+
+				objectRef, ok := objectRefFromTags(tags)
+				if !ok {
+					return nil, fmt.Errorf("expected object tags in nestingLevel=object spans")
+				}
+
+				extensionSemaphores[extId].Schedule(func(ctx context.Context) (semaphore.Publish, error) {
+					result, err := ext.FetchForObject(ctx, objectRef, tags, start, end)
+					if err != nil {
+						return nil, fmt.Errorf("fetch extension trace for object: %w", err)
+					}
+
+					return func() error { return addNewSpans(ext, result, span) }, nil
+				})
+			}
+		} else if tag, exists := tags.FindByKey(zconstants.TraceSource); exists && tag.VStr == zconstants.TraceSourceAudit {
+			for extId, ext := range extensions {
+				ext := ext
+
+				objectRef, ok := objectRefFromTags(tags)
+				if !ok {
+					return nil, fmt.Errorf("expected object tags in traceSource=audit spans")
+				}
+
+				resourceVersion, ok := tags.FindByKey("resourceVersion")
+				if !ok {
+					return nil, fmt.Errorf("expected resourceVersion tag in traceSource=audit spans")
+				}
+
+				extensionSemaphores[extId].Schedule(func(ctx context.Context) (semaphore.Publish, error) {
+					result, err := ext.FetchForVersion(ctx, objectRef, resourceVersion.VStr, tags, start, end)
+					if err != nil {
+						return nil, fmt.Errorf("fetch extension trace for object: %w", err)
+					}
+
+					return func() error { return addNewSpans(ext, result, span) }, nil
+				})
+			}
+		}
+	}
+
+	fullSem := semaphore.NewUnbounded()
+	for extId := range extensionSemaphores {
+		extId := extId
+
+		fullSem.Schedule(func(ctx context.Context) (semaphore.Publish, error) {
+			semRunCtx, cancelFunc := context.WithTimeout(ctx, extensions[extId].TotalTimeout())
+			defer cancelFunc()
+
+			err := extensionSemaphores[extId].Run(semRunCtx)
+			if err != nil {
+				if errors.Is(err, semRunCtx.Err()) {
+					transformer.Logger.WithField("extension", extensions[extId].Kind()).
+						Debug("context timed out, dropping error silently to carry results obtained before timeout")
+				} else {
+					return nil, err
+				}
+			}
+
+			return nil, nil
+		})
+	}
+
+	err := fullSem.Run(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return newSpans, nil
+}
+
+func objectRefFromTags(tags model.KeyValues) (utilobject.Rich, bool) {
+	var object utilobject.Rich
+
+	assign := func(name string, field *string) bool {
+		value, ok := tags.FindByKey(name)
+		if ok && value.VType == model.StringType {
+			*field = value.VStr
+			return true
+		}
+		return false
+	}
+
+	ok := assign("cluster", &object.Cluster) &&
+		assign("group", &object.Group) &&
+		assign("version", &object.Version) &&
+		assign("resource", &object.Resource) &&
+		assign("namespace", &object.Namespace) &&
+		assign("name", &object.Name)
+
+	return object, ok
+}
+
+type LoadExtensionCache struct {
+	Cache []tracecache.ExtensionCache
+}
+
+var _ ExtensionProcessor = &LoadExtensionCache{}
+
+func (x *LoadExtensionCache) ProcessExtensions(
+	ctx context.Context,
+	transformer *Transformer,
+	extensions []extension.Provider,
+	spans []*model.Span,
+	start, end time.Time,
+) ([]*model.Span, error) {
+	newSpans := []*model.Span{}
+
+	for _, cache := range x.Cache {
+		factory, hasFactory := transformer.ExtensionFactory.Indexed[cache.ProviderKind]
+		if !hasFactory {
+			return nil, fmt.Errorf("no factory for provider kind %q", cache.ProviderKind)
+		}
+
+		provider, err := factory.Configure(cache.ProviderConfig)
+		if err != nil {
+			return nil, fmt.Errorf("invalid provider factory: %w", err)
+		}
+
+		resultSpans, err := provider.LoadCache(ctx, []byte(cache.CachedIdentifier))
+		if err != nil {
+			return nil, fmt.Errorf("cannot load trace from cached identifier: %w", err)
+		}
+
+		if resultSpans != nil {
+			setTraceId(resultSpans, spans[0].TraceID)
+			setRootParent(resultSpans, cache.ParentTrace, cache.ParentSpan)
+
+			newSpans = append(newSpans, resultSpans...)
+		}
+	}
+
+	return newSpans, nil
+}
+
+func setTraceId(spans []*model.Span, traceId model.TraceID) {
+	for _, span := range spans {
+		span.TraceID = traceId
+
+		for i := range span.References {
+			span.References[i].TraceID = traceId
+		}
+	}
+}
+
+func setRootParent(spans []*model.Span, parentTrace model.TraceID, parentSpan model.SpanID) {
+	for _, span := range spans {
+		if len(span.References) == 0 {
+			span.References = []model.SpanRef{
+				model.NewChildOfRef(parentTrace, parentSpan),
+			}
+		}
+	}
+}
diff --git a/pkg/frontend/tf/transform.go b/pkg/frontend/tf/transform.go
index 6b04db66..8472a0f8 100644
--- a/pkg/frontend/tf/transform.go
+++ b/pkg/frontend/tf/transform.go
@@ -18,15 +18,18 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/jaegertracing/jaeger/model"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/util/sets"
 
+	"github.com/kubewharf/kelemetry/pkg/frontend/extension"
 	tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
 	tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
 	"github.com/kubewharf/kelemetry/pkg/manager"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func init() {
@@ -41,9 +44,10 @@ func (options *TransformerOptions) Setup(fs *pflag.FlagSet) {
 func (options *TransformerOptions) EnableFlag() *bool { return nil }
 
 type Transformer struct {
-	options TransformerOptions
-	Logger  logrus.FieldLogger
-	Configs tfconfig.Provider
+	options          TransformerOptions
+	Logger           logrus.FieldLogger
+	Configs          tfconfig.Provider
+	ExtensionFactory *manager.List[extension.ProviderFactory]
 }
 
 func (transformer *Transformer) Options() manager.Options        { return &transformer.options }
@@ -51,7 +55,14 @@ func (transformer *Transformer) Init() error                     { return nil }
 func (transformer *Transformer) Start(ctx context.Context) error { return nil }
 func (transformer *Transformer) Close(ctx context.Context) error { return nil }
 
-func (transformer *Transformer) Transform(trace *model.Trace, rootObject *tftree.GroupingKey, configId tfconfig.Id) error {
+func (transformer *Transformer) Transform(
+	ctx context.Context,
+	trace *model.Trace,
+	rootObject *utilobject.Key,
+	configId tfconfig.Id,
+	extensionProcessor ExtensionProcessor,
+	start, end time.Time,
+) error {
 	if len(trace.Spans) == 0 {
 		return fmt.Errorf("cannot transform empty trace")
 	}
@@ -61,7 +72,7 @@ func (transformer *Transformer) Transform(trace *model.Trace, rootObject *tftree
 		config = transformer.Configs.GetById(transformer.Configs.DefaultId())
 	}
 
-	tree := tftree.NewSpanTree(trace)
+	tree := tftree.NewSpanTree(trace.Spans)
 
 	transformer.groupDuplicates(tree)
 
@@ -70,7 +81,7 @@ func (transformer *Transformer) Transform(trace *model.Trace, rootObject *tftree
 		hasRootSpan := false
 
 		for _, span := range tree.GetSpans() {
-			if key, hasKey := tftree.GroupingKeyFromSpan(span); hasKey && key == *rootObject {
+			if key, hasKey := utilobject.FromSpan(span); hasKey && key == *rootObject {
 				rootSpan = span.SpanID
 				hasRootSpan = true
 			}
@@ -91,6 +102,14 @@ func (transformer *Transformer) Transform(trace *model.Trace, rootObject *tftree
 		}
 	}
 
+	newSpans, err := extensionProcessor.ProcessExtensions(ctx, transformer, config.Extensions, trace.Spans, start, end)
+	if err != nil {
+		return fmt.Errorf("cannot prepare extension trace: %w", err)
+	}
+
+	newSpans = append(newSpans, tree.GetSpans()...)
+	tree = tftree.NewSpanTree(newSpans)
+
 	for _, step := range config.Steps {
 		step.Run(tree)
 	}
@@ -102,10 +121,10 @@ func (transformer *Transformer) Transform(trace *model.Trace, rootObject *tftree
 
 // merge spans of the same object from multiple traces
 func (transformer *Transformer) groupDuplicates(tree *tftree.SpanTree) {
-	commonSpans := map[tftree.GroupingKey][]model.SpanID{}
+	commonSpans := map[utilobject.Key][]model.SpanID{}
 
 	for _, span := range tree.GetSpans() {
-		if key, hasKey := tftree.GroupingKeyFromSpan(span); hasKey {
+		if key, hasKey := utilobject.FromSpan(span); hasKey {
 			commonSpans[key] = append(commonSpans[key], span.SpanID)
 		}
 	}
diff --git a/pkg/frontend/tf/tree/grouping.go b/pkg/frontend/tf/tree/grouping.go
deleted file mode 100644
index 8536ed1d..00000000
--- a/pkg/frontend/tf/tree/grouping.go
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright 2023 The Kelemetry Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tftree
-
-import (
-	"github.com/jaegertracing/jaeger/model"
-	"k8s.io/apimachinery/pkg/util/sets"
-
-	"github.com/kubewharf/kelemetry/pkg/util/zconstants"
-)
-
-type GroupingKey struct {
-	Cluster   string `json:"cluster"`
-	Group     string `json:"group"`
-	Resource  string `json:"resource"`
-	Namespace string `json:"namespace"`
-	Name      string `json:"name"`
-	Field     string `json:"field"`
-}
-
-func GroupingKeyFromMap(tags map[string]string) (key GroupingKey, ok bool) {
-	for mapKey, field := range map[string]*string{
-		"cluster":   &key.Cluster,
-		"group":     &key.Group,
-		"resource":  &key.Resource,
-		"namespace": &key.Namespace,
-		"name":      &key.Name,
-	} {
-		*field, ok = tags[mapKey]
-		if !ok {
-			return key, false
-		}
-	}
-
-	if field, hasField := tags["field"]; hasField {
-		key.Field = field
-	} else {
-		key.Field = "object"
-	}
-
-	return key, true
-}
-
-func GroupingKeyFromSpan(span *model.Span) (GroupingKey, bool) {
-	tags := model.KeyValues(span.Tags)
-	traceSource, hasTraceSource := tags.FindByKey(zconstants.TraceSource)
-	if !hasTraceSource || traceSource.VStr != zconstants.TraceSourceObject {
-		return GroupingKey{}, false
-	}
-
-	cluster, _ := tags.FindByKey("cluster")
-	group, _ := tags.FindByKey("group")
-	resource, _ := tags.FindByKey("resource")
-	namespace, _ := tags.FindByKey("namespace")
-	name, _ := tags.FindByKey("name")
-	field, _ := tags.FindByKey(zconstants.NestLevel)
-	key := GroupingKey{
-		Cluster:   cluster.VStr,
-		Group:     group.VStr,
-		Resource:  resource.VStr,
-		Namespace: namespace.VStr,
-		Name:      name.VStr,
-		Field:     field.VStr,
-	}
-	return key, true
-}
-
-func GroupingKeysFromSpans(spans []*model.Span) sets.Set[GroupingKey] {
-	keys := sets.New[GroupingKey]()
-
-	for _, span := range spans {
-		if key, ok := GroupingKeyFromSpan(span); ok {
-			keys.Insert(key)
-		}
-	}
-	return keys
-}
diff --git a/pkg/frontend/tf/tree/tree.go b/pkg/frontend/tf/tree/tree.go
index 63d8a9cc..c8327f42 100644
--- a/pkg/frontend/tf/tree/tree.go
+++ b/pkg/frontend/tf/tree/tree.go
@@ -39,15 +39,15 @@ type stackEntry struct {
 	unprocessedChildren []model.SpanID
 }
 
-func NewSpanTree(trace *model.Trace) *SpanTree {
+func NewSpanTree(spans []*model.Span) *SpanTree {
 	tree := &SpanTree{
-		spanMap:      make(map[model.SpanID]*model.Span, len(trace.Spans)),
-		childrenMap:  make(map[model.SpanID]map[model.SpanID]struct{}, len(trace.Spans)),
+		spanMap:      make(map[model.SpanID]*model.Span, len(spans)),
+		childrenMap:  make(map[model.SpanID]map[model.SpanID]struct{}, len(spans)),
 		visitorStack: make(map[model.SpanID]*stackEntry),
-		exited:       make(map[model.SpanID]struct{}, len(trace.Spans)),
+		exited:       make(map[model.SpanID]struct{}, len(spans)),
 	}
 
-	for _, span := range trace.Spans {
+	for _, span := range spans {
 		tree.spanMap[span.SpanID] = span
 		if len(span.References) == 0 {
 			tree.Root = span
diff --git a/pkg/frontend/tracecache/interface.go b/pkg/frontend/tracecache/interface.go
index 74e4c970..6e061403 100644
--- a/pkg/frontend/tracecache/interface.go
+++ b/pkg/frontend/tracecache/interface.go
@@ -19,8 +19,10 @@ import (
 	"encoding/json"
 	"time"
 
-	tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
+	"github.com/jaegertracing/jaeger/model"
+
 	"github.com/kubewharf/kelemetry/pkg/manager"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func init() {
@@ -40,10 +42,20 @@ type Entry struct {
 }
 
 type EntryValue struct {
-	Identifiers []json.RawMessage   `json:"identifiers"`
-	StartTime   time.Time           `json:"startTime"`
-	EndTime     time.Time           `json:"endTime"`
-	RootObject  *tftree.GroupingKey `json:"rootObject"`
+	Identifiers []json.RawMessage `json:"identifiers"`
+	StartTime   time.Time         `json:"startTime"`
+	EndTime     time.Time         `json:"endTime"`
+	RootObject  *utilobject.Key   `json:"rootObject"`
+
+	Extensions []ExtensionCache `json:"extensions"`
+}
+
+type ExtensionCache struct {
+	ParentTrace      model.TraceID   `json:"parentTrace"`
+	ParentSpan       model.SpanID    `json:"parentSpan"`
+	ProviderKind     string          `json:"providerKind"`
+	ProviderConfig   json.RawMessage `json:"providerConfig"`
+	CachedIdentifier json.RawMessage `json:"cachedIdentifier"`
 }
 
 type mux struct {
diff --git a/pkg/imports.go b/pkg/imports.go
index 553723fd..b86cd1e8 100644
--- a/pkg/imports.go
+++ b/pkg/imports.go
@@ -41,6 +41,8 @@ import (
 	_ "github.com/kubewharf/kelemetry/pkg/frontend"
 	_ "github.com/kubewharf/kelemetry/pkg/frontend/backend/jaeger-storage"
 	_ "github.com/kubewharf/kelemetry/pkg/frontend/clusterlist/options"
+	_ "github.com/kubewharf/kelemetry/pkg/frontend/extension/httptrace"
+	_ "github.com/kubewharf/kelemetry/pkg/frontend/extension/jaeger-storage"
 	_ "github.com/kubewharf/kelemetry/pkg/frontend/http/redirect"
 	_ "github.com/kubewharf/kelemetry/pkg/frontend/http/trace"
 	_ "github.com/kubewharf/kelemetry/pkg/frontend/tf/config/file"
diff --git a/pkg/k8s/config/interface.go b/pkg/k8s/config/interface.go
index 9f2b0cb0..4b2816e5 100644
--- a/pkg/k8s/config/interface.go
+++ b/pkg/k8s/config/interface.go
@@ -21,6 +21,7 @@ import (
 	"k8s.io/client-go/rest"
 
 	"github.com/kubewharf/kelemetry/pkg/manager"
+	"github.com/kubewharf/kelemetry/pkg/metrics"
 )
 
 func init() {
@@ -43,6 +44,24 @@ type Config interface {
 type Cluster struct {
 	Config                *rest.Config
 	DefaultRequestTimeout time.Duration
+	UseOldResourceVersion bool
+}
+
+func (cluster *Cluster) ChooseResourceVersion(oldRv string, newRv *string) (string, error) {
+	useOld := false
+	if cluster != nil {
+		useOld = cluster.UseOldResourceVersion
+	}
+
+	if useOld {
+		return oldRv, nil
+	}
+
+	if newRv != nil {
+		return *newRv, nil
+	}
+
+	return "", metrics.MakeLabeledError("NoNewRv")
 }
 
 type mux struct {
diff --git a/pkg/k8s/config/mapoption/mapoption.go b/pkg/k8s/config/mapoption/mapoption.go
index 3b9d66cb..d940e0e3 100644
--- a/pkg/k8s/config/mapoption/mapoption.go
+++ b/pkg/k8s/config/mapoption/mapoption.go
@@ -38,6 +38,7 @@ type options struct {
 	master            map[string]string
 	kubeconfig        map[string]string
 	requestTimeout    map[string]string
+	useOldRvClusters  []string
 }
 
 func (options *options) Setup(fs *pflag.FlagSet) {
@@ -55,6 +56,12 @@ func (options *options) Setup(fs *pflag.FlagSet) {
 		map[string]string{},
 		"map of server-side request timeout, used in metrics",
 	)
+	fs.StringSliceVar(
+		&options.useOldRvClusters,
+		"kube-use-old-resource-version-clusters",
+		[]string{},
+		"list of clusters that do not have new resource version in audit",
+	)
 }
 
 func (options *options) EnableFlag() *bool { return nil }
@@ -98,10 +105,21 @@ func (provider *Provider) Init() error {
 			}
 		}
 
-		provider.configs[name] = &k8sconfig.Cluster{
+		useOldRv := false
+		for _, oldRvCluster := range provider.options.useOldRvClusters {
+			if oldRvCluster == name {
+				useOldRv = true
+				break
+			}
+		}
+
+		clusterConfig := &k8sconfig.Cluster{
 			Config:                config,
 			DefaultRequestTimeout: defaultRequestTimeout,
+			UseOldResourceVersion: useOldRv,
 		}
+
+		provider.configs[name] = clusterConfig
 	}
 
 	_, hasTarget := provider.configs[provider.options.targetClusterName]
diff --git a/pkg/k8s/objectcache/objectcache.go b/pkg/k8s/objectcache/objectcache.go
index 451f851c..1036242a 100644
--- a/pkg/k8s/objectcache/objectcache.go
+++ b/pkg/k8s/objectcache/objectcache.go
@@ -32,7 +32,7 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/k8s"
 	"github.com/kubewharf/kelemetry/pkg/manager"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func init() {
@@ -95,11 +95,11 @@ func (oc *ObjectCache) Start(ctx context.Context) error { return nil }
 
 func (oc *ObjectCache) Close(ctx context.Context) error { return nil }
 
-func (oc *ObjectCache) Get(ctx context.Context, object util.ObjectRef) (*unstructured.Unstructured, error) {
+func (oc *ObjectCache) Get(ctx context.Context, object utilobject.VersionedKey) (*unstructured.Unstructured, error) {
 	metric := &CacheRequestMetric{Cluster: object.Cluster, Error: "Unknown"}
 	defer oc.CacheRequestMetric.DeferCount(oc.Clock.Now(), metric)
 
-	key := objectKey(object)
+	key := objectKey(object.Key)
 
 	for {
 		fetchCtx, cancelFunc := context.WithTimeout(ctx, oc.options.fetchTimeout)
@@ -173,7 +173,7 @@ func decodeCached(cached []byte, metric *CacheRequestMetric) (*unstructured.Unst
 
 func (oc *ObjectCache) penetrate(
 	ctx context.Context,
-	object util.ObjectRef,
+	object utilobject.VersionedKey,
 ) (_raw *unstructured.Unstructured, _err error, _metricCode string) {
 	// first, try fetching from server
 	clusterClient, err := oc.Clients.Cluster(object.Cluster)
@@ -181,7 +181,7 @@ func (oc *ObjectCache) penetrate(
 		return nil, fmt.Errorf("cannot initialize clients for cluster %q: %w", object.Cluster, err), "UnknownCluster"
 	}
 
-	nsClient := clusterClient.DynamicClient().Resource(object.GroupVersionResource)
+	nsClient := clusterClient.DynamicClient().Resource(object.GroupVersionResource())
 	var client dynamic.ResourceInterface = nsClient
 	if object.Namespace != "" {
 		client = nsClient.Namespace(object.Namespace)
@@ -199,7 +199,7 @@ func (oc *ObjectCache) penetrate(
 	}
 
 	// not found from apiserver, try deletion snapshot instead
-	snapshot, err := oc.DiffCache.FetchSnapshot(ctx, object, diffcache.SnapshotNameDeletion)
+	snapshot, err := oc.DiffCache.FetchSnapshot(ctx, object.Key, diffcache.SnapshotNameDeletion)
 	if err != nil {
 		return nil, metrics.LabelError(fmt.Errorf("cannot fallback to snapshot: %w", err), "SnapshotFetch"), "SnapshotFetch"
 	}
@@ -218,6 +218,6 @@ func (oc *ObjectCache) penetrate(
 	return nil, nil, "NotFoundAnywhere"
 }
 
-func objectKey(object util.ObjectRef) []byte {
+func objectKey(object utilobject.Key) []byte {
 	return []byte(fmt.Sprintf("%s/%s/%s/%s", object.Group, object.Resource, object.Namespace, object.Name))
 }
diff --git a/pkg/k8s/objectcache/objectcache_test.go b/pkg/k8s/objectcache/objectcache_test.go
index 01fa7a76..5c2c9088 100644
--- a/pkg/k8s/objectcache/objectcache_test.go
+++ b/pkg/k8s/objectcache/objectcache_test.go
@@ -29,7 +29,7 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/k8s"
 	"github.com/kubewharf/kelemetry/pkg/k8s/objectcache"
 	"github.com/kubewharf/kelemetry/pkg/metrics"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func TestGet(t *testing.T) {
@@ -70,11 +70,15 @@ func TestGet(t *testing.T) {
 	assert.NoError(cache.Init())
 
 	for i := 0; i < 2; i++ {
-		uns, err := cache.Get(context.Background(), util.ObjectRef{
-			Cluster:              "test-cluster",
-			GroupVersionResource: corev1.SchemeGroupVersion.WithResource("configmaps"),
-			Namespace:            "default",
-			Name:                 "test-cm",
+		uns, err := cache.Get(context.Background(), utilobject.VersionedKey{
+			Key: utilobject.Key{
+				Cluster:   "test-cluster",
+				Namespace: "default",
+				Name:      "test-cm",
+				Group:     corev1.GroupName,
+				Resource:  "configmaps",
+			},
+			Version: corev1.SchemeGroupVersion.Version,
 		})
 		assert.NoError(err)
 
diff --git a/pkg/manager/interface.go b/pkg/manager/interface.go
index f3d8d8bf..8da92740 100644
--- a/pkg/manager/interface.go
+++ b/pkg/manager/interface.go
@@ -389,7 +389,12 @@ func (manager *Manager) initListComponent(list ListInterface) {
 
 type List[T any] struct {
 	BaseComponent
-	Impls []T
+	Impls   []T
+	Indexed map[string]T
+}
+
+type IndexedListImpl interface {
+	ListIndex() string
 }
 
 type ListInterface interface {
@@ -408,10 +413,16 @@ func (list *List[T]) listRemoveImpl(obj Component) {
 	newList := []T{}
 	for _, impl := range list.Impls {
 		if any(impl) == obj {
+			if list.Indexed != nil {
+				delete(list.Indexed, any(impl).(IndexedListImpl).ListIndex())
+			}
+
 			continue
 		}
+
 		newList = append(newList, impl)
 	}
+
 	list.Impls = newList
 }
 
@@ -425,8 +436,17 @@ func (f *listComponentFactory[T]) Call(args []reflect.Value) []reflect.Value {
 		impls = append(impls, arg.Interface().(T))
 	}
 
+	var indexed map[string]T
+	if reflectutil.TypeOf[T]().Implements(reflectutil.TypeOf[IndexedListImpl]()) {
+		indexed = make(map[string]T, len(impls))
+		for _, impl := range impls {
+			indexed[any(impl).(IndexedListImpl).ListIndex()] = impl
+		}
+	}
+
 	return []reflect.Value{reflect.ValueOf(&List[T]{
-		Impls: impls,
+		Impls:   impls,
+		Indexed: indexed,
 	})}
 }
 
diff --git a/pkg/ownerlinker/linker.go b/pkg/ownerlinker/linker.go
index 56ae6dfb..eaf0bc7b 100644
--- a/pkg/ownerlinker/linker.go
+++ b/pkg/ownerlinker/linker.go
@@ -16,11 +16,9 @@ package ownerlinker
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
 	"github.com/kubewharf/kelemetry/pkg/aggregator/linker"
@@ -28,7 +26,7 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/k8s/discovery"
 	"github.com/kubewharf/kelemetry/pkg/k8s/objectcache"
 	"github.com/kubewharf/kelemetry/pkg/manager"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func init() {
@@ -60,60 +58,67 @@ func (ctrl *Controller) Init() error                     { return nil }
 func (ctrl *Controller) Start(ctx context.Context) error { return nil }
 func (ctrl *Controller) Close(ctx context.Context) error { return nil }
 
-func (ctrl *Controller) Lookup(ctx context.Context, object util.ObjectRef) (*util.ObjectRef, error) {
+func (ctrl *Controller) Lookup(ctx context.Context, object utilobject.Rich) *utilobject.Rich {
+	raw := object.Raw
+
 	logger := ctrl.Logger.WithFields(object.AsFields("object"))
 
-	raw := object.Raw
 	if raw == nil {
 		logger.Debug("Fetching dynamic object")
 
 		var err error
-		raw, err = ctrl.ObjectCache.Get(ctx, object)
+		raw, err = ctrl.ObjectCache.Get(ctx, object.VersionedKey)
+
 		if err != nil {
-			return nil, fmt.Errorf("cannot fetch object value: %w", err)
+			logger.WithError(err).Error("cannot fetch object value")
+			return nil
 		}
+
 		if raw == nil {
-			return nil, fmt.Errorf("object does not exist")
+			logger.Debug("object no longer exists")
+			return nil
 		}
 	}
 
-	var owner metav1.OwnerReference
-	hasOwner := false
-	for _, testOwner := range raw.GetOwnerReferences() {
-		if testOwner.Controller != nil && *testOwner.Controller {
-			owner, hasOwner = testOwner, true
-			break
+	for _, owner := range raw.GetOwnerReferences() {
+		if owner.Controller != nil && *owner.Controller {
+			groupVersion, err := schema.ParseGroupVersion(owner.APIVersion)
+			if err != nil {
+				logger.WithError(err).Warn("invalid owner apiVersion")
+				continue
+			}
+
+			gvk := groupVersion.WithKind(owner.Kind)
+			cdc, err := ctrl.DiscoveryCache.ForCluster(object.Cluster)
+			if err != nil {
+				logger.WithError(err).Error("cannot access cluster from object reference")
+				continue
+			}
+
+			gvr, exists := cdc.LookupResource(gvk)
+			if !exists {
+				logger.WithField("gvk", gvk).Warn("Object contains owner reference of unknown GVK")
+				continue
+			}
+
+			ret := &utilobject.Rich{
+				VersionedKey: utilobject.VersionedKey{
+					Key: utilobject.Key{
+						Cluster:   object.Cluster,
+						Group:     gvr.Group,
+						Resource:  gvr.Group,
+						Namespace: object.Namespace,
+						Name:      owner.Name,
+					},
+					Version: gvr.Version,
+				},
+				Uid: owner.UID,
+			}
+			logger.WithField("owner", ret).Debug("Resolved owner")
+
+			return ret
 		}
 	}
 
-	if !hasOwner {
-		return nil, nil
-	}
-
-	groupVersion, err := schema.ParseGroupVersion(owner.APIVersion)
-	if err != nil {
-		return nil, fmt.Errorf("invalid owner apiVersion: %w", err)
-	}
-
-	gvk := groupVersion.WithKind(owner.Kind)
-	cdc, err := ctrl.DiscoveryCache.ForCluster(object.Cluster)
-	if err != nil {
-		return nil, fmt.Errorf("cannot access cluster from object reference: %w", err)
-	}
-
-	gvr, exists := cdc.LookupResource(gvk)
-	if !exists {
-		return nil, fmt.Errorf("object contains owner reference of unknown gvk %v", gvk)
-	}
-
-	ret := &util.ObjectRef{
-		Cluster:              object.Cluster, // inherited from the same cluster
-		GroupVersionResource: gvr,
-		Namespace:            object.Namespace,
-		Name:                 owner.Name,
-		Uid:                  owner.UID,
-	}
-	logger.WithField("owner", ret).Debug("Resolved owner")
-
-	return ret, nil
+	return nil
 }
diff --git a/pkg/rulelinker/linker.go b/pkg/rulelinker/linker.go
index 848252f7..b30a519e 100644
--- a/pkg/rulelinker/linker.go
+++ b/pkg/rulelinker/linker.go
@@ -28,7 +28,7 @@ import (
 	"github.com/kubewharf/kelemetry/pkg/k8s/discovery"
 	"github.com/kubewharf/kelemetry/pkg/k8s/objectcache"
 	"github.com/kubewharf/kelemetry/pkg/manager"
-	"github.com/kubewharf/kelemetry/pkg/util"
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
 )
 
 func init() {
@@ -62,15 +62,25 @@ func (ctrl *Controller) Init() error                     { return nil }
 func (ctrl *Controller) Start(ctx context.Context) error { return nil }
 func (ctrl *Controller) Close(ctx context.Context) error { return nil }
 
-func (ctrl *Controller) Lookup(ctx context.Context, object util.ObjectRef) (*util.ObjectRef, error) {
+func (ctrl *Controller) Lookup(ctx context.Context, object utilobject.Rich) *utilobject.Rich {
 	logger := ctrl.Logger.WithFields(object.AsFields("object"))
 
+	parent, err := ctrl.lookup(ctx, logger, object)
+	if err != nil {
+		logger.Error(err)
+		return nil
+	}
+
+	return parent
+}
+
+func (ctrl *Controller) lookup(ctx context.Context, logger logrus.FieldLogger, object utilobject.Rich) (*utilobject.Rich, error) {
 	childRaw := object.Raw
 	if childRaw == nil {
 		logger.Debug("Fetching dynamic object")
 
 		var err error
-		childRaw, err = ctrl.ObjectCache.Get(ctx, object)
+		childRaw, err = ctrl.ObjectCache.Get(ctx, object.VersionedKey)
 		if err != nil {
 			return nil, fmt.Errorf("cannot fetch object value: %w", err)
 		}
@@ -86,7 +96,7 @@ func (ctrl *Controller) Lookup(ctx context.Context, object util.ObjectRef) (*uti
 
 	var matchedRule *kelemetryv1a1.LinkRule
 	for _, rule := range store.List() {
-		matched, err := kelemetryv1a1util.TestObject(object, childRaw, &rule.Filter)
+		matched, err := kelemetryv1a1util.TestObject(object.Key, childRaw, &rule.Filter)
 		if err != nil {
 			logger.WithField("rule", rule.Name).WithError(err).Error("error testing object for rule match")
 		} else if matched {
@@ -110,8 +120,9 @@ func (ctrl *Controller) Lookup(ctx context.Context, object util.ObjectRef) (*uti
 		return nil, fmt.Errorf("cannot get parent specified by rule: %w", err)
 	}
 
-	parentRef.Uid = parentRaw.GetUID()
-	parentRef.Raw = parentRaw
-
-	return &parentRef, nil
+	return &utilobject.Rich{
+		VersionedKey: parentRef,
+		Uid: parentRaw.GetUID(),
+		Raw: parentRaw,
+	}, nil
 }
diff --git a/pkg/util/filter/object_filter.go b/pkg/util/filter/object_filter.go
new file mode 100644
index 00000000..576c6711
--- /dev/null
+++ b/pkg/util/filter/object_filter.go
@@ -0,0 +1,86 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filter
+
+import (
+	"github.com/dlclark/regexp2"
+
+	utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+)
+
+type ObjectFilters struct {
+	Cluster   Regex `json:"cluster"`
+	Group     Regex `json:"group"`
+	Version   Regex `json:"version"`
+	Resource  Regex `json:"resource"`
+	Namespace Regex `json:"namespace"`
+	Name      Regex `json:"name"`
+}
+
+type Regex struct {
+	// golang regex not support (?!..), use third party regular engine for go
+	Pattern *regexp2.Regexp
+}
+
+func (regex *Regex) UnmarshalText(text []byte) (err error) {
+	regex.Pattern, err = regexp2.Compile(string(text), 0)
+	return err
+}
+
+func (regex *Regex) MatchString(s string) bool {
+	match, _ := regex.Pattern.MatchString(s)
+	return match
+}
+
+func (f *ObjectFilters) Check(object utilobject.Rich) bool {
+	if f.Cluster.Pattern != nil && !f.Cluster.MatchString(object.Cluster) {
+		return false
+	}
+
+	if f.Group.Pattern != nil && !f.Group.MatchString(object.Group) {
+		return false
+	}
+
+	if f.Version.Pattern != nil && !f.Version.MatchString(object.Version) {
+		return false
+	}
+
+	if f.Resource.Pattern != nil && !f.Resource.MatchString(object.Resource) {
+		return false
+	}
+
+	if f.Namespace.Pattern != nil && !f.Namespace.MatchString(object.Namespace) {
+		return false
+	}
+
+	if f.Name.Pattern != nil && !f.Name.MatchString(object.Name) {
+		return false
+	}
+
+	return true
+}
+
+type TagFilters map[string]Regex
+
+func (f TagFilters) Check(tags map[string]string) bool {
+	for key, filter := range f {
+		val := tags[key]
+		if filter.Pattern != nil && !filter.MatchString(val) {
+			return false
+		}
+	}
+
+	return true
+}
diff --git a/pkg/util/jaeger/constants.go b/pkg/util/jaeger/constants.go
new file mode 100644
index 00000000..019b3499
--- /dev/null
+++ b/pkg/util/jaeger/constants.go
@@ -0,0 +1,23 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utiljaeger
+
+var SpanStorageTypesToAddFlag = []string{
+	"cassandra",
+	"elasticsearch",
+	"kafka",
+	"grpc-plugin",
+	"badger",
+}
diff --git a/pkg/util/object/key.go b/pkg/util/object/key.go
new file mode 100644
index 00000000..1208c076
--- /dev/null
+++ b/pkg/util/object/key.go
@@ -0,0 +1,147 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utilobject
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+
+	"github.com/kubewharf/kelemetry/pkg/util/zconstants"
+)
+
+type Key struct {
+	Cluster   string `json:"cluster"`
+	Group     string `json:"group"`
+	Resource  string `json:"resource"`
+	Namespace string `json:"namespace"`
+	Name      string `json:"name"`
+}
+
+func (key Key) Clone() Key {
+	return Key{
+		Cluster:   strings.Clone(key.Cluster),
+		Group:     strings.Clone(key.Group),
+		Resource:  strings.Clone(key.Resource),
+		Namespace: strings.Clone(key.Namespace),
+		Name:      strings.Clone(key.Name),
+	}
+}
+
+func (key Key) GroupResource() schema.GroupResource {
+	return schema.GroupResource{Group: key.Group, Resource: key.Resource}
+}
+
+func (key Key) String() string {
+	return fmt.Sprintf("%s/%s/%s/%s/%s", key.Cluster, key.Group, key.Resource, key.Namespace, key.Name)
+}
+
+func (key Key) AsFields(prefix string) logrus.Fields {
+	return logrus.Fields{
+		prefix + "Cluster":   key.Cluster,
+		prefix + "Group":     key.Group,
+		prefix + "Resource":  key.Resource,
+		prefix + "Namespace": key.Namespace,
+		prefix + "Name":      key.Name,
+	}
+}
+
+func FromMap(tags map[string]string) (key Key, ok bool) {
+	for mapKey, field := range map[string]*string{
+		"cluster":   &key.Cluster,
+		"group":     &key.Group,
+		"resource":  &key.Resource,
+		"namespace": &key.Namespace,
+		"name":      &key.Name,
+	} {
+		*field, ok = tags[mapKey]
+		if !ok {
+			return key, false
+		}
+	}
+
+	return key, true
+}
+
+func FromSpan(span *model.Span) (Key, bool) {
+	tags := model.KeyValues(span.Tags)
+	traceSource, hasTraceSource := tags.FindByKey(zconstants.TraceSource)
+	if !hasTraceSource || traceSource.VStr != zconstants.TraceSourceObject {
+		return Key{}, false
+	}
+
+	cluster, _ := tags.FindByKey("cluster")
+	group, _ := tags.FindByKey("group")
+	resource, _ := tags.FindByKey("resource")
+	namespace, _ := tags.FindByKey("namespace")
+	name, _ := tags.FindByKey("name")
+	key := Key{
+		Cluster:   cluster.VStr,
+		Group:     group.VStr,
+		Resource:  resource.VStr,
+		Namespace: namespace.VStr,
+		Name:      name.VStr,
+	}
+	return key, true
+}
+
+func FromSpans(spans []*model.Span) sets.Set[Key] {
+	keys := sets.New[Key]()
+
+	for _, span := range spans {
+		if key, ok := FromSpan(span); ok {
+			keys.Insert(key)
+		}
+	}
+	return keys
+}
+
+type VersionedKey struct {
+	Key
+	Version string `json:"version"`
+}
+
+func (key VersionedKey) Clone() VersionedKey {
+	return VersionedKey{
+		Key:     key.Key.Clone(),
+		Version: strings.Clone(key.Version),
+	}
+}
+
+func (key VersionedKey) GroupVersionResource() schema.GroupVersionResource {
+	return key.GroupResource().WithVersion(key.Version)
+}
+
+func (key VersionedKey) GroupVersion() schema.GroupVersion {
+	return schema.GroupVersion{Group: key.Group, Version: key.Version}
+}
+
+func FromObject(object metav1.Object, cluster string, gvr schema.GroupVersionResource) VersionedKey {
+	return VersionedKey{
+		Key: Key{
+			Cluster:   cluster,
+			Group:     gvr.Group,
+			Resource:  gvr.Resource,
+			Namespace: object.GetNamespace(),
+			Name:      object.GetName(),
+		},
+		Version: gvr.Version,
+	}
+}
diff --git a/pkg/util/object/rich.go b/pkg/util/object/rich.go
new file mode 100644
index 00000000..fd30b84f
--- /dev/null
+++ b/pkg/util/object/rich.go
@@ -0,0 +1,82 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utilobject
+
+import (
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
+)
+
+type Rich struct {
+	VersionedKey
+
+	Uid types.UID
+
+	Raw *unstructured.Unstructured `json:"-"`
+}
+
+// Returns a new Rich reference with all strings cloned again
+// to ensure this object does not reference more data than it needs.
+//
+// Does not copy the Raw field.
+func (ref Rich) Clone() Rich {
+	return Rich{
+		VersionedKey: ref.VersionedKey.Clone(),
+		Uid:          types.UID(strings.Clone(string(ref.Uid))),
+	}
+}
+
+func (ref Rich) String() string {
+	return ref.Key.String()
+}
+
+func (ref Rich) AsFields(prefix string) logrus.Fields {
+	fields := ref.Key.AsFields(prefix)
+	fields[prefix+"Uid"] = ref.Uid
+	return fields
+}
+
+func RichFromUnstructured(
+	uns *unstructured.Unstructured,
+	cluster string,
+	gvr schema.GroupVersionResource,
+) Rich {
+	return Rich{
+		VersionedKey: FromObject(uns, cluster, gvr),
+		Uid:          uns.GetUID(),
+		Raw:          uns,
+	}
+}
+
+func RichFromAudit(object *auditv1.ObjectReference, cluster string) Rich {
+	return Rich{
+		VersionedKey: VersionedKey{
+			Key: Key{
+				Cluster:   cluster,
+				Group:     object.APIGroup,
+				Resource:  object.Resource,
+				Namespace: object.Namespace,
+				Name:      object.Name,
+			},
+			Version: object.APIVersion,
+		},
+		Uid: object.UID,
+	}
+}
diff --git a/pkg/util/object_ref.go b/pkg/util/object_ref.go
deleted file mode 100644
index bab56650..00000000
--- a/pkg/util/object_ref.go
+++ /dev/null
@@ -1,103 +0,0 @@
-// Copyright 2023 The Kelemetry Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package util
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/sirupsen/logrus"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/types"
-	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
-)
-
-type ObjectRef struct {
-	Cluster string
-
-	schema.GroupVersionResource
-
-	Namespace string
-	Name      string
-
-	// .metadata.uid of the object. May be absent or not required in some cases.
-	Uid types.UID
-
-	// A cached copy of the object, if any.
-	Raw *unstructured.Unstructured `json:"-"`
-}
-
-// Returns a new ObjectRef with all strings cloned again
-// to ensure this object does not reference more data than it needs.
-//
-// Does not copy the Raw field.
-func (ref ObjectRef) Clone() ObjectRef {
-	return ObjectRef{
-		Cluster: strings.Clone(ref.Cluster),
-		GroupVersionResource: schema.GroupVersionResource{
-			Group:    strings.Clone(ref.Group),
-			Version:  strings.Clone(ref.Version),
-			Resource: strings.Clone(ref.Resource),
-		},
-		Namespace: strings.Clone(ref.Namespace),
-		Name:      strings.Clone(ref.Name),
-		Uid:       types.UID(strings.Clone(string(ref.Uid))),
-	}
-}
-
-func (ref ObjectRef) String() string {
-	return fmt.Sprintf("%s/%s/%s/%s/%s", ref.Cluster, ref.Group, ref.Resource, ref.Namespace, ref.Name)
-}
-
-func (ref ObjectRef) AsFields(prefix string) logrus.Fields {
-	return logrus.Fields{
-		prefix + "Cluster":   ref.Cluster,
-		prefix + "Group":     ref.Group,
-		prefix + "Resource":  ref.Resource,
-		prefix + "Namespace": ref.Namespace,
-		prefix + "Name":      ref.Name,
-		prefix + "Uid":       ref.Uid,
-	}
-}
-
-func ObjectRefFromUnstructured(
-	uns *unstructured.Unstructured,
-	cluster string,
-	gvr schema.GroupVersionResource,
-) ObjectRef {
-	return ObjectRef{
-		Cluster:              cluster,
-		GroupVersionResource: gvr,
-		Namespace:            uns.GetNamespace(),
-		Name:                 uns.GetName(),
-		Uid:                  uns.GetUID(),
-		Raw:                  uns,
-	}
-}
-
-func ObjectRefFromAudit(object *auditv1.ObjectReference, cluster string, uid types.UID) ObjectRef {
-	return ObjectRef{
-		Cluster: cluster,
-		GroupVersionResource: schema.GroupVersionResource{
-			Group:    object.APIGroup,
-			Version:  object.APIVersion,
-			Resource: object.Resource,
-		},
-		Namespace: object.Namespace,
-		Name:      object.Name,
-		Uid:       uid,
-	}
-}
diff --git a/pkg/util/semaphore/semaphore.go b/pkg/util/semaphore/semaphore.go
new file mode 100644
index 00000000..c04070c5
--- /dev/null
+++ b/pkg/util/semaphore/semaphore.go
@@ -0,0 +1,157 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Provides a short-lived task pool that runs fallible tasks and publishes results on the same goroutine.
+// Similar to golang.org/x/sync/semaphore, but this supports scheduling tasks recursively (schedule while Wait is being called).
+package semaphore
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+)
+
+// Publish is a function that runs on the goroutine that calls `Run`.
+type (
+	Publish = func() error
+	Task    = func(ctx context.Context) (Publish, error)
+)
+
+type Semaphore struct {
+	// Channel for limiting the number of concurrent tasks.
+	// Tasks can only be run when the channel is not saturated.
+	// Send before run, receive after run.
+	//
+	// limitCh can be nil if the semaphore is unbounded (limit == -1).
+	limitCh chan struct{}
+
+	// Done by worker goroutines to indicate that a scheduled task is complete.
+	doneWg sync.WaitGroup
+
+	// Sent from worker goroutines, received by main goroutine to publish the result.
+	publishCh chan Publish
+
+	// Sent from worker goroutines, received by main goroutine to return the error to caller.
+	errCh chan error
+
+	// Oneshot channel closed by the main goroutine to indicate to workers that the semaphore has been invalidated.
+	errNotifyCh chan struct{}
+
+	// Oneshot channel closed by the main goroutine to indicate that ctx is ready.
+	initCtxCh chan struct{}
+	//nolint:containedctx // this context is used for async passing
+	ctx context.Context
+}
+
+func NewUnbounded() *Semaphore { return New(-1) }
+
+func New(limit int) *Semaphore {
+	var limitCh chan struct{}
+	if limit >= 0 {
+		limitCh = make(chan struct{}, limit)
+	}
+
+	sem := &Semaphore{
+		limitCh:     limitCh,
+		errNotifyCh: make(chan struct{}),
+		publishCh:   make(chan Publish),
+		errCh:       make(chan error, 1), // only first error needs to get sent, always TrySend here
+		initCtxCh:   make(chan struct{}),
+	}
+	return sem
+}
+
+func (sem *Semaphore) Schedule(task Task) {
+	sem.doneWg.Add(1)
+	go func() {
+		defer sem.doneWg.Done()
+
+		<-sem.initCtxCh
+		ctx := sem.ctx
+
+		if sem.limitCh != nil {
+			// wait for semaphore acquisition
+
+			select {
+			case <-sem.errNotifyCh:
+				return
+			case sem.limitCh <- struct{}{}:
+			}
+
+			defer func() { <-sem.limitCh }()
+		}
+
+		select {
+		case <-sem.errNotifyCh:
+			// The previous select with limitCh has random, check again to reduce unnecessary runs
+			return
+		default:
+		}
+
+		publish, err := task(ctx)
+		if err != nil {
+			select {
+			case sem.errCh <- err:
+			default:
+				// no need to publish if there is another error in the channel
+			}
+		} else {
+			if publish != nil {
+				select {
+				case sem.publishCh <- publish:
+				case <-sem.errNotifyCh:
+					// no need to publish if the caller received error
+				}
+			}
+		}
+	}()
+}
+
+func (sem *Semaphore) Run(ctx context.Context) error {
+	ctx, cancelFunc := context.WithCancel(ctx)
+	defer cancelFunc()
+
+	sem.ctx = ctx
+	close(sem.initCtxCh)
+
+	doneCh := make(chan struct{})
+
+	go func() {
+		sem.doneWg.Wait()
+		close(doneCh)
+	}()
+
+	var stopped atomic.Bool
+	defer stopped.Store(true)
+
+	for {
+		select {
+		case <-doneCh:
+			// publishCh must be fully sent before calling sem.doneWg.Done()
+			return nil
+		case publish := <-sem.publishCh:
+			err := publish()
+			if err != nil {
+				close(sem.errNotifyCh)
+				return err
+			}
+		case err := <-sem.errCh:
+			close(sem.errNotifyCh)
+			return err
+		case <-ctx.Done():
+			close(sem.errNotifyCh)
+			return ctx.Err()
+		}
+	}
+}
diff --git a/pkg/util/semaphore/semaphore_test.go b/pkg/util/semaphore/semaphore_test.go
new file mode 100644
index 00000000..72b870ee
--- /dev/null
+++ b/pkg/util/semaphore/semaphore_test.go
@@ -0,0 +1,182 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package semaphore_test
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/kubewharf/kelemetry/pkg/util/semaphore"
+)
+
+var ErrTest = errors.New("test error")
+
+func TestLimit(t *testing.T) {
+	assert := assert.New(t)
+	sem := semaphore.New(2)
+
+	run := make(chan struct{}, 16)
+	var counter atomic.Int32
+	stopCh := make(chan struct{})
+
+	var completions int
+
+	for i := 0; i < 3; i++ {
+		sem.Schedule(func(ctx context.Context) (semaphore.Publish, error) {
+			run <- struct{}{}
+			currentCount := counter.Add(1)
+			assert.LessOrEqual(currentCount, int32(2))
+
+			<-stopCh
+
+			return func() error {
+				completions += 1
+				return nil
+			}, nil
+		})
+	}
+
+	runOkCh := make(chan struct{})
+
+	go func() {
+		err := sem.Run(context.Background())
+		assert.NoError(err)
+		close(runOkCh)
+	}()
+
+	received := 0
+	for i := 0; i < 2; i++ {
+		<-run
+		received += 1
+	}
+
+	time.Sleep(time.Millisecond)
+	assert.Empty(run)
+
+	counter.Store(0) // reset it to 0 so that third goroutine does not crash
+	close(stopCh)
+
+	<-runOkCh
+	assert.Equal(3, completions)
+}
+
+func TestEmpty(t *testing.T) {
+	assert := assert.New(t)
+	sem := semaphore.New(2)
+	err := sem.Run(context.Background())
+	assert.NoError(err)
+}
+
+func TestErrorPreempt(t *testing.T) {
+	assert := assert.New(t)
+	sem := semaphore.New(4)
+
+	othersStarted := make(chan struct{}, 16)
+
+	scheduleOther := func() {
+		sem.Schedule(func(ctx context.Context) (semaphore.Publish, error) {
+			othersStarted <- struct{}{}
+			_, open := <-ctx.Done()
+			assert.False(open, "context should be canceled")
+			return nil, errors.New("other errors")
+		})
+	}
+
+	sem.Schedule(func(ctx context.Context) (semaphore.Publish, error) {
+		for i := 0; i < 3; i++ {
+			<-othersStarted
+		}
+		for i := 0; i < 2; i++ {
+			// schedule new tasks after error source starts
+			// to avoid all waiting tasks getting created first
+			scheduleOther()
+		}
+		return nil, ErrTest
+	})
+
+	for i := 0; i < 3; i++ {
+		scheduleOther()
+	}
+
+	err := sem.Run(context.Background())
+	assert.ErrorIs(err, ErrTest)
+}
+
+func TestParentContextCancel(t *testing.T) {
+	assert := assert.New(t)
+	sem := semaphore.New(2)
+
+	started := make(chan struct{}, 16)
+	var completed atomic.Int32
+
+	for i := 0; i < 3; i++ {
+		sem.Schedule(func(ctx context.Context) (semaphore.Publish, error) {
+			started <- struct{}{}
+			_, open := <-ctx.Done()
+			time.Sleep(time.Millisecond)
+			assert.False(open, "context should be canceled")
+			completed.Add(1)
+			return nil, ErrTest
+		})
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		for i := 0; i < 2; i++ {
+			<-started
+		}
+
+		cancel()
+	}()
+
+	err := sem.Run(ctx)
+	assert.LessOrEqual(completed.Load(), int32(2), "should not execute new tasks after cancelation")
+	assert.True(errors.Is(err, ErrTest) || errors.Is(err, ctx.Err()), "error is either ContextCanceled or TestErr")
+}
+
+func TestRecursiveSchedule(t *testing.T) {
+	assert := assert.New(t)
+	sem := semaphore.New(3)
+
+	completions := 0
+
+	var f func(depth int) semaphore.Publish
+	f = func(depth int) semaphore.Publish {
+		if depth < 4 {
+			for i := 0; i < 2; i++ {
+				sem.Schedule(func(ctx context.Context) (semaphore.Publish, error) {
+					return f(depth + 1), nil
+				})
+			}
+		}
+		return func() error {
+			completions += 1
+			return nil
+		}
+	}
+
+	sem.Schedule(func(ctx context.Context) (semaphore.Publish, error) {
+		return f(0), nil
+	})
+
+	err := sem.Run(context.Background())
+	assert.NoError(err)
+	assert.Equal(31, completions)
+}