-
Notifications
You must be signed in to change notification settings - Fork 0
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add Prometheus metrics for S3 and IAM requests #71
Closed
anurag4DSB
wants to merge
12
commits into
feature/COSI-65-add-metrics-scrapable-by-prometheus
from
feature/COSI-19-add-s3-and-iam-metrics
Closed
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit
Hold shift + click to select a range
4d96040
COSI-75: e2e-test-for-brown-field-scenario
anurag4DSB 08e8c70
COSI-75: re-organize-greenfield-custom-resources
anurag4DSB 1990d8a
Add metrics package for Prometheus instrumentation
anurag4DSB 9f352dc
Integrate Prometheus metrics server in COSI driver
anurag4DSB 0ab5bd9
Instrumented gRPC server with Prometheus exporter
anurag4DSB a162a1a
Metrics package unit tests
anurag4DSB aa85ea0
Add metrics package to Codecov configuration
anurag4DSB 3433918
Add Prometheus metrics for S3 and IAM requests
anurag4DSB b046d9e
Add Prometheus metrics instrumentation for S3 ops
anurag4DSB ad6a043
Add Prometheus metrics instrumentation for IAM ops
anurag4DSB 8973af0
Added unit tests for IAM and S3 metrics
anurag4DSB 6701bdc
Add documentation for metrics overview
anurag4DSB File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -253,3 +253,94 @@ var _ = Describe("Metrics Server Functions", func() { | |
}) | ||
}) | ||
}) | ||
|
||
var _ = Describe("S3 and IAM metrics", func() { | ||
var ( | ||
server *http.Server | ||
listener net.Listener | ||
addr string | ||
) | ||
|
||
BeforeEach(func() { | ||
var err error | ||
listener, err = net.Listen("tcp", "127.0.0.1:0") | ||
Expect(err).NotTo(HaveOccurred()) | ||
|
||
server, err = metrics.StartMetricsServerWithListener(listener) | ||
Expect(err).NotTo(HaveOccurred()) | ||
|
||
addr = listener.Addr().String() | ||
}) | ||
|
||
AfterEach(func() { | ||
if server != nil { | ||
_ = server.Close() | ||
} | ||
if listener != nil { | ||
_ = listener.Close() | ||
} | ||
}) | ||
|
||
It("should correctly increment S3 metrics", func() { | ||
metrics.S3RequestsTotal.WithLabelValues("CreateBucket", "success").Inc() | ||
metrics.S3RequestsTotal.WithLabelValues("CreateBucket", "error").Add(2) | ||
|
||
resp, err := http.Get(fmt.Sprintf("http://%s%s", addr, constants.MetricsPath)) | ||
Expect(err).NotTo(HaveOccurred()) | ||
Expect(resp.StatusCode).To(Equal(http.StatusOK)) | ||
|
||
body, err := io.ReadAll(resp.Body) | ||
Expect(err).NotTo(HaveOccurred()) | ||
Expect(resp.Body.Close()).To(Succeed()) | ||
Expect(string(body)).To(ContainSubstring(`s3_requests_total{method="CreateBucket",status="success"} 1`)) | ||
Expect(string(body)).To(ContainSubstring(`s3_requests_total{method="CreateBucket",status="error"} 2`)) | ||
}) | ||
|
||
It("should correctly record IAM request durations", func() { | ||
timer := prometheus.NewTimer(metrics.IAMRequestDuration.WithLabelValues("CreateUser", "success")) | ||
time.Sleep(50 * time.Millisecond) | ||
timer.ObserveDuration() | ||
|
||
resp, err := http.Get(fmt.Sprintf("http://%s%s", addr, constants.MetricsPath)) | ||
Expect(err).NotTo(HaveOccurred()) | ||
Expect(resp.StatusCode).To(Equal(http.StatusOK)) | ||
|
||
body, err := io.ReadAll(resp.Body) | ||
Expect(err).NotTo(HaveOccurred()) | ||
Expect(resp.Body.Close()).To(Succeed()) | ||
Expect(string(body)).To(ContainSubstring("iam_request_duration_seconds_bucket")) | ||
Expect(string(body)).To(ContainSubstring("iam_request_duration_seconds_sum")) | ||
}) | ||
|
||
It("should handle invalid metric labels gracefully", func() { | ||
Expect(func() { | ||
metrics.S3RequestsTotal.WithLabelValues("InvalidMethod", "").Inc() | ||
}).NotTo(Panic()) | ||
}) | ||
|
||
It("should return 404 for invalid metrics paths", func() { | ||
resp, err := http.Get(fmt.Sprintf("http://%s/invalid-path", addr)) | ||
Expect(err).NotTo(HaveOccurred()) | ||
Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) | ||
Expect(resp.Body.Close()).To(Succeed()) | ||
}) | ||
|
||
It("should support concurrent requests to the metrics endpoint", func() { | ||
const numRequests = 10 | ||
done := make(chan bool, numRequests) | ||
|
||
for i := 0; i < numRequests; i++ { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You could use a Also, it's good practice to call |
||
go func() { | ||
resp, err := http.Get(fmt.Sprintf("http://%s%s", addr, constants.MetricsPath)) | ||
Expect(err).NotTo(HaveOccurred()) | ||
Expect(resp.StatusCode).To(Equal(http.StatusOK)) | ||
Expect(resp.Body.Close()).To(Succeed()) | ||
done <- true | ||
}() | ||
} | ||
|
||
for i := 0; i < numRequests; i++ { | ||
<-done | ||
} | ||
}) | ||
}) |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
FWIW the prometheus module contains test utilities for this kind of thing, here this function could be used: https://pkg.go.dev/github.com/prometheus/[email protected]/prometheus/testutil#ScrapeAndCompare.
It's totally up to you if you'd like to use them or prefer custom test code though (and in case you need an example, some of the metrics PRs for metadata migration use some of those test functions).