Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add basic metrics and minor fixes #22

Merged
merged 1 commit into from
Dec 16, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ ifeq (, $(shell which controller-gen))
CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
cd $$CONTROLLER_GEN_TMP_DIR ;\
go mod init tmp ;\
go get sigs.k8s.io/controller-tools/cmd/[email protected] ;\
go install sigs.k8s.io/controller-tools/cmd/[email protected] ;\
rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
}
CONTROLLER_GEN=$(GOBIN)/controller-gen
Expand Down
13 changes: 13 additions & 0 deletions handlers/idler/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package idler

import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)

var (
idleEvents = promauto.NewCounter(prometheus.CounterOpts{
Name: "aergia_idling_events",
Help: "The total number of events that aergia has processed to idle environments",
})
)
5 changes: 3 additions & 2 deletions handlers/idler/service-kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -267,15 +267,16 @@ func (h *Idler) patchIngress(ctx context.Context, opLog logr.Logger, namespace c
}
}
if patched {
// update the namespace to indicate it is not idled
// update the namespace to indicate it is idled
namespaceCopy := namespace.DeepCopy()
mergePatch, _ := json.Marshal(map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]string{
"idling.amazee.io/idled": "false",
"idling.amazee.io/idled": "true",
},
},
})
idleEvents.Inc()
if err := h.Client.Patch(ctx, namespaceCopy, client.RawPatch(types.MergePatchType, mergePatch)); err != nil {
return fmt.Errorf(fmt.Sprintf("Error patching namespace %s", namespace.Name))
}
Expand Down
10 changes: 10 additions & 0 deletions handlers/idler/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"
client "sigs.k8s.io/controller-runtime/pkg/client"
)

Expand All @@ -32,6 +33,15 @@ func (h *Idler) ServiceIdler() {
// in kubernetes, we can reliably check for the existence of this label so that
// we only check namespaces that have been deployed by a lagoon at one point
labelRequirements := generateLabelRequirements(h.Selectors.Service.Namespace)
// only evaluate namespaces that are not idled
selector := generateSelector(idlerSelector{
Name: "idling.amazee.io/idled",
Operator: selection.NotEquals,
Values: []string{
"true",
},
})
labelRequirements = append(labelRequirements, *selector)
listOption = (&client.ListOptions{}).ApplyOptions([]client.ListOption{
client.MatchingLabelsSelector{
Selector: labels.NewSelector().Add(labelRequirements...),
Expand Down
30 changes: 30 additions & 0 deletions handlers/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package metrics

import (
"fmt"
"net/http"
"time"

"github.com/go-logr/logr"
"github.com/prometheus/client_golang/prometheus/promhttp"
)

// NewServer returns a *http.Server serving prometheus metrics in a new
// goroutine.
// Caller should defer Shutdown() for cleanup.
func NewServer(log logr.Logger, addr string) *http.Server {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
s := http.Server{
Addr: addr,
Handler: mux,
ReadTimeout: 16 * time.Second,
WriteTimeout: 16 * time.Second,
}
go func() {
if err := s.ListenAndServe(); err != http.ErrServerClosed {
log.Error(fmt.Errorf("metrics server did not shut down cleanly"), err.Error())
}
}()
return &s
}
16 changes: 12 additions & 4 deletions handlers/unidler/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,9 @@ func (h *Unidler) ingressHandler(path string) func(http.ResponseWriter, *http.Re
if allowUnidle {
// if a namespace exists, it means that the custom-http-errors code is defined in the ingress object
// so do something with that here, like kickstart the idler process to unidle targets
opLog.Info(fmt.Sprintf("Got request in namespace %s", ns))
if h.Debug {
opLog.Info(fmt.Sprintf("Request for %s verfied: %t from xff:%s; tcip:%s; ua: %s, ", ns, verfied, xForwardedFor, trueClientIP, requestUserAgent))
}

file := fmt.Sprintf("%v/unidle.html", path)
forceScaled := h.checkForceScaled(ctx, ns, opLog)
Expand All @@ -103,14 +105,16 @@ func (h *Unidler) ingressHandler(path string) func(http.ResponseWriter, *http.Re
// only unidle environments that aren't force scaled
// actually do the unidling here, lock to prevent multiple unidle operations from running
if verfied {
allowedRequests.Inc()
w.Header().Set("X-Aergia-Allowed", "true")
_, ok := h.Locks.Load(ns)
if !ok {
_, _ = h.Locks.LoadOrStore(ns, ns)
go h.Unidle(ctx, namespace, opLog)
}
} else {
w.Header().Set("X-Aergia-Denied", "true")
verificationRequired.Inc()
w.Header().Set("X-Aergia-Verification-Required", "true")
}
}
if h.Debug == true {
Expand All @@ -133,12 +137,15 @@ func (h *Unidler) ingressHandler(path string) func(http.ResponseWriter, *http.Re
Verifier: signedNamespace,
})
} else {
// respond with 503 to match the standard request
// respond with forbidden
w.Header().Set("X-Aergia-Denied", "true")
h.genericError(w, r, opLog, ext, format, path, "", 503)
blockedRequests.Inc()
h.genericError(w, r, opLog, ext, format, path, "", 403)
}
} else {
w.Header().Set("X-Aergia-Denied", "true")
w.Header().Set("X-Aergia-No-Namespace", "true")
noNamespaceRequests.Inc()
h.genericError(w, r, opLog, ext, format, path, "", code)
}
h.setMetrics(r, start)
Expand Down Expand Up @@ -184,6 +191,7 @@ func (h *Unidler) verifyRequest(r *http.Request, ns *corev1.Namespace) (string,
// if hmac verification is enabled, perform the verification of the request
signedNamespace := hmacSigner(ns.Name, []byte(h.VerifiedSecret))
verifier := r.URL.Query().Get("verifier")
verificationRequests.Inc()
return signedNamespace, hmacVerifier(ns.Name, verifier, []byte(h.VerifiedSecret))
}
return "", true
Expand Down
30 changes: 30 additions & 0 deletions handlers/unidler/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ import (
"net/http"
"strconv"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)

func (h *Unidler) setMetrics(r *http.Request, start time.Time) {
Expand All @@ -16,3 +19,30 @@ func (h *Unidler) setMetrics(r *http.Request, start time.Time) {
h.RequestCount.WithLabelValues(proto).Inc()
h.RequestDuration.WithLabelValues(proto).Observe(duration)
}

var (
allowedRequests = promauto.NewCounter(prometheus.CounterOpts{
Name: "aergia_allowed_requests",
Help: "The total number of requests that aergia has allowed",
})
verificationRequests = promauto.NewCounter(prometheus.CounterOpts{
Name: "aergia_verification_requests",
Help: "The total number of verificiation requests that aergia has recieved",
})
verificationRequired = promauto.NewCounter(prometheus.CounterOpts{
Name: "aergia_verification_required_requests",
Help: "The total number of verificiation required requests that aergia has received",
})
blockedRequests = promauto.NewCounter(prometheus.CounterOpts{
Name: "aergia_blocked_by_block_list",
Help: "The total number of requests that aergia has blocked by an allow or block list rule",
})
noNamespaceRequests = promauto.NewCounter(prometheus.CounterOpts{
Name: "aergia_no_namespace",
Help: "The total number of requests that aergia has received where no namespace was found",
})
unidleEvents = promauto.NewCounter(prometheus.CounterOpts{
Name: "aergia_unidling_events",
Help: "The total number of events that aergia has processed to unidle an environments",
})
)
7 changes: 3 additions & 4 deletions handlers/unidler/unidler.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,9 +191,7 @@ func (h *Unidler) Unidle(ctx context.Context, namespace *corev1.Namespace, opLog
// this could still result in 503 for users until the resulting services/endpoints are active and receiving traffic
for _, deploy := range deployments.Items {
opLog.Info(fmt.Sprintf("Waiting for %s to be running - %s", deploy.ObjectMeta.Name, namespace.Name))
timeout, cancel := context.WithTimeout(ctx, defaultPollTimeout)
defer cancel()
wait.PollUntilWithContext(timeout, defaultPollDuration, h.hasRunningPod(ctx, namespace.Name, deploy.Name))
wait.PollUntilContextTimeout(ctx, defaultPollDuration, defaultPollTimeout, true, h.hasRunningPod(ctx, namespace.Name, deploy.Name))
}
// remove the 503 code from any ingress objects that have it in this namespace
h.removeCodeFromIngress(ctx, namespace.Name, opLog)
Expand All @@ -202,10 +200,11 @@ func (h *Unidler) Unidle(ctx context.Context, namespace *corev1.Namespace, opLog
mergePatch, _ := json.Marshal(map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]string{
"idling.amazee.io/idled": "true",
"idling.amazee.io/idled": "false",
},
},
})
unidleEvents.Inc()
if err := h.Client.Patch(ctx, namespaceCopy, client.RawPatch(types.MergePatchType, mergePatch)); err != nil {
opLog.Info(fmt.Sprintf("Error patching namespace %s", namespace.Name))
}
Expand Down
6 changes: 6 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,15 @@ limitations under the License.
package main

import (
"context"
"flag"
"fmt"
"os"
"time"

"github.com/amazeeio/aergia-controller/controllers"
"github.com/amazeeio/aergia-controller/handlers/idler"
"github.com/amazeeio/aergia-controller/handlers/metrics"
"github.com/amazeeio/aergia-controller/handlers/unidler"
u "github.com/amazeeio/aergia-controller/handlers/unidler"
prometheusapi "github.com/prometheus/client_golang/api"
Expand Down Expand Up @@ -250,6 +252,10 @@ func main() {
os.Exit(1)
}

setupLog.Info("starting aergia metrics server")
m := metrics.NewServer(setupLog, ":9912")
defer m.Shutdown(context.Background())

setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
setupLog.Error(err, "problem running manager")
Expand Down
Loading