Skip to content

Commit

Permalink
chore: update from v2 to v3 and change metrics from rbac proxy to bui…
Browse files Browse the repository at this point in the history
…ltin
  • Loading branch information
shreddedbacon committed Dec 5, 2024
1 parent b59050b commit 60a8323
Show file tree
Hide file tree
Showing 74 changed files with 454 additions and 1,338 deletions.
14 changes: 9 additions & 5 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,17 @@ COPY go.sum go.sum
RUN go mod download

# Copy the go source
COPY main.go main.go
COPY apis/ apis/
COPY controllers/ controllers/
COPY internal/ internal/
COPY cmd/ cmd/
COPY api/ api/
COPY internal/controllers internal/controllers
COPY internal/harbor internal/harbor
COPY internal/helpers internal/helpers
COPY internal/messenger internal/messenger
COPY internal/metrics internal/metrics
COPY internal/utilities internal/utilities

# Build
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=on go build -a -o manager main.go
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=on go build -a -o manager cmd/main.go

# Use distroless as minimal base image to package the manager binary
# Refer to https://github.com/GoogleContainerTools/distroless for more details
Expand Down
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -62,12 +62,12 @@ test: manifests generate fmt vet envtest ## Run tests.
# Build manager binary
.PHONY: manager
manager: generate fmt vet
go build -o bin/manager main.go
go build -o bin/manager cmd/main.go

# Run against the configured Kubernetes cluster in ~/.kube/config
.PHONY: run
run: generate fmt vet manifests
go run ./main.go --controller-namespace=${CONTROLLER_NAMESPACE}
go run ./cmd/main.go --controller-namespace=${CONTROLLER_NAMESPACE}

# Install CRDs into a cluster
.PHONY: install
Expand Down Expand Up @@ -130,7 +130,7 @@ ifeq (, $(shell which controller-gen))
CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
cd $$CONTROLLER_GEN_TMP_DIR ;\
go mod init tmp ;\
go install sigs.k8s.io/controller-tools/cmd/[email protected].2 ;\
go install sigs.k8s.io/controller-tools/cmd/[email protected].5 ;\
rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
}
CONTROLLER_GEN=$(GOBIN)/controller-gen
Expand Down
38 changes: 32 additions & 6 deletions PROJECT
Original file line number Diff line number Diff line change
@@ -1,17 +1,43 @@
domain: lagoon.sh
multigroup: true
layout:
- go.kubebuilder.io/v3
projectName: remote-controller
repo: github.com/uselagoon/remote-controller
resources:
- group: crd
- api:
crdVersion: v1
namespaced: true
controller: true
domain: lagoon.sh
group: crd
kind: LagoonBuild
path: remote-controller/api/lagoon/v1beta2
version: v1beta2
- group: crd
- api:
crdVersion: v1
namespaced: true
controller: true
domain: lagoon.sh
group: crd
kind: LagoonTask
path: remote-controller/api/lagoon/v1beta2
version: v1beta2
- group: crd
- api:
crdVersion: v1
namespaced: true
controller: false
domain: lagoon.sh
group: crd
kind: LagoonBuild
path: remote-controller/api/lagoon/v1beta1
version: v1beta1
- group: crd
- api:
crdVersion: v1
namespaced: true
controller: false
domain: lagoon.sh
group: crd
kind: LagoonTask
path: remote-controller/api/lagoon/v1beta1
version: v1beta1
version: "2"
version: "3"
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
46 changes: 25 additions & 21 deletions main.go → cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,21 +35,21 @@ import (

"github.com/uselagoon/remote-controller/internal/harbor"
"github.com/uselagoon/remote-controller/internal/helpers"
"github.com/uselagoon/remote-controller/internal/metrics"
"github.com/uselagoon/remote-controller/internal/utilities/deletions"
"github.com/uselagoon/remote-controller/internal/utilities/pruner"

cron "gopkg.in/robfig/cron.v2"

"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"

"github.com/hashicorp/golang-lru/v2/expirable"
k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1"
lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2"
harborctrl "github.com/uselagoon/remote-controller/controllers/harbor"
lagoonv1beta1ctrl "github.com/uselagoon/remote-controller/controllers/v1beta1"
lagoonv1beta2ctrl "github.com/uselagoon/remote-controller/controllers/v1beta2"
lagoonv1beta1 "github.com/uselagoon/remote-controller/api/lagoon/v1beta1"
lagoonv1beta2 "github.com/uselagoon/remote-controller/api/lagoon/v1beta2"
harborctrl "github.com/uselagoon/remote-controller/internal/controllers/harbor"
lagoonv1beta1ctrl "github.com/uselagoon/remote-controller/internal/controllers/v1beta1"
lagoonv1beta2ctrl "github.com/uselagoon/remote-controller/internal/controllers/v1beta2"
"github.com/uselagoon/remote-controller/internal/messenger"
k8upv1alpha1 "github.com/vshn/k8up/api/v1alpha1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
Expand Down Expand Up @@ -184,10 +184,10 @@ func main() {

var unauthenticatedRegistry string

flag.StringVar(&metricsAddr, "metrics-addr", ":8080",
"The address the metric endpoint binds to.")
flag.BoolVar(&secureMetrics, "metrics-secure", false,
"If set the metrics endpoint is served securely")
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")

Expand All @@ -210,7 +210,7 @@ func main() {
"The number of startup attempts before exiting.")
flag.IntVar(&startupConnectionInterval, "startup-connection-interval-seconds", 30,
"The duration between startup attempts.")
flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&enableMQ, "enable-message-queue", true,
"Enable message queue to provide updates back to Lagoon.")
Expand Down Expand Up @@ -487,13 +487,21 @@ func main() {
if !enableHTTP2 {
tlsOpts = append(tlsOpts, disableHTTP2)
}
metricsServerOptions := metricsserver.Options{
BindAddress: metricsAddr,
SecureServing: secureMetrics,
TLSOpts: tlsOpts,
}
if secureMetrics {
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Metrics: metricsserver.Options{
BindAddress: metricsAddr,
SecureServing: secureMetrics,
TLSOpts: tlsOpts,
},
Scheme: scheme,
Metrics: metricsServerOptions,
LeaderElection: enableLeaderElection,
LeaderElectionID: leaderElectionID,
})
Expand Down Expand Up @@ -1009,10 +1017,6 @@ func main() {
}
// +kubebuilder:scaffold:builder

setupLog.Info("starting lagoon metrics server")
m := metrics.NewServer(setupLog, ":9912")
defer m.Shutdown(context.Background())

setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
setupLog.Error(err, "problem running manager")
Expand Down
2 changes: 2 additions & 0 deletions config/default/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ resources:
- ../crd
- ../rbac
- ../manager
# [METRICS] Expose the controller manager metrics service.
- metrics_service.yaml
patches:
- path: envs.yaml
- path: manager_auth_proxy_patch.yaml
17 changes: 5 additions & 12 deletions config/default/manager_auth_proxy_patch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,10 @@ spec:
template:
spec:
containers:
- name: kube-rbac-proxy
image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
args:
- "--secure-listen-address=0.0.0.0:8443"
- "--upstream=http://127.0.0.1:8080/"
- "--logtostderr=true"
- "--v=10"
ports:
- containerPort: 8443
name: https
- name: manager
args:
- "--metrics-addr=127.0.0.1:8080"
- "--enable-leader-election"
- "--metrics-bind-address=:8443"
- "--leader-elect"
- "--build-pod-cleanup-cron=*/1 * * * *"
- "--task-pod-cleanup-cron=*/1 * * * *"
- "--harbor-credential-cron=*/1 * * * *"
Expand All @@ -34,3 +24,6 @@ spec:
- "--enable-deprecated-apis"
- "--lagoon-feature-flag-support-k8upv2"
- "--skip-tls-verify"
ports:
- containerPort: 8443
name: https
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ spec:
ports:
- name: https
port: 8443
targetPort: https
protocol: TCP
targetPort: 8443
selector:
control-plane: controller-manager
control-plane: controller-manager
5 changes: 3 additions & 2 deletions config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,12 @@ spec:
- command:
- /manager
args:
- --enable-leader-election
- --leader-elect
image: controller:latest
name: manager
resources:
requests:
cpu: 100m
memory: 20Mi
memory: 400Mi
serviceAccountName: controller-manager
terminationGracePeriodSeconds: 10
22 changes: 16 additions & 6 deletions config/rbac/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
resources:
# All RBAC will be applied under this service account in
# the deployment namespace. You may comment out this resource
# if your manager will use a service account that exists at
# runtime. Be sure to update RoleBinding and ClusterRoleBinding
# subjects if changing service account names.
- service_account.yaml
- role.yaml
- role_binding.yaml
- leader_election_role.yaml
- leader_election_role_binding.yaml
# Comment the following 3 lines if you want to disable
# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
# which protects your /metrics endpoint.
- auth_proxy_service.yaml
- auth_proxy_role.yaml
- auth_proxy_role_binding.yaml
# The following RBAC configurations are used to protect
# the metrics endpoint with authn/authz. These configurations
# ensure that only authorized users and service accounts
# can access the metrics endpoint. Comment the following
# permissions if you want to disable this protection.
# More info: https://book.kubebuilder.io/reference/metrics.html
- metrics_auth_role.yaml
- metrics_auth_role_binding.yaml
- metrics_reader_role.yaml
- metrics_reader_role_binding.yaml
9 changes: 7 additions & 2 deletions config/rbac/leader_election_role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,21 @@ rules:
- patch
- delete
- apiGroups:
- ""
- coordination.k8s.io
resources:
- configmaps/status
- leases
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
4 changes: 2 additions & 2 deletions config/rbac/leader_election_role_binding.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ roleRef:
name: leader-election-role
subjects:
- kind: ServiceAccount
name: default
namespace: system
name: controller-manager
namespace: system
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: proxy-role
name: metrics-auth-role
rules:
- apiGroups: ["authentication.k8s.io"]
resources:
Expand All @@ -11,3 +11,15 @@ rules:
resources:
- subjectaccessreviews
verbs: ["create"]
- apiGroups:
- authentication.k8s.io
resources:
- tokenreviews
verbs:
- create
- apiGroups:
- authorization.k8s.io
resources:
- subjectaccessreviews
verbs:
- create
12 changes: 12 additions & 0 deletions config/rbac/metrics_auth_role_binding.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: metrics-auth-rolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: metrics-auth-role
subjects:
- kind: ServiceAccount
name: controller-manager
namespace: system
9 changes: 9 additions & 0 deletions config/rbac/metrics_reader_role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: metrics-reader
rules:
- nonResourceURLs:
- "/metrics"
verbs:
- get
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: proxy-rolebinding
name: metrics-reader-rolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: proxy-role
name: metrics-reader
subjects:
- kind: ServiceAccount
name: default
namespace: system
name: controller-manager
namespace: system
2 changes: 1 addition & 1 deletion config/rbac/role_binding.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ roleRef:
name: manager-role
subjects:
- kind: ServiceAccount
name: default
name: controller-manager
namespace: system
Loading

0 comments on commit 60a8323

Please sign in to comment.