diff --git a/.github/workflows/chart.yaml b/.github/workflows/chart.yaml index 4535160..1f9540e 100644 --- a/.github/workflows/chart.yaml +++ b/.github/workflows/chart.yaml @@ -60,25 +60,6 @@ jobs: - name: Run audit run: | polaris audit --helm-chart ./charts/well-known --helm-values ./charts/well-known/values.yaml --format pretty --set-exit-code-on-danger --set-exit-code-below-score 90 - - kubescape-scan: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Install Kubescape - run: curl -s https://raw.githubusercontent.com/armosec/kubescape/master/install.sh | /bin/bash - - - name: Set up Helm - uses: azure/setup-helm@v3 - with: - version: v3.7.1 - - - name: Scan helm - run: helm template ./charts/well-known --namespace fake -f ./charts/well-known/ci/pluto-values.yaml | ~/.kubescape/bin/kubescape scan --controls-config .github/kubescape-controls-inputs.json -v --fail-threshold 15 - pluto-scan: runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index d202cc3..e6cde99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20 AS build-server +FROM golang:1.22 AS build-server WORKDIR /workspace/server # Copy the Go Modules manifests @@ -19,16 +19,21 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -o well-known ./ FROM alpine AS downloader -RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 -RUN chmod +x /usr/local/bin/dumb-init +ARG TARGETPLATFORM +ARG TINI_VERSION=v0.19.0 +RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then ARCHITECTURE=amd64; elif [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then ARCHITECTURE=arm; elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then ARCHITECTURE=arm64; else ARCHITECTURE=amd64; fi \ + && wget -O /usr/local/bin/tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-${ARCHITECTURE} +RUN chmod +x /usr/local/bin/tini + +# # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details FROM gcr.io/distroless/static:nonroot WORKDIR /app -COPY --from=downloader /usr/local/bin/dumb-init /app/dumb-init +COPY --from=downloader /usr/local/bin/tini /app/tini COPY --from=build-server /workspace/server/well-known /app/well-known USER 65532:65532 -ENTRYPOINT ["/app/dumb-init", "--", "/app/well-known"] +ENTRYPOINT ["/app/tini", "--", "/app/well-known"] diff --git a/charts/well-known/ci/pluto-values.yaml b/charts/well-known/ci/pluto-values.yaml index 17b1f3f..45ee7cc 100644 --- a/charts/well-known/ci/pluto-values.yaml +++ b/charts/well-known/ci/pluto-values.yaml @@ -6,4 +6,4 @@ autoscaling: networkpolicies: enabled: true - kubeApiServerCIDR: 1.2.3.4/32 \ No newline at end of file + kubeApiServerCIDR: 1.2.3.4/32 diff --git a/charts/well-known/templates/configmap.yaml b/charts/well-known/templates/configmap.yaml deleted file mode 100644 index dc4f7c4..0000000 --- a/charts/well-known/templates/configmap.yaml +++ /dev/null @@ -1,50 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: {{ include "well-known.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "well-known.labels" . | nindent 4 }} -data: - default.conf: | - server { - listen 8080; - server_name _; - - {{- if not .Values.webserver.config.accessLogEnabled }} - access_log off; - {{- end }} - - location /.well-known/ { - default_type application/json; - root /usr/share/nginx/html; - try_files $uri $uri/ $uri.json $uri/index.json =404; - } - - error_page 400 404 405 =200 @40*_json; - location @40*_json { - default_type application/json; - return 200 '{"code":"1", "message": "Not Found"}'; - } - - error_page 500 502 503 504 =200 @50*_json; - location @50*_json { - default_type application/json; - return 200 '{"code":"1", "message": "Unknown Error"}'; - } - } - server { - listen 8082; - server_name localhost; - root /usr/share/nginx/html; - - access_log off; - allow 127.0.0.1; - deny all; - - location /healthz { - allow 127.0.0.1; - stub_status; - server_tokens on; - } - } diff --git a/charts/well-known/templates/deployment.yaml b/charts/well-known/templates/deployment.yaml index d6f1d53..0fb2d1c 100644 --- a/charts/well-known/templates/deployment.yaml +++ b/charts/well-known/templates/deployment.yaml @@ -30,36 +30,6 @@ spec: securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: - - name: webserver - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.webserver.image.repository }}:{{ .Values.webserver.image.tag }}" - imagePullPolicy: {{ .Values.webserver.image.pullPolicy }} - ports: - - name: http - containerPort: 8080 - protocol: TCP - - name: probe - containerPort: 8082 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: probe - readinessProbe: - httpGet: - path: /healthz - port: probe - volumeMounts: - - name: config - mountPath: /etc/nginx/conf.d/default.conf - subPath: default.conf - - name: data - mountPath: /usr/share/nginx/html/.well-known - - mountPath: /tmp - name: tmp-volume - resources: - {{- toYaml .Values.webserver.resources | nindent 12 }} - name: {{ .Chart.Name }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} @@ -76,6 +46,9 @@ spec: apiVersion: v1 fieldPath: metadata.name ports: + - name: http + containerPort: 8080 + protocol: TCP - name: probe containerPort: 8081 protocol: TCP @@ -101,13 +74,3 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} - volumes: - - name: config - configMap: - name: {{ include "well-known.fullname" . }} - - name: data - configMap: - name: {{ include "well-known.fullname" . }}-data - optional: true - - name: tmp-volume - emptyDir: {} \ No newline at end of file diff --git a/charts/well-known/values.yaml b/charts/well-known/values.yaml index 1392356..21adf0e 100644 --- a/charts/well-known/values.yaml +++ b/charts/well-known/values.yaml @@ -18,22 +18,7 @@ resources: cpu: 20m memory: 32Mi -webserver: - image: - repository: nginxinc/nginx-unprivileged - pullPolicy: Always - tag: "1.25" - resources: - limits: - cpu: 50m - memory: 24Mi - requests: - cpu: 10m - memory: 10Mi - config: - accessLogEnabled: false - -podDisruptionBudget: +podDisruptionBudget: maxUnavailable: 1 imagePullSecrets: [] @@ -93,8 +78,9 @@ autoscaling: networkpolicies: enabled: false - kubeApi: [] # kubectl get svc -n default kubernetes -oyaml - # - addresses: + kubeApi: [] + # kubectl get svc -n default kubernetes -oyaml + # - addresses: # - 10.0.0.153 # - 10.0.0.90 # ports: diff --git a/go.mod b/go.mod index e33108b..e5df10d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module well-known -go 1.20 +go 1.22 require ( k8s.io/api v0.28.3 diff --git a/server/main.go b/server/main.go index b5e7d12..aca0fb6 100644 --- a/server/main.go +++ b/server/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "encoding/json" "flag" "net/http" "os" @@ -11,8 +10,6 @@ import ( "syscall" "time" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -21,19 +18,21 @@ import ( "k8s.io/client-go/util/homedir" klog "k8s.io/klog/v2" - "github.com/bep/debounce" "k8s.io/client-go/tools/clientcmd" - - "github.com/davegardnerisme/deephash" ) -var kubeconfig string -var namespace string -var cmName string -var id string -var leaseLockName string +var ( + kubeconfig string + namespace string + cmName string + id string + leaseLockName string -func main() { + serverPort string + healthPort string +) + +func parseFlags() { klog.InitFlags(nil) if home := homedir.HomeDir(); home != "" { flag.StringVar(&kubeconfig, "kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") @@ -44,9 +43,16 @@ func main() { flag.StringVar(&cmName, "configmap", "well-known-generated", "") flag.StringVar(&id, "id", os.Getenv("POD_NAME"), "the holder identity name") flag.StringVar(&leaseLockName, "lease-lock-name", "well-known", "the lease lock resource name") + flag.StringVar(&serverPort, "server-port", "8080", "server port") + flag.StringVar(&healthPort, "health-port", "8081", "health port") flag.Parse() - // creates the in-cluster config + if id == "" { + klog.Fatal("id is required") + } +} + +func getClientset() *kubernetes.Clientset { config, err := rest.InClusterConfig() if err != nil { config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) @@ -60,6 +66,13 @@ func main() { klog.Fatal(err) } + return clientset +} + +func main() { + // Parse flags + parseFlags() + // use a Go context so we can tell the leaderelection code when we // want to step down ctx, cancel := context.WithCancel(context.Background()) @@ -76,34 +89,41 @@ func main() { cancel() }() - // we use the Lease lock type since edits to Leases are less common - // and fewer objects in the cluster watch "all Leases". - lock := &resourcelock.LeaseLock{ - LeaseMeta: metav1.ObjectMeta{ - Name: leaseLockName, - Namespace: namespace, - }, - Client: clientset.CoordinationV1(), - LockConfig: resourcelock.ResourceLockConfig{ - Identity: id, - }, - } + // Connect to the cluster + clientset := getClientset() + + wks := NewWellKnownService(clientset, namespace, cmName) + // Start the server go func() { - http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - }) + klog.Infof("Running /.well-known/{id} endpoint on :%s", serverPort) + if err := http.ListenAndServe(":"+serverPort, GetServer(wks)); err != nil { + klog.Error(err) + os.Exit(1) + } + }() - klog.Info("Running /healthz endpoint on :8081") - if err := http.ListenAndServe(":8081", nil); err != nil { + // Start the health server + go func() { + klog.Infof("Running /healthz endpoint on :%s", healthPort) + if err := http.ListenAndServe(":"+healthPort, GetHealthServer()); err != nil { klog.Error(err) os.Exit(1) } }() - // start the leader election code loop + // Start the leader election code loop leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ - Lock: lock, + Lock: &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Name: leaseLockName, + Namespace: namespace, + }, + Client: clientset.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: id, + }, + }, ReleaseOnCancel: true, LeaseDuration: 60 * time.Second, RenewDeadline: 15 * time.Second, @@ -112,7 +132,7 @@ func main() { OnStartedLeading: func(ctx context.Context) { for { // ensure that we keep observing - loop(ctx, clientset) + wks.DiscoveryLoop(ctx) } }, OnStoppedLeading: func() { @@ -128,106 +148,3 @@ func main() { }, }) } - -func loop(ctx context.Context, clientset *kubernetes.Clientset) { - watch, err := clientset. - CoreV1(). - Services(namespace). - Watch(ctx, metav1.ListOptions{}) - if err != nil { - klog.Error(err) - os.Exit(1) - } - - debounced := debounce.New(500 * time.Millisecond) - hash := []byte{} - - for event := range watch.ResultChan() { - svc, ok := event.Object.(*v1.Service) - if !ok { - continue - } - klog.V(1).Infof("Change detected on %s", svc.GetName()) - - debounced(func() { - reg, err := discoverData(clientset, namespace) - if err != nil { - klog.Error(err) - return - } - - newHash := deephash.Hash(reg) - if string(hash) == string(newHash) { - klog.V(1).Info("No changes detected") - return - } - hash = newHash - - klog.Info("Writing configmap") - if err := updateConfigMap(ctx, clientset, reg); err != nil { - klog.Error(err) - } - }) - } -} - -func discoverData(clientset *kubernetes.Clientset, ns string) (wkRegistry, error) { - reg := make(wkRegistry, 0) - - svcs, err := clientset. - CoreV1(). - Services(namespace). - List(context.Background(), metav1.ListOptions{}) - if err != nil { - return reg, err - } - - for _, svc := range svcs.Items { - for name, value := range svc.ObjectMeta.Annotations { - name = resolveName(name) - if name == "" { - continue - } - - if _, ok := reg[name]; !ok { - reg[name] = make(wkData, 0) - } - - var d map[string]interface{} - err := json.Unmarshal([]byte(value), &d) - if err != nil { - klog.Error(err) - } - - reg[name].append(d) - } - } - return reg, nil -} - -func updateConfigMap(ctx context.Context, client kubernetes.Interface, reg wkRegistry) error { - cm := &v1.ConfigMap{Data: reg.encode()} - cm.Namespace = namespace - cm.Name = cmName - - _, err := client. - CoreV1(). - ConfigMaps(namespace). - Update(ctx, cm, metav1.UpdateOptions{}) - if errors.IsNotFound(err) { - _, err = client. - CoreV1(). - ConfigMaps(namespace). - Create(ctx, cm, metav1.CreateOptions{}) - if err == nil { - klog.Infof("Created ConfigMap %s/%s\n", cm.GetNamespace(), cm.GetName()) - } - return err - } else if err != nil { - klog.Error(err) - return err - } - - klog.Infof("Updated ConfigMap %s/%s\n", cm.GetNamespace(), cm.GetName()) - return nil -} diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..8e5ec0e --- /dev/null +++ b/server/server.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" +) + +type WellKnownGetter interface { + GetData(ctx context.Context) (*wkRegistry, error) +} + +func GetHealthServer() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + return mux +} + +func GetServer(wks WellKnownGetter) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/.well-known/{id}", func(w http.ResponseWriter, r *http.Request) { + reg, err := wks.GetData(r.Context()) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Failed to fetch well-known records")) + return + } + + if reg == nil { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("Not found")) + return + } + + items := reg + id := r.PathValue("id") + if val, ok := (*items)[id]; ok { + b, err := json.Marshal(val) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Failed to encode")) + return + } + + w.WriteHeader(http.StatusOK) + w.Write(b) + return + } + + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("Not found")) + }) + + return mux +} diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 0000000..70fca47 --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +type fakeWellKnownGetter struct { + reg *wkRegistry +} + +func (f *fakeWellKnownGetter) GetData(ctx context.Context) (*wkRegistry, error) { + return f.reg, nil +} + +func Test_GetServer(t *testing.T) { + wks := &fakeWellKnownGetter{ + reg: &wkRegistry{ + "test": { + "key": "value", + }, + "empty": {}, + }, + } + + tt := []struct { + name string + path string + expected string + code int + }{ + { + name: "existing", + path: "/.well-known/test", + expected: `{"key":"value"}`, + code: http.StatusOK, + }, + { + name: "non-existing", + path: "/.well-known/non-existing", + expected: "Not found", + code: http.StatusNotFound, + }, + { + name: "empty", + path: "/.well-known/empty", + expected: "{}", + code: http.StatusOK, + }, + } + + server := GetServer(wks) + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest("GET", tc.path, nil) + w := httptest.NewRecorder() + server.ServeHTTP(w, req) + + if w.Code != tc.code { + t.Errorf("Expected status code %d, got %d", tc.code, w.Code) + } + + if w.Body.String() != tc.expected { + t.Errorf("Expected body %s, got %s", tc.expected, w.Body.String()) + } + + }) + } +} diff --git a/server/wellknown.go b/server/wellknown.go new file mode 100644 index 0000000..cd297aa --- /dev/null +++ b/server/wellknown.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "time" + + "github.com/bep/debounce" + "github.com/davegardnerisme/deephash" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + klog "k8s.io/klog/v2" +) + +var regLocal *wkRegistry + +type WellKnownService struct { + clientset *kubernetes.Clientset + namespace string + cmName string + + localCache *wkRegistry +} + +func NewWellKnownService(clientset *kubernetes.Clientset, namespace string, cmName string) *WellKnownService { + return &WellKnownService{ + clientset: clientset, + namespace: namespace, + cmName: cmName, + } +} + +func (s *WellKnownService) GetData(ctx context.Context) (*wkRegistry, error) { + if s.localCache != nil { + return s.localCache, nil + } + + cm, err := s.clientset.CoreV1().ConfigMaps(s.namespace).Get(ctx, s.cmName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return nil, nil + } else if err != nil { + return nil, err + } + + reg := make(wkRegistry, 0) + for name, data := range cm.Data { + var d wkData + if err := json.Unmarshal([]byte(data), &d); err != nil { + klog.Error(err) + } + reg[name] = d + } + + return ®, nil +} + +func (s *WellKnownService) UpdateConfigMap(ctx context.Context, reg wkRegistry) error { + s.localCache = ® + + cm := &v1.ConfigMap{Data: reg.encode()} + cm.Namespace = s.namespace + cm.Name = s.cmName + + _, err := s.clientset. + CoreV1(). + ConfigMaps(s.namespace). + Update(ctx, cm, metav1.UpdateOptions{}) + if errors.IsNotFound(err) { + _, err = s.clientset. + CoreV1(). + ConfigMaps(s.namespace). + Create(ctx, cm, metav1.CreateOptions{}) + if err == nil { + klog.Infof("Created ConfigMap %s/%s\n", cm.GetNamespace(), cm.GetName()) + } + return err + } else if err != nil { + klog.Error(err) + return err + } + + klog.Infof("Updated ConfigMap %s/%s\n", cm.GetNamespace(), cm.GetName()) + return nil +} + +func (s *WellKnownService) DiscoveryLoop(ctx context.Context) { + watch, err := s.clientset. + CoreV1(). + Services(s.namespace). + Watch(ctx, metav1.ListOptions{}) + if err != nil { + klog.Error(err) + os.Exit(1) + } + + debounced := debounce.New(500 * time.Millisecond) + hash := []byte{} + + for event := range watch.ResultChan() { + svc, ok := event.Object.(*v1.Service) + if !ok { + continue + } + klog.V(1).Infof("Change detected on %s", svc.GetName()) + + debounced(func() { + reg, err := s.collectData(ctx) + if err != nil { + klog.Error(err) + return + } + + newHash := deephash.Hash(reg) + if string(hash) == string(newHash) { + klog.V(1).Info("No changes detected") + return + } + hash = newHash + + klog.Info("Writing configmap") + if err := s.UpdateConfigMap(ctx, reg); err != nil { + klog.Error(err) + } + }) + } +} + +func (s *WellKnownService) collectData(ctx context.Context) (wkRegistry, error) { + reg := make(wkRegistry, 0) + + svcs, err := s.clientset. + CoreV1(). + Services(s.namespace). + List(ctx, metav1.ListOptions{}) + if err != nil { + return reg, err + } + + for _, svc := range svcs.Items { + for name, value := range svc.ObjectMeta.Annotations { + name = resolveName(name) + if name == "" { + continue + } + + if _, ok := reg[name]; !ok { + reg[name] = make(wkData, 0) + } + + var d map[string]interface{} + err := json.Unmarshal([]byte(value), &d) + if err != nil { + klog.Error(err) + } + + reg[name].append(d) + } + } + return reg, nil +}