From ae46242a643af298478805c57eb6344dd941eecd Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 17 Oct 2024 15:50:45 +0200 Subject: [PATCH] feat: enable HTTPS in OpenShift clusters This change adds an end-to-end test ensuring that the operator's /metrics endpoint works as expected. Signed-off-by: Simon Pasquier --- ...bility-operator.clusterserviceversion.yaml | 11 +- .../observability-operator_v1_service.yaml | 2 + cmd/operator/main.go | 4 +- .../observability-operator-deployment.yaml | 9 ++ .../observability-operator-service.yaml | 2 + go.mod | 1 + go.sum | 2 + pkg/operator/operator.go | 126 +++++++++++++-- test/e2e/framework/assertions.go | 150 ++++++++++++++---- test/e2e/framework/framework.go | 81 ++++++++-- test/e2e/main_test.go | 33 ++-- test/e2e/metrics_test.go | 26 +++ test/e2e/uiplugin_test.go | 1 - 13 files changed, 366 insertions(+), 82 deletions(-) create mode 100644 test/e2e/metrics_test.go diff --git a/bundle/manifests/observability-operator.clusterserviceversion.yaml b/bundle/manifests/observability-operator.clusterserviceversion.yaml index 5a1b459d..06af6f91 100644 --- a/bundle/manifests/observability-operator.clusterserviceversion.yaml +++ b/bundle/manifests/observability-operator.clusterserviceversion.yaml @@ -42,7 +42,7 @@ metadata: categories: Monitoring certified: "false" containerImage: observability-operator:0.4.2 - createdAt: "2024-10-08T12:44:05Z" + createdAt: "2024-10-17T13:23:55Z" description: A Go based Kubernetes operator to setup and manage highly available Monitoring Stack using Prometheus, Alertmanager and Thanos Querier. operators.operatorframework.io/builder: operator-sdk-v1.36.1 @@ -745,6 +745,10 @@ spec: capabilities: drop: - ALL + volumeMounts: + - mountPath: /etc/tls/private + name: observability-operator-tls + readOnly: true securityContext: runAsNonRoot: true serviceAccountName: observability-operator-sa @@ -753,6 +757,11 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/infra operator: Exists + volumes: + - name: observability-operator-tls + secret: + optional: true + secretName: observability-operator-tls strategy: deployment installModes: - supported: false diff --git a/bundle/manifests/observability-operator_v1_service.yaml b/bundle/manifests/observability-operator_v1_service.yaml index 9eb77732..c84b67ff 100644 --- a/bundle/manifests/observability-operator_v1_service.yaml +++ b/bundle/manifests/observability-operator_v1_service.yaml @@ -1,6 +1,8 @@ apiVersion: v1 kind: Service metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: observability-operator-tls creationTimestamp: null labels: app.kubernetes.io/component: operator diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 2d03daf7..0e456831 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -114,7 +114,10 @@ func main() { os.Exit(1) } + ctx := ctrl.SetupSignalHandler() + op, err := operator.New( + ctx, operator.NewOperatorConfiguration( operator.WithMetricsAddr(metricsAddr), operator.WithHealthProbeAddr(healthProbeAddr), @@ -134,7 +137,6 @@ func main() { os.Exit(1) } - ctx := ctrl.SetupSignalHandler() setupLog.Info("starting manager") if err := op.Start(ctx); err != nil { setupLog.Error(err, "terminating") diff --git a/deploy/operator/observability-operator-deployment.yaml b/deploy/operator/observability-operator-deployment.yaml index c1a1fe55..d34c007a 100644 --- a/deploy/operator/observability-operator-deployment.yaml +++ b/deploy/operator/observability-operator-deployment.yaml @@ -53,5 +53,14 @@ spec: httpGet: path: /healthz port: 8081 + volumeMounts: + - mountPath: /etc/tls/private + name: observability-operator-tls + readOnly: true serviceAccountName: observability-operator-sa + volumes: + - name: observability-operator-tls + secret: + secretName: observability-operator-tls + optional: true terminationGracePeriodSeconds: 30 diff --git a/deploy/operator/observability-operator-service.yaml b/deploy/operator/observability-operator-service.yaml index f62ea8dd..b836b193 100644 --- a/deploy/operator/observability-operator-service.yaml +++ b/deploy/operator/observability-operator-service.yaml @@ -6,6 +6,8 @@ metadata: app.kubernetes.io/component: operator app.kubernetes.io/name: observability-operator app.kubernetes.io/part-of: observability-operator + annotations: + service.beta.openshift.io/serving-cert-secret-name: observability-operator-tls spec: selector: app.kubernetes.io/name: observability-operator diff --git a/go.mod b/go.mod index 5efd40cf..f5426c82 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( k8s.io/api v0.31.1 k8s.io/apiextensions-apiserver v0.31.1 k8s.io/apimachinery v0.31.1 + k8s.io/apiserver v0.31.1 k8s.io/client-go v0.31.1 k8s.io/component-base v0.31.1 k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 diff --git a/go.sum b/go.sum index 621f7e58..068717cc 100644 --- a/go.sum +++ b/go.sum @@ -357,6 +357,8 @@ k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/ k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apiserver v0.31.1 h1:Sars5ejQDCRBY5f7R3QFHdqN3s61nhkpaX8/k1iEw1c= +k8s.io/apiserver v0.31.1/go.mod h1:lzDhpeToamVZJmmFlaLwdYZwd7zB+WYRYIboqA1kGxM= k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8= diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 5b2e5151..0bcfa385 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -2,8 +2,16 @@ package operator import ( "context" + "crypto/tls" "fmt" - + "os" + "path/filepath" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -15,15 +23,22 @@ import ( uictrl "github.com/rhobs/observability-operator/pkg/controllers/uiplugin" ) -// NOTE: The instance selector label is hardcoded in static assets. -// Any change to that must be reflected here as well -const instanceSelector = "app.kubernetes.io/managed-by=observability-operator" +const ( + // NOTE: The instance selector label is hardcoded in static assets. + // Any change to that must be reflected here as well + instanceSelector = "app.kubernetes.io/managed-by=observability-operator" + + ObservabilityOperatorName = "observability-operator" -const ObservabilityOperatorName = "observability-operator" + // The mount path for the serving certificate seret is hardcoded in the + // static assets. + tlsMountPath = "/etc/tls/private" +) // Operator embedds manager and exposes only the minimal set of functions type Operator struct { - manager manager.Manager + manager manager.Manager + servingCertController *dynamiccertificates.DynamicServingCertificateController } type OpenShiftFeatureGates struct { @@ -102,14 +117,90 @@ func NewOperatorConfiguration(opts ...func(*OperatorConfiguration)) *OperatorCon return cfg } -func New(cfg *OperatorConfiguration) (*Operator, error) { - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: NewScheme(cfg), - Metrics: metricsserver.Options{ - BindAddress: cfg.MetricsAddr, - }, - HealthProbeBindAddress: cfg.HealthProbeAddr, - }) +func New(ctx context.Context, cfg *OperatorConfiguration) (*Operator, error) { + restConfig := ctrl.GetConfigOrDie() + + metricsOpts := metricsserver.Options{ + BindAddress: cfg.MetricsAddr, + } + + var servingCertController *dynamiccertificates.DynamicServingCertificateController + if cfg.FeatureGates.OpenShift.Enabled { + // When running in OpenShift, the server uses HTTPS thanks to the + // service CA operator. + certFile := filepath.Join(tlsMountPath, "tls.crt") + keyFile := filepath.Join(tlsMountPath, "tls.key") + + // Wait for the files to be mounted into the container. + var pollErr error + err := wait.PollUntilContextTimeout(ctx, time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) { + for _, f := range []string{certFile, keyFile} { + if _, err := os.Stat(f); err != nil { + pollErr = err + return false, nil + } + } + + return true, nil + }) + if err != nil { + return nil, fmt.Errorf("%w: %w", err, pollErr) + } + + // DynamicCertKeyPairContent automatically reloads the certificate and key from disk. + certKeyProvider, err := dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", certFile, keyFile) + if err != nil { + return nil, err + } + if err := certKeyProvider.RunOnce(ctx); err != nil { + return nil, fmt.Errorf("failed to initialize cert/key content: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, err + } + + // ConfigMapCAController automatically reloads the client CA. + clientCAProvider, err := dynamiccertificates.NewDynamicCAFromConfigMapController( + "client-ca", + metav1.NamespaceSystem, + "extension-apiserver-authentication", + "client-ca-file", + kubeClient, + ) + if err != nil { + return nil, fmt.Errorf("failed to initialize client CA controller: %w", err) + } + + servingCertController = dynamiccertificates.NewDynamicServingCertificateController( + &tls.Config{ + ClientAuth: tls.NoClientCert, + }, + clientCAProvider, + certKeyProvider, + nil, + nil, + ) + if err := servingCertController.RunOnce(); err != nil { + return nil, fmt.Errorf("failed to initialize serving certificate controller: %w", err) + } + + metricsOpts.SecureServing = true + metricsOpts.TLSOpts = []func(*tls.Config){ + func(c *tls.Config) { + c.GetConfigForClient = servingCertController.GetConfigForClient + }, + } + } + + mgr, err := ctrl.NewManager( + restConfig, + ctrl.Options{ + Scheme: NewScheme(cfg), + Metrics: metricsOpts, + HealthProbeBindAddress: cfg.HealthProbeAddr, + }) if err != nil { return nil, fmt.Errorf("unable to create manager: %w", err) } @@ -141,11 +232,16 @@ func New(cfg *OperatorConfiguration) (*Operator, error) { } return &Operator{ - manager: mgr, + manager: mgr, + servingCertController: servingCertController, }, nil } func (o *Operator) Start(ctx context.Context) error { + if o.servingCertController != nil { + go o.servingCertController.Run(1, ctx.Done()) + } + if err := o.manager.Start(ctx); err != nil { return fmt.Errorf("unable to start manager: %w", err) } diff --git a/test/e2e/framework/assertions.go b/test/e2e/framework/assertions.go index 064953e2..3430759b 100644 --- a/test/e2e/framework/assertions.go +++ b/test/e2e/framework/assertions.go @@ -3,6 +3,8 @@ package framework import ( "bytes" "context" + "crypto/tls" + "errors" "fmt" "io" "net/http" @@ -14,7 +16,7 @@ import ( monv1 "github.com/rhobs/obo-prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -64,7 +66,7 @@ func (f *Framework) AssertResourceNeverExists(name, namespace string, resource c Name: name, Namespace: namespace, } - if err := f.K8sClient.Get(context.Background(), key, resource); errors.IsNotFound(err) { + if err := f.K8sClient.Get(context.Background(), key, resource); apierrors.IsNotFound(err) { return false, nil } @@ -93,7 +95,7 @@ func (f *Framework) AssertResourceAbsent(name, namespace string, resource client Name: name, Namespace: namespace, } - if err := f.K8sClient.Get(context.Background(), key, resource); errors.IsNotFound(err) { + if err := f.K8sClient.Get(context.Background(), key, resource); apierrors.IsNotFound(err) { return true, nil } @@ -115,6 +117,7 @@ func (f *Framework) AssertResourceEventuallyExists(name, namespace string, resou } return func(t *testing.T) { + t.Helper() if err := wait.PollUntilContextTimeout(context.Background(), option.PollInterval, option.WaitTimeout, true, func(ctx context.Context) (bool, error) { key := types.NamespacedName{ Name: name, @@ -140,6 +143,7 @@ func (f *Framework) AssertStatefulsetReady(name, namespace string, fns ...Option fn(&option) } return func(t *testing.T) { + t.Helper() key := types.NamespacedName{Name: name, Namespace: namespace} if err := wait.PollUntilContextTimeout(context.Background(), option.PollInterval, option.WaitTimeout, true, func(ctx context.Context) (bool, error) { pod := &appsv1.StatefulSet{} @@ -161,6 +165,7 @@ func (f *Framework) AssertDeploymentReady(name, namespace string, fns ...OptionF fn(&option) } return func(t *testing.T) { + t.Helper() key := types.NamespacedName{Name: name, Namespace: namespace} if err := wait.PollUntilContextTimeout(context.Background(), option.PollInterval, option.WaitTimeout, true, func(ctx context.Context) (bool, error) { deployment := &appsv1.Deployment{} @@ -180,7 +185,7 @@ func (f *Framework) GetResourceWithRetry(t *testing.T, name, namespace string, o err := wait.PollUntilContextTimeout(context.Background(), option.PollInterval, option.WaitTimeout, true, func(ctx context.Context) (bool, error) { key := types.NamespacedName{Name: name, Namespace: namespace} - if err := f.K8sClient.Get(context.Background(), key, obj); errors.IsNotFound(err) { + if err := f.K8sClient.Get(context.Background(), key, obj); apierrors.IsNotFound(err) { // retry return false, nil } @@ -193,16 +198,42 @@ func (f *Framework) GetResourceWithRetry(t *testing.T, name, namespace string, o } } -func assertSamples(t *testing.T, metrics []byte, expected map[string]float64) { - t.Helper() +func ParseMetrics(metrics []byte) (model.Vector, error) { sDecoder := expfmt.SampleDecoder{ - Dec: expfmt.NewDecoder(bytes.NewReader(metrics), expfmt.NewFormat(expfmt.FormatType(expfmt.TypeTextPlain))), + Dec: expfmt.NewDecoder( + bytes.NewReader(metrics), + expfmt.NewFormat(expfmt.FormatType(expfmt.TypeTextPlain)), + ), + Opts: &expfmt.DecodeOptions{ + Timestamp: model.TimeFromUnixNano(0), + }, + } + + var ( + samples model.Vector + decSamples = make(model.Vector, 0, 50) + ) + for { + err := sDecoder.Decode(&decSamples) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + samples = append(samples, decSamples...) + decSamples = decSamples[:0] } - samples := model.Vector{} - err := sDecoder.Decode(&samples) + return samples, nil +} + +func assertSamples(t *testing.T, metrics []byte, expected map[string]float64) { + t.Helper() + + samples, err := ParseMetrics(metrics) if err != nil { - t.Errorf("error decoding samples") + t.Errorf("error decoding samples: %s", err) } for _, s := range samples { @@ -216,12 +247,11 @@ func assertSamples(t *testing.T, metrics []byte, expected map[string]float64) { } } -// GetOperatorPod gets the operator pod assuming the operator is deployed in -// "operators" namespace. +// GetOperatorPod gets the operator's pod. func (f *Framework) GetOperatorPod(t *testing.T) *v1.Pod { // get the operator deployment operator := appsv1.Deployment{} - f.AssertResourceEventuallyExists("observability-operator", "operators", &operator)(t) + f.AssertResourceEventuallyExists("observability-operator", f.OperatorNamespace, &operator)(t) selector, err := metav1.LabelSelectorAsSelector(operator.Spec.Selector) if err != nil { @@ -247,35 +277,95 @@ func (f *Framework) GetOperatorPod(t *testing.T) *v1.Pod { return &pods.Items[0] } -func (f *Framework) GetOperatorMetrics(t *testing.T) []byte { - pod := f.GetOperatorPod(t) +type HTTPOptions struct { + scheme string +} - stopChan := make(chan struct{}) - defer close(stopChan) +func WithHTTPS() func(*HTTPOptions) { + return func(o *HTTPOptions) { + o.scheme = "https" + } +} + +func (f *Framework) GetPodMetrics(t *testing.T, pod *v1.Pod, opts ...func(*HTTPOptions)) ([]byte, error) { + t.Helper() + + var ( + stopChan = make(chan struct{}) + w = &bytes.Buffer{} + ) + defer func() { + close(stopChan) + if w.Len() > 0 { + fmt.Println("port-forward logs:\n", w.String()) + } + }() + + var pollErr error if err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, DefaultTestTimeout, true, func(ctx context.Context) (bool, error) { - err := f.StartPortForward(pod.Name, pod.Namespace, "8080", stopChan) - return err == nil, nil + err := f.StartPortForward(pod.Name, pod.Namespace, "8080", stopChan, w) + if err != nil { + pollErr = err + return false, nil + } + + return true, nil }); err != nil { - t.Fatal(err) + return nil, fmt.Errorf("failed to start port-forwarding: %w: %w", err, pollErr) } - resp, err := http.Get("http://localhost:8080/metrics") - if err != nil { - t.Error(err) + httpOptions := HTTPOptions{ + scheme: "http", + } + for _, o := range opts { + o(&httpOptions) } - defer resp.Body.Close() - metrics, err := io.ReadAll(resp.Body) + req, err := http.NewRequest("GET", fmt.Sprintf("%s://localhost:8080/metrics", httpOptions.scheme), nil) if err != nil { - t.Error(err) + return nil, err + } + + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.TLSClientConfig = &tls.Config{ + ServerName: fmt.Sprintf("observability-operator.%s.svc", pod.Namespace), + RootCAs: f.RootCA, + } + + // Avoid transient connectivity issues by retrying a couple of times if needed. + var metrics []byte + pollErr = nil + if err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, DefaultTestTimeout, true, func(ctx context.Context) (bool, error) { + resp, err := (&http.Client{Transport: tr}).Do(req) + if err != nil { + return false, fmt.Errorf("failed to get /metrics: %w", err) + } + defer resp.Body.Close() + + metrics, err = io.ReadAll(resp.Body) + if err != nil { + return false, fmt.Errorf("failed to read /metrics: %w", err) + } + + return true, nil + }); err != nil { + return nil, fmt.Errorf("%w: %w", err, pollErr) } - return metrics + + return metrics, nil } // AssertNoReconcileErrors asserts that there are no reconcilation errors func (f *Framework) AssertNoReconcileErrors(t *testing.T) { t.Helper() - metrics := f.GetOperatorMetrics(t) + + pod := f.GetOperatorPod(t) + + metrics, err := f.GetPodMetrics(t, pod) + if err != nil { + t.Fatalf("pod %s/%s: %s", pod.Namespace, pod.Name, err) + } + assertSamples(t, metrics, map[string]float64{ `{__name__="controller_runtime_reconcile_errors_total", controller="monitoringstack"}`: 0, @@ -343,7 +433,7 @@ func (f *Framework) AssertAlertmanagerAbsent(t *testing.T, name, namespace strin } err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, DefaultTestTimeout, true, func(ctx context.Context) (bool, error) { err := f.K8sClient.Get(context.Background(), key, &am) - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return true, nil } return false, nil @@ -378,7 +468,7 @@ func (f *Framework) AssertPrometheusReplicaStatus(name, namespace string, expect Name: name, Namespace: namespace, } - if err := f.K8sClient.Get(context.Background(), key, &prom); errors.IsNotFound(err) { + if err := f.K8sClient.Get(context.Background(), key, &prom); apierrors.IsNotFound(err) { return false, nil } if prom.Status.Replicas != expectedReplicas { diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 5391e4ae..b9dbe7d7 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -3,16 +3,22 @@ package framework import ( "bytes" "context" + "crypto/x509" "fmt" + "io" "net/http" "net/url" + "path" "strings" "testing" + configv1 "github.com/openshift/api/config/v1" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -28,27 +34,75 @@ type Framework struct { K8sClient client.Client Retain bool IsOpenshiftCluster bool + RootCA *x509.CertPool + OperatorNamespace string +} + +// Setup finalizes the initilization of the Framework object by setting +// parameters which are specific to OpenShift. +func (f *Framework) Setup() error { + clusterVersion := &configv1.ClusterVersion{} + if err := f.K8sClient.Get(context.Background(), client.ObjectKey{Name: "version"}, clusterVersion); err != nil { + if meta.IsNoMatchError(err) { + return nil + } + + return fmt.Errorf("failed to get clusterversion %w", err) + } + + f.IsOpenshiftCluster = true + + // Load the service CA operator's certificate authority. + var ( + cm v1.ConfigMap + key = client.ObjectKey{ + Namespace: "openshift-config", + Name: "openshift-service-ca.crt", + } + ) + if err := f.K8sClient.Get(context.Background(), key, &cm); err != nil { + return err + } + + b, found := cm.Data["service-ca.crt"] + if !found { + return errors.New("failed to find 'service-ca.crt'") + } + + rootCA := x509.NewCertPool() + if !rootCA.AppendCertsFromPEM([]byte(b)) { + return errors.New("invalid service CA") + } + f.RootCA = rootCA + + return nil } // StartPortForward initiates a port forwarding connection to a pod on the localhost interface. // // The function call blocks until the port forwarding proxy server is ready to receive connections. -func (f *Framework) StartPortForward(podName string, ns string, port string, stopChan chan struct{}) error { +func (f *Framework) StartPortForward(podName string, ns string, port string, stopChan chan struct{}, out io.Writer) error { roundTripper, upgrader, err := spdy.RoundTripperFor(f.Config) if err != nil { - return errors.Wrap(err, "error creating RoundTripper") + return fmt.Errorf("error creating RoundTripper: %w", err) + } + + u := fmt.Sprintf("https://%s", strings.TrimPrefix(strings.TrimPrefix(f.Config.Host, "http://"), "https://")) + serverURL, err := url.Parse(u) + if err != nil { + return err } + serverURL.Path = path.Join( + serverURL.Path, + fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", ns, podName), + ) - path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", ns, podName) - hostIP := strings.TrimLeft(f.Config.Host, "htps:/") - serverURL := url.URL{Scheme: "https", Path: path, Host: hostIP} - dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, &serverURL) + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, serverURL) readyChan := make(chan struct{}, 1) - out, errOut := new(bytes.Buffer), new(bytes.Buffer) - forwarder, err := portforward.New(dialer, []string{port}, stopChan, readyChan, out, errOut) + forwarder, err := portforward.New(dialer, []string{port}, stopChan, readyChan, out, out) if err != nil { - return errors.Wrap(err, "failed to create portforward") + return fmt.Errorf("failed to create portforward: %w", err) } go func() { @@ -69,10 +123,17 @@ func (f *Framework) StartServicePortForward(serviceName string, ns string, port if err != nil { return err } + if len(pods) == 0 { return fmt.Errorf("no pods found for service %s/%s", serviceName, ns) } - return f.StartPortForward(pods[0].Name, ns, port, stopChan) + + w := &bytes.Buffer{} + err = f.StartPortForward(pods[0].Name, ns, port, stopChan, w) + if err != nil { + fmt.Println("port-forward logs:\n", w.String()) + } + return err } func (f *Framework) GetStatefulSetPods(name string, namespace string) ([]corev1.Pod, error) { diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 881cdde4..1eb8c59d 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -10,7 +10,6 @@ import ( configv1 "github.com/openshift/api/config/v1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -25,7 +24,10 @@ var ( const e2eTestNamespace = "e2e-tests" -var retain = flag.Bool("retain", false, "When set, the namespace in which tests are run will not be cleaned up") +var ( + retain = flag.Bool("retain", false, "When set, the namespace in which tests are run will not be cleaned up") + operatorInstallNS = flag.String("operatorInstallNS", "openshift-operator", "The namespace where the operator is installed") +) func TestMain(m *testing.M) { flag.Parse() @@ -98,19 +100,14 @@ func setupFramework() error { return err } - isOpenshiftCluster, err := isOpenshiftCluster(k8sClient) - if err != nil { - return err - } - f = &framework.Framework{ - K8sClient: k8sClient, - Config: cfg, - Retain: *retain, - IsOpenshiftCluster: isOpenshiftCluster, + K8sClient: k8sClient, + Config: cfg, + Retain: *retain, + OperatorNamespace: *operatorInstallNS, } - return nil + return f.Setup() } func createNamespace(name string) (func(), error) { @@ -133,15 +130,3 @@ func createNamespace(name string) (func(), error) { return cleanup, nil } - -func isOpenshiftCluster(k8sClient client.Client) (bool, error) { - clusterVersion := &configv1.ClusterVersion{} - err := k8sClient.Get(context.Background(), client.ObjectKey{Name: "version"}, clusterVersion) - if err == nil { - return true, nil - } else if meta.IsNoMatchError(err) { - return false, nil - } else { - return false, fmt.Errorf("failed to get clusterversion %w", err) - } -} diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go new file mode 100644 index 00000000..dfccae92 --- /dev/null +++ b/test/e2e/metrics_test.go @@ -0,0 +1,26 @@ +package e2e + +import ( + "testing" + + "gotest.tools/v3/assert" + + "github.com/rhobs/observability-operator/test/e2e/framework" +) + +func TestOperatorMetrics(t *testing.T) { + pod := f.GetOperatorPod(t) + + var opts []func(*framework.HTTPOptions) + if f.IsOpenshiftCluster { + opts = append(opts, framework.WithHTTPS()) + } + + metrics, err := f.GetPodMetrics(t, pod, opts...) + assert.NilError(t, err) + + v, err := framework.ParseMetrics(metrics) + assert.NilError(t, err) + + assert.Assert(t, len(v) > 0, "no metrics") +} diff --git a/test/e2e/uiplugin_test.go b/test/e2e/uiplugin_test.go index 55ce087f..b6c05094 100644 --- a/test/e2e/uiplugin_test.go +++ b/test/e2e/uiplugin_test.go @@ -17,7 +17,6 @@ import ( "github.com/rhobs/observability-operator/test/e2e/framework" ) -var operatorInstallNS = flag.String("operatorInstallNS", "openshift-operator", "The namespace where the operator is installed") var uiPluginInstallNS string func TestUIPlugin(t *testing.T) {