From aa55160aeefad2438f9fac8dd429dab55a56c85f Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 25 Feb 2024 18:26:47 -0300 Subject: [PATCH] add integration test framework --- .github/workflows/build.yaml | 31 +- Makefile | 22 +- README.md | 3 +- go.mod | 2 + pkg/converters/utils/services.go | 14 +- tests/framework/framework.go | 400 ++++++++++++++++++++++++++ tests/framework/options/objects.go | 50 ++++ tests/framework/options/request.go | 20 ++ tests/integration/integration_test.go | 53 ++++ 9 files changed, 586 insertions(+), 9 deletions(-) create mode 100644 tests/framework/framework.go create mode 100644 tests/framework/options/objects.go create mode 100644 tests/framework/options/request.go create mode 100644 tests/integration/integration_test.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f96dac17c..c4d7d5243 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,11 +13,34 @@ jobs: steps: - uses: actions/setup-go@v4 with: - go-version: 1.21.6 + go-version: "1.21.6" - uses: actions/checkout@v4 - name: golangci-lint uses: golangci/golangci-lint-action@v3 - name: Run build - run: go build -o haproxy-ingress pkg/main.go - - name: Run tests - run: go test ./... + run: make build + - name: Run unit tests + run: make test + integration: + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: sudo apt-get install -y lua-json + - name: Install HAProxy + uses: timwolla/action-install-haproxy@main + id: install-haproxy + with: + branch: "2.2" + use_openssl: yes + use_lua: yes + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: "1.21.6" + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + username: jcmoraisjr + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Run integration tests + run: make test-integration diff --git a/Makefile b/Makefile index 3eadec688..f1f34701f 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,10 @@ KUBECONFIG?=$(HOME)/.kube/config CONTROLLER_CONFIGMAP?= CONTROLLER_ARGS?= +LOCALBIN?=$(shell pwd)/bin +LOCAL_GOTESTSUM=$(LOCALBIN)/gotestsum +LOCAL_SETUP_ENVTEST=$(LOCALBIN)/setup-envtest + .PHONY: build build: CGO_ENABLED=0 go build \ @@ -32,14 +36,28 @@ run: build --configmap=$(CONTROLLER_CONFIGMAP)\ $(CONTROLLER_ARGS) +,PHONY: gotestsum +gotestsum: + test -x $(LOCAL_GOTESTSUM) || GOBIN=$(LOCALBIN) go install gotest.tools/gotestsum@latest + .PHONY: lint lint: golangci-lint run .PHONY: test -test: lint +test: lint gotestsum ## fix race and add -race param - go test -tags cgo ./... + $(LOCAL_GOTESTSUM) -- -tags=cgo ./pkg/... + +.PHONY: setup-envtest +setup-envtest: + test -x $(LOCAL_SETUP_ENVTEST) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + $(LOCAL_SETUP_ENVTEST) use 1.29.1 --bin-dir $(LOCALBIN) + +.PHONY: test-integration +test-integration: gotestsum setup-envtest + KUBEBUILDER_ASSETS="$(shell $(LOCAL_SETUP_ENVTEST) use 1.29.1 --bin-dir $(LOCALBIN) -i -p path)"\ + $(LOCAL_GOTESTSUM) -- -count=1 -tags=cgo ./tests/integration/... .PHONY: linux-build linux-build: diff --git a/README.md b/README.md index f6fd97f9b..9d33b22dc 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,9 @@ The following `make` targets are supported: * `build` (default): Compiles HAProxy Ingress using the default OS and arch, and generates an executable at `bin/controller`. * `run`: Runs HAProxy Ingress locally. -* `lint`: Runs [`golangci-lint`](https://golangci-lint.run/). +* `lint`: Runs [`golangci-lint`](https://golangci-lint.run/), needs golangci-lint in the path. * `test`: Runs unit tests. +* `test-integration`: Runs integration tests, needs haproxy 2.2+ in the path. * `linux-build`: Compiles HAProxy Ingress and generates an ELF (Linux) executable despite the source platform at `rootfs/haproxy-ingress-controller`. Used by `image` step. * `image`: Compiles HAProxy Ingress locally and generates a Docker image. * `docker-build`: Compiles HAProxy Ingress and generates a Docker image using a multi-stage Dockerfile. diff --git a/go.mod b/go.mod index 1c24086e7..d3bd86e61 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/prometheus/client_golang v1.18.0 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.18.0 gopkg.in/go-playground/pool.v3 v3.1.1 @@ -58,6 +59,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect diff --git a/pkg/converters/utils/services.go b/pkg/converters/utils/services.go index 4a8c8c499..ebb72b273 100644 --- a/pkg/converters/utils/services.go +++ b/pkg/converters/utils/services.go @@ -76,15 +76,25 @@ type Endpoint struct { } func createEndpoints(endpoints *api.Endpoints, svcPort *api.ServicePort) (ready, notReady []*Endpoint, err error) { + var ipOverride string + if ann := endpoints.Annotations; ann != nil { + ipOverride = ann["haproxy-ingress.github.io/ip-override"] + } + resolveIP := func(ip string) string { + if ipOverride != "" { + return ipOverride + } + return ip + } for _, subset := range endpoints.Subsets { for _, epPort := range subset.Ports { if matchPort(svcPort, &epPort) { port := int(epPort.Port) for _, addr := range subset.Addresses { - ready = append(ready, newEndpoint(addr.IP, port, addr.TargetRef)) + ready = append(ready, newEndpoint(resolveIP(addr.IP), port, addr.TargetRef)) } for _, addr := range subset.NotReadyAddresses { - notReady = append(notReady, newEndpoint(addr.IP, port, addr.TargetRef)) + notReady = append(notReady, newEndpoint(resolveIP(addr.IP), port, addr.TargetRef)) } } } diff --git a/tests/framework/framework.go b/tests/framework/framework.go new file mode 100644 index 000000000..f8ac395b4 --- /dev/null +++ b/tests/framework/framework.go @@ -0,0 +1,400 @@ +package framework + +import ( + "context" + "fmt" + "io" + "math/rand" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + goruntime "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + core "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + ctrlconfig "github.com/jcmoraisjr/haproxy-ingress/pkg/controller/config" + "github.com/jcmoraisjr/haproxy-ingress/pkg/controller/launch" + "github.com/jcmoraisjr/haproxy-ingress/tests/framework/options" +) + +func NewFramework(t *testing.T) *framework { + wd, err := os.Getwd() + require.NoError(t, err) + if filepath.Base(wd) == "integration" { + err := os.Chdir(filepath.Join("..", "..")) + require.NoError(t, err) + } + _, err = os.Stat("rootfs") + require.NoError(t, err) + + major, minor, full := haproxyVersion(t) + if major < 2 || (major == 2 && minor < 2) { + require.Fail(t, "unsupported haproxy version", "need haproxy 2.2 or newer, found %s", full) + } + t.Logf("using haproxy %s\n", full) + + config := startApiserver(t) + startController(t, config) + + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(gatewayv1alpha2.AddToScheme(scheme)) + utilruntime.Must(gatewayv1beta1.AddToScheme(scheme)) + codec := serializer.NewCodecFactory(scheme) + + cli, err := client.NewWithWatch(config, client.Options{Scheme: scheme}) + require.NoError(t, err) + + return &framework{ + scheme: scheme, + codec: codec, + cli: cli, + } +} + +type framework struct { + scheme *runtime.Scheme + codec serializer.CodecFactory + cli client.WithWatch +} + +// HAProxy version 3.0-dev4-dec0175 2024/02/23 - https://haproxy.org/ +// HAProxy version 2.9.5-260dbb8 2024/02/15 - https://haproxy.org/ +// HAProxy version 2.8.6-f6bd011 2024/02/15 - https://haproxy.org/ +// HAProxy version 2.6.16-c6a7346 2023/12/13 - https://haproxy.org/ +// HAProxy version 2.4.25-6cfe787 2023/12/14 - https://haproxy.org/ +// HA-Proxy version 2.2.32-4081d5a 2023/12/19 - https://haproxy.org/ +// HA-Proxy version 2.0.34-868040b 2023/12/19 - https://haproxy.org/ +var haproxyVersionRegex = regexp.MustCompile(`^HA-?Proxy version ([0-9]+)\.([0-9]+)([.-][dev0-9]+)`) + +func haproxyVersion(t *testing.T) (major, minor int, full string) { + cmd := exec.Command("haproxy", "-v") + out, err := cmd.CombinedOutput() + require.NoError(t, err, "need haproxy 2.2 or newer installed") + digits := haproxyVersionRegex.FindStringSubmatch(string(out)) + atoi := func(s string) int { + i, err := strconv.Atoi(s) + require.NoError(t, err) + return i + } + major = atoi(digits[1]) + minor = atoi(digits[2]) + full = fmt.Sprintf("%d.%d%s", major, minor, digits[3]) + return major, minor, full +} + +func startApiserver(t *testing.T) *rest.Config { + t.Log("starting apiserver") + + e := envtest.Environment{ + // run `make setup-envtest` to download envtest binaries. + BinaryAssetsDirectory: filepath.Join("bin", "k8s", fmt.Sprintf("1.29.1-%s-%s", goruntime.GOOS, goruntime.GOARCH)), + } + config, err := e.Start() + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, e.Stop()) + }) + return config +} + +func startController(t *testing.T, config *rest.Config) { + t.Log("starting controller") + + err := os.RemoveAll("/tmp/haproxy-ingress") + require.NoError(t, err) + err = os.MkdirAll("/tmp/haproxy-ingress/etc/haproxy/lua/", 0755) + require.NoError(t, err) + luadir, err := os.ReadDir("rootfs/etc/lua/") + require.NoError(t, err) + for _, d := range luadir { + if d.IsDir() { + continue + } + f1, err := os.Open(filepath.Join("rootfs/etc/lua", d.Name())) + require.NoError(t, err) + f2, err := os.OpenFile( + filepath.Join("/tmp/haproxy-ingress/etc/haproxy/lua", d.Name()), + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, + 0644) + require.NoError(t, err) + _, err = io.Copy(f2, f1) + require.NoError(t, err) + } + + opt := ctrlconfig.NewOptions() + opt.UpdateStatus = false + opt.WatchGateway = false + opt.MasterWorker = true + opt.LocalFSPrefix = "/tmp/haproxy-ingress" + opt.PublishAddress = "127.0.0.1" + ctx, cancel := context.WithCancel(context.Background()) + cfg, err := ctrlconfig.CreateWithConfig(ctx, config, opt) + require.NoError(t, err) + + done := make(chan bool) + go func() { + err := launch.Run(cfg) + require.NoError(t, err) + done <- true + }() + + t.Cleanup(func() { + cancel() + <-done + }) +} + +type Response struct { + HTTPResponse *http.Response + Body string + EchoResponse bool + ReqHeaders map[string]string +} + +func (f *framework) Request(ctx context.Context, t *testing.T, method, host, path string, o ...options.Request) Response { + t.Logf("request method=%s host=%s path=%s\n", method, host, path) + opt := options.ParseRequestOptions(o...) + + req, err := http.NewRequestWithContext(ctx, method, "http://127.0.0.1", nil) + require.NoError(t, err) + req.Host = host + req.URL.Path = path + cli := http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + var res *http.Response + if opt.ExpectResponseCode > 0 { + require.EventuallyWithT(t, func(collect *assert.CollectT) { + res, err = cli.Do(req) + require.NoError(collect, err) + assert.Equal(collect, opt.ExpectResponseCode, res.StatusCode) + }, 5*time.Second, time.Second) + } else { + res, err = cli.Do(req) + require.NoError(t, err) + } + require.NotNil(t, res, "request closure reassigned the response") + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + reqHeaders := make(map[string]string) + t.Logf("response body:\n%s\n", body) + strbody := string(body) + echoResponse := strings.HasPrefix(strbody, "echoserver:\n") + if echoResponse { + for _, l := range strings.Split(strbody, "\n")[1:] { + if l == "" { + continue + } + eq := strings.Index(l, "=") + k := strings.ToLower(l[:eq]) + v := l[eq+1:] + reqHeaders[k] = v + } + } + return Response{ + HTTPResponse: res, + Body: strbody, + EchoResponse: echoResponse, + ReqHeaders: reqHeaders, + } +} + +func (f *framework) CreateService(ctx context.Context, t *testing.T, serverPort int32, o ...options.Object) *core.Service { + opt := options.ParseObjectOptions(o...) + data := ` +apiVersion: v1 +Kind: Service +metadata: + name: "" + namespace: default +spec: + ports: + - port: 8080 + targetPort: 0 +` + ep := f.CreateEndpoints(ctx, t, serverPort) + name := ep.Name + + svc := f.CreateObject(t, data).(*core.Service) + svc.Name = name + svc.Spec.Ports[0].TargetPort = intstr.IntOrString{IntVal: serverPort} + opt.Apply(svc) + + t.Logf("creating service %s/%s\n", svc.Namespace, svc.Name) + + err := f.cli.Create(ctx, svc) + require.NoError(t, err) + + t.Cleanup(func() { + svc := core.Service{} + svc.Namespace = "default" + svc.Name = name + err := f.cli.Delete(ctx, &svc) + require.NoError(t, client.IgnoreNotFound(err)) + }) + return svc +} + +func (f *framework) CreateEndpoints(ctx context.Context, t *testing.T, serverPort int32) *core.Endpoints { + data := ` +apiVersion: v1 +Kind: Endpoints +metadata: + annotations: + haproxy-ingress.github.io/ip-override: 127.0.0.1 + name: "" + namespace: default +subsets: +- addresses: + - ip: ::ffff + ports: + - port: 0 +` + name := randomName("svc") + + ep := f.CreateObject(t, data).(*core.Endpoints) + ep.Name = name + ep.Subsets[0].Ports[0].Port = serverPort + + t.Logf("creating endpoints %s/%s\n", ep.Namespace, ep.Name) + + err := f.cli.Create(ctx, ep) + require.NoError(t, err) + + t.Cleanup(func() { + ep := core.Endpoints{} + ep.Namespace = "default" + ep.Name = name + err := f.cli.Delete(ctx, &ep) + require.NoError(t, client.IgnoreNotFound(err)) + }) + return ep +} + +func (f *framework) Host(ing *networking.Ingress, ruleId ...int) string { + var rule int + if len(ruleId) > 0 { + rule = ruleId[0] + } + if rules := ing.Spec.Rules; len(rules) >= rule { + return rules[rule].Host + } + return "" +} + +func (f *framework) CreateIngress(ctx context.Context, t *testing.T, svc *core.Service, o ...options.Object) *networking.Ingress { + opt := options.ParseObjectOptions(o...) + data := ` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.class: haproxy + name: "" + namespace: default +spec: + rules: + - host: "" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: "" + port: + number: 8080 +` + name := randomName("ing") + hostname := name + ".local" + + ing := f.CreateObject(t, data).(*networking.Ingress) + ing.Name = name + ing.Spec.Rules[0].Host = hostname + ing.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = svc.Name + opt.Apply(ing) + if opt.IngressOpt.DefaultTLS { + ing.Spec.TLS = []networking.IngressTLS{{Hosts: []string{hostname}}} + } + + t.Logf("creating ingress %s/%s host=%s\n", ing.Namespace, ing.Name, ing.Spec.Rules[0].Host) + + err := f.cli.Create(ctx, ing) + require.NoError(t, err) + + t.Cleanup(func() { + ing := networking.Ingress{} + ing.Namespace = "default" + ing.Name = name + err := f.cli.Delete(ctx, &ing) + require.NoError(t, client.IgnoreNotFound(err)) + }) + return ing +} + +func (f *framework) CreateObject(t *testing.T, data string) runtime.Object { + obj, _, err := f.codec.UniversalDeserializer().Decode([]byte(data), nil, nil) + require.NoError(t, err) + return obj +} + +func (f *framework) CreateHTTPServer(ctx context.Context, t *testing.T) int32 { + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + content := "echoserver:\n" + for name, values := range r.Header { + for _, value := range values { + content += fmt.Sprintf("%s=%s\n", name, value) + } + } + _, err := w.Write([]byte(content)) + require.NoError(t, err) + }) + + serverPort := int32(32768 + rand.Intn(32767)) + server := http.Server{ + Addr: fmt.Sprintf(":%d", serverPort), + Handler: mux, + } + t.Logf("creating http server at :%d\n", serverPort) + + done := make(chan bool) + go func() { + _ = server.ListenAndServe() + done <- true + }() + + t.Cleanup(func() { + err := server.Shutdown(context.Background()) + require.NoError(t, err) + <-done + }) + return serverPort +} + +func randomName(prefix string) string { + return fmt.Sprintf("%s-%08d", prefix, rand.Intn(1e8)) +} diff --git a/tests/framework/options/objects.go b/tests/framework/options/objects.go new file mode 100644 index 000000000..256423f14 --- /dev/null +++ b/tests/framework/options/objects.go @@ -0,0 +1,50 @@ +package options + +import "sigs.k8s.io/controller-runtime/pkg/client" + +type Object func(o *objectOpt) + +func AddConfigKeyAnnotations(ann map[string]string) Object { + annprefix := "haproxy-ingress.github.io/" + return func(o *objectOpt) { + if o.Ann == nil { + o.Ann = make(map[string]string) + } + for k, v := range ann { + o.Ann[annprefix+k] = v + } + } +} + +func DefaultHostTLS() Object { + return func(o *objectOpt) { + o.IngressOpt.DefaultTLS = true + } +} + +type objectOpt struct { + Ann map[string]string + IngressOpt +} + +type IngressOpt struct { + DefaultTLS bool +} + +func (o *objectOpt) Apply(obj client.Object) { + ann := obj.GetAnnotations() + if ann == nil { + ann = make(map[string]string, len(o.Ann)) + } + for k, v := range o.Ann { + ann[k] = v + } + obj.SetAnnotations(ann) +} + +func ParseObjectOptions(opts ...Object) (opt objectOpt) { + for _, o := range opts { + o(&opt) + } + return opt +} diff --git a/tests/framework/options/request.go b/tests/framework/options/request.go new file mode 100644 index 000000000..76fdfac4a --- /dev/null +++ b/tests/framework/options/request.go @@ -0,0 +1,20 @@ +package options + +type Request func(o *requestOpt) + +func ExpectResponseCode(code int) Request { + return func(o *requestOpt) { + o.ExpectResponseCode = code + } +} + +type requestOpt struct { + ExpectResponseCode int +} + +func ParseRequestOptions(opts ...Request) (opt requestOpt) { + for _, o := range opts { + o(&opt) + } + return opt +} diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go new file mode 100644 index 000000000..7363e13e4 --- /dev/null +++ b/tests/integration/integration_test.go @@ -0,0 +1,53 @@ +package integration_test + +import ( + "context" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + + ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" + "github.com/jcmoraisjr/haproxy-ingress/tests/framework" + "github.com/jcmoraisjr/haproxy-ingress/tests/framework/options" +) + +func TestIntegration(t *testing.T) { + ctx := context.Background() + + f := framework.NewFramework(t) + httpPort := f.CreateHTTPServer(ctx, t) + + t.Run("hello world", func(t *testing.T) { + t.Parallel() + svc := f.CreateService(ctx, t, httpPort) + ing := f.CreateIngress(ctx, t, svc) + res := f.Request(ctx, t, http.MethodGet, f.Host(ing), "/", options.ExpectResponseCode(http.StatusOK)) + assert.True(t, res.EchoResponse) + assert.Equal(t, "http", res.ReqHeaders["x-forwarded-proto"]) + }) + + t.Run("should not redirect to https", func(t *testing.T) { + t.Parallel() + svc := f.CreateService(ctx, t, httpPort) + ing := f.CreateIngress(ctx, t, svc, + options.DefaultHostTLS(), + options.AddConfigKeyAnnotations(map[string]string{ingtypes.BackSSLRedirect: "false"}), + ) + res := f.Request(ctx, t, http.MethodGet, f.Host(ing), "/", options.ExpectResponseCode(http.StatusOK)) + assert.True(t, res.EchoResponse) + }) + + t.Run("should redirect to https", func(t *testing.T) { + t.Parallel() + svc := f.CreateService(ctx, t, httpPort) + ing := f.CreateIngress(ctx, t, svc, + options.DefaultHostTLS(), + options.AddConfigKeyAnnotations(map[string]string{ingtypes.BackSSLRedirect: "true"}), + ) + res := f.Request(ctx, t, http.MethodGet, f.Host(ing), "/", options.ExpectResponseCode(http.StatusFound)) + assert.False(t, res.EchoResponse) + assert.Equal(t, fmt.Sprintf("https://%s/", f.Host(ing)), res.HTTPResponse.Header.Get("location")) + }) +}