From 73e0a29774457a6842a80a84a7c666fd169cbbd1 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Sat, 12 Oct 2024 11:20:08 -0300 Subject: [PATCH] feat: VirtualHostMode determines `Host` header of internal requests By default, internal requests (i.e., requests for marking and flushing the web server log) use `localhost` for the `Host` header value. In scenarios, where tests are run against a virtual host this will cause the internal requests to be processed by the default virtual host while the test requests will be processed by the targeted virtual host. If the web server logs are segregated by virtual host, internal and test requests will end up in different log files and go-ftw will not be able to find the markers it is looking for. With `virtual_host_mode: true`, the value of the `Host` header from the test input will be used for internal requests, to ensure that internal requests will also be processed by the targeted virtual host. Fixes #361 --- go.mod | 5 ++-- go.sum | 12 ++++------ runner/run.go | 60 +++++++++++++++++++++++++++++----------------- runner/run_test.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index c9c375d9..9d7b2483 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.3 require ( github.com/Masterminds/sprig v2.22.0+incompatible github.com/corazawaf/coraza/v3 v3.2.2 - github.com/coreruleset/ftw-tests-schema/v2 v2.1.0 + github.com/coreruleset/ftw-tests-schema/v2 v2.1.1 github.com/creativeprojects/go-selfupdate v1.4.0 github.com/go-logr/zerologr v1.2.3 github.com/google/uuid v1.6.0 @@ -19,7 +19,7 @@ require ( github.com/knadh/koanf/providers/rawbytes v0.1.0 github.com/knadh/koanf/v2 v2.1.2 github.com/kyokomi/emoji/v2 v2.2.13 - github.com/magefile/mage bdc92f694516 + github.com/magefile/mage v1.15.1-0.20231118170541-2385abb49a1f github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.10.0 @@ -34,7 +34,6 @@ require ( github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.0 // indirect github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/go.sum b/go.sum index 58582b44..8644c751 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/corazawaf/coraza/v3 v3.2.2/go.mod h1:73JSSNpNrWeF8K+TqKAc7Apxm3uz2rBr github.com/corazawaf/libinjection-go v0.2.2 h1:Chzodvb6+NXh6wew5/yhD0Ggioif9ACrQGR4qjTCs1g= github.com/corazawaf/libinjection-go v0.2.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreruleset/ftw-tests-schema/v2 v2.1.0 h1:2ilKzKRG5UzzxBcrJLXFtPalStdQ9jzzaYFuFk0OEk0= -github.com/coreruleset/ftw-tests-schema/v2 v2.1.0/go.mod h1:ZHVFX5ses4+5IxUP0ufCNg/VqRWxziH6ZuUca092Hxo= +github.com/coreruleset/ftw-tests-schema/v2 v2.1.1 h1:X7dv75NPgPdKtf8zG6ziguGAYogvdAyzxr+152Cgo9g= +github.com/coreruleset/ftw-tests-schema/v2 v2.1.1/go.mod h1:QEkxQti2T54tDWFPocxgAtLhw6J+zXV44JkqGFFP0is= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creativeprojects/go-selfupdate v1.4.0 h1:4ePPd2CPCNl/YoPXeVxpuBLDUZh8rMEKP5ac+1Y/r5c= github.com/creativeprojects/go-selfupdate v1.4.0/go.mod h1:oPG7LmzEmS6OxfqEm620k5VKxP45xFZNKMkp4V5qqUY= @@ -44,8 +44,6 @@ github.com/go-logr/zerologr v1.2.3 h1:up5N9vcH9Xck3jJkXzgyOxozT14R47IyDODz8LM1KS github.com/go-logr/zerologr v1.2.3/go.mod h1:BxwGo7y5zgSHYR1BjbnHPyF/5ZjVKfKxAZANVu6E8Ho= github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/goccy/go-yaml v1.9.2 h1:2Njwzw+0+pjU2gb805ZC1B/uBuAs2VcZ3K+ZgHwDs7w= -github.com/goccy/go-yaml v1.9.2/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -108,8 +106,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO7U= github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE= -github.com/magefile/mage v1.15.1-0.20241124190125-32e01077f0aa h1:RAKYgtUC3OPrwUYKQcwB0wWXCHo09ZWh5TFlnF0nVfA= -github.com/magefile/mage v1.15.1-0.20241124190125-32e01077f0aa/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magefile/mage v1.15.1-0.20231118170541-2385abb49a1f h1:iiLWLoibjCL0XND6inF7bs2nc20lU/FYkiR//VIOLUc= +github.com/magefile/mage v1.15.1-0.20231118170541-2385abb49a1f/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -217,8 +215,6 @@ golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/runner/run.go b/runner/run.go index 8a46b417..71d3f8cd 100644 --- a/runner/run.go +++ b/runner/run.go @@ -134,7 +134,7 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error { //gocyclo:ignore func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase schema.Test, stage schema.Stage) error { stageStartTime := time.Now() - stageID := uuid.NewString() + stageId := uuid.NewString() // Apply global overrides initially testInput := (test.Input)(stage.Input) test.ApplyInputOverrides(runContext.Config, &testInput) @@ -166,7 +166,7 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch } if notRunningInCloudMode(ftwCheck) { - startMarker, err := markAndFlush(runContext, dest, stageID) + startMarker, err := markAndFlush(runContext, &testInput, stageId) if err != nil && !expectErr { return fmt.Errorf("failed to find start marker: %w", err) } @@ -190,7 +190,7 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch } if notRunningInCloudMode(ftwCheck) { - endMarker, err := markAndFlush(runContext, dest, stageID) + endMarker, err := markAndFlush(runContext, &testInput, stageId) if err != nil && !expectErr { return fmt.Errorf("failed to find end marker: %w", err) @@ -220,25 +220,13 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch return nil } -func markAndFlush(runContext *TestRunContext, dest *ftwhttp.Destination, stageID string) ([]byte, error) { - rline := &ftwhttp.RequestLine{ - Method: "GET", - // Use the `/status` endpoint of `httpbin` (http://httpbingo.org), if possible, - // to minimize the amount of data transferred and in the log. - // `httpbin` is used by the CRS test setup. - URI: "/status/200", - Version: "HTTP/1.1", - } - - headers := &ftwhttp.Header{ - "Accept": "*/*", - "User-Agent": "go-ftw test agent", - "Host": "localhost", - runContext.Config.LogMarkerHeaderName: stageID, +func markAndFlush(runContext *TestRunContext, testInput *test.Input, stageId string) ([]byte, error) { + req := buildMarkerRequest(runContext, testInput, stageId) + dest := &ftwhttp.Destination{ + DestAddr: testInput.GetDestAddr(), + Port: testInput.GetPort(), + Protocol: testInput.GetProtocol(), } - - req := ftwhttp.NewRequest(rline, *headers, nil, true) - for i := runContext.Config.MaxMarkerRetries; i > 0; i-- { err := runContext.Client.NewOrReusedConnection(*dest) if err != nil { @@ -250,7 +238,7 @@ func markAndFlush(runContext *TestRunContext, dest *ftwhttp.Destination, stageID return nil, fmt.Errorf("ftw/run: failed sending request to %+v: %w", dest, err) } - marker := runContext.LogLines.CheckLogForMarker(stageID, runContext.Config.MaxMarkerLogLines) + marker := runContext.LogLines.CheckLogForMarker(stageId, runContext.Config.MaxMarkerLogLines) if marker != nil { return marker, nil } @@ -258,6 +246,34 @@ func markAndFlush(runContext *TestRunContext, dest *ftwhttp.Destination, stageID return nil, fmt.Errorf("can't find log marker. Am I reading the correct log? Log file: %s", runContext.Config.LogFile) } +func buildMarkerRequest(runContext *TestRunContext, testInput *test.Input, stageId string) *ftwhttp.Request { + host := "localhost" + if testInput.VirtualHostMode { + // Use the value of the `Host` header of the test for + // internal requests as well, so that all requests target + // the same virtual host. + host = testInput.GetHeaders().Get("Host") + } + + headers := &ftwhttp.Header{ + "Accept": "*/*", + "User-Agent": "go-ftw test agent", + "Host": host, + runContext.Config.LogMarkerHeaderName: stageId, + } + + rline := &ftwhttp.RequestLine{ + Method: "GET", + // Use the `/status` endpoint of `httpbin` (http://httpbingo.org), if possible, + // to minimize the amount of data transferred and in the log. + // `httpbin` is used by the CRS test setup. + URI: "/status/200", + Version: "HTTP/1.1", + } + + return ftwhttp.NewRequest(rline, *headers, nil, true) +} + func needToSkipTest(runContext *TestRunContext, testCase *schema.Test) bool { include := runContext.Include exclude := runContext.Exclude diff --git a/runner/run_test.go b/runner/run_test.go index 8a7bb0ce..ae023617 100644 --- a/runner/run_test.go +++ b/runner/run_test.go @@ -14,6 +14,7 @@ import ( "text/template" "github.com/coreruleset/ftw-tests-schema/v2/types" + "github.com/google/uuid" "github.com/rs/zerolog/log" "github.com/stretchr/testify/suite" @@ -578,3 +579,62 @@ func (s *runTestSuite) TestIsolatedSanity() { err = RunStage(rc, &check.FTWCheck{}, types.Test{}, stage) s.ErrorContains(err, "'isolated' is only valid if 'expected_ids' has exactly one entry") } + +func (s *runTestSuite) TestVirtualHostMode_Default() { + method := "POST" + input := &test.Input{ + Method: &method, + Headers: ftwhttp.Header{ + "Host": "not-localhost_virtual-host", + }, + DestAddr: &s.dest.DestAddr, + Port: &s.dest.Port, + Protocol: &s.dest.Protocol, + } + context := &TestRunContext{ + Config: config.NewDefaultConfig(), + } + request := buildMarkerRequest(context, input, uuid.NewString()) + + s.Equal("localhost", request.Headers().Get("Host")) +} + +func (s *runTestSuite) TestVirtualHostMode_False() { + method := "POST" + input := &test.Input{ + Method: &method, + Headers: ftwhttp.Header{ + "Host": "not-localhost_virtual-host", + }, + DestAddr: &s.dest.DestAddr, + Port: &s.dest.Port, + Protocol: &s.dest.Protocol, + VirtualHostMode: false, + } + context := &TestRunContext{ + Config: config.NewDefaultConfig(), + } + request := buildMarkerRequest(context, input, uuid.NewString()) + + s.Equal("localhost", request.Headers().Get("Host")) +} + +func (s *runTestSuite) TestVirtualHostMode_True() { + method := "POST" + input := &test.Input{ + Method: &method, + Headers: ftwhttp.Header{ + "Host": "not-localhost_virtual-host", + }, + DestAddr: &s.dest.DestAddr, + Port: &s.dest.Port, + Protocol: &s.dest.Protocol, + VirtualHostMode: true, + } + context := &TestRunContext{ + Config: config.NewDefaultConfig(), + } + request := buildMarkerRequest(context, input, uuid.NewString()) + + s.Equal("not-localhost_virtual-host", request.Headers().Get("Host")) +}