feat: VirtualHostMode determines Host header of internal requests (#…

…370)

By default, internal requests (i.e., requests for marking and flushing
the web server log) use `localhost` for the `Host` header value. In
scenarios, where tests are run against a virtual host this will cause
the internal requests to be processed by the default virtual host while
the test requests will be processed by the targeted virtual host. If the
web server logs are segregated by virtual host, internal and test
requests will end up in different log files and go-ftw will not be able
to find the markers it is looking for.

With `virtual_host_mode: true`, the value of the `Host` header from the
test input will be used for internal requests, to ensure that internal
requests will also be processed by the targeted virtual host.

Fixes #361

go.mod

@@ -7,7 +7,7 @@ toolchain go1.23.3
 require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/corazawaf/coraza/v3 v3.2.2
-	github.com/coreruleset/ftw-tests-schema/v2 v2.1.0
+	github.com/coreruleset/ftw-tests-schema/v2 v2.1.1
 	github.com/creativeprojects/go-selfupdate v1.4.0
 	github.com/go-logr/zerologr v1.2.3
 	github.com/google/uuid v1.6.0
@@ -19,7 +19,7 @@ require (
 	github.com/knadh/koanf/providers/rawbytes v0.1.0
 	github.com/knadh/koanf/v2 v2.1.2
 	github.com/kyokomi/emoji/v2 v2.2.13
-	github.com/magefile/mage bdc92f694516
+	github.com/magefile/mage v1.15.1-0.20231118170541-2385abb49a1f
 	github.com/rs/zerolog v1.33.0
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.10.0
@@ -34,7 +34,6 @@ require (
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.0 // indirect
 	github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect
-	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 )
 

go.sum

@@ -21,8 +21,8 @@ github.com/corazawaf/coraza/v3 v3.2.2/go.mod h1:73JSSNpNrWeF8K+TqKAc7Apxm3uz2rBr
 github.com/corazawaf/libinjection-go v0.2.2 h1:Chzodvb6+NXh6wew5/yhD0Ggioif9ACrQGR4qjTCs1g=
 github.com/corazawaf/libinjection-go v0.2.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreruleset/ftw-tests-schema/v2 v2.1.0 h1:2ilKzKRG5UzzxBcrJLXFtPalStdQ9jzzaYFuFk0OEk0=
-github.com/coreruleset/ftw-tests-schema/v2 v2.1.0/go.mod h1:ZHVFX5ses4+5IxUP0ufCNg/VqRWxziH6ZuUca092Hxo=
+github.com/coreruleset/ftw-tests-schema/v2 v2.1.1 h1:X7dv75NPgPdKtf8zG6ziguGAYogvdAyzxr+152Cgo9g=
+github.com/coreruleset/ftw-tests-schema/v2 v2.1.1/go.mod h1:QEkxQti2T54tDWFPocxgAtLhw6J+zXV44JkqGFFP0is=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creativeprojects/go-selfupdate v1.4.0 h1:4ePPd2CPCNl/YoPXeVxpuBLDUZh8rMEKP5ac+1Y/r5c=
 github.com/creativeprojects/go-selfupdate v1.4.0/go.mod h1:oPG7LmzEmS6OxfqEm620k5VKxP45xFZNKMkp4V5qqUY=
@@ -44,8 +44,6 @@ github.com/go-logr/zerologr v1.2.3 h1:up5N9vcH9Xck3jJkXzgyOxozT14R47IyDODz8LM1KS
 github.com/go-logr/zerologr v1.2.3/go.mod h1:BxwGo7y5zgSHYR1BjbnHPyF/5ZjVKfKxAZANVu6E8Ho=
 github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
 github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
-github.com/goccy/go-yaml v1.9.2 h1:2Njwzw+0+pjU2gb805ZC1B/uBuAs2VcZ3K+ZgHwDs7w=
-github.com/goccy/go-yaml v1.9.2/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -108,8 +106,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO7U=
 github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE=
-github.com/magefile/mage v1.15.1-0.20241124190125-32e01077f0aa h1:RAKYgtUC3OPrwUYKQcwB0wWXCHo09ZWh5TFlnF0nVfA=
-github.com/magefile/mage v1.15.1-0.20241124190125-32e01077f0aa/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/magefile/mage v1.15.1-0.20231118170541-2385abb49a1f h1:iiLWLoibjCL0XND6inF7bs2nc20lU/FYkiR//VIOLUc=
+github.com/magefile/mage v1.15.1-0.20231118170541-2385abb49a1f/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -217,8 +215,6 @@ golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
 golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
-golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=

runner/run.go

@@ -134,7 +134,7 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error {
 //gocyclo:ignore
 func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase schema.Test, stage schema.Stage) error {
 	stageStartTime := time.Now()
-	stageID := uuid.NewString()
+	stageId := uuid.NewString()
 	// Apply global overrides initially
 	testInput := (test.Input)(stage.Input)
 	test.ApplyInputOverrides(runContext.Config, &testInput)
@@ -166,7 +166,7 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch
 	}
 
 	if notRunningInCloudMode(ftwCheck) {
-		startMarker, err := markAndFlush(runContext, dest, stageID)
+		startMarker, err := markAndFlush(runContext, &testInput, stageId)
 		if err != nil && !expectErr {
 			return fmt.Errorf("failed to find start marker: %w", err)
 		}
@@ -190,7 +190,7 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch
 	}
 
 	if notRunningInCloudMode(ftwCheck) {
-		endMarker, err := markAndFlush(runContext, dest, stageID)
+		endMarker, err := markAndFlush(runContext, &testInput, stageId)
 		if err != nil && !expectErr {
 			return fmt.Errorf("failed to find end marker: %w", err)
 
@@ -220,25 +220,13 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch
 	return nil
 }
 
-func markAndFlush(runContext *TestRunContext, dest *ftwhttp.Destination, stageID string) ([]byte, error) {
-	rline := &ftwhttp.RequestLine{
-		Method: "GET",
-		// Use the `/status` endpoint of `httpbin` (http://httpbingo.org), if possible,
-		// to minimize the amount of data transferred and in the log.
-		// `httpbin` is used by the CRS test setup.
-		URI:     "/status/200",
-		Version: "HTTP/1.1",
-	}
-
-	headers := &ftwhttp.Header{
-		"Accept":                              "*/*",
-		"User-Agent":                          "go-ftw test agent",
-		"Host":                                "localhost",
-		runContext.Config.LogMarkerHeaderName: stageID,
+func markAndFlush(runContext *TestRunContext, testInput *test.Input, stageId string) ([]byte, error) {
+	req := buildMarkerRequest(runContext, testInput, stageId)
+	dest := &ftwhttp.Destination{
+		DestAddr: testInput.GetDestAddr(),
+		Port:     testInput.GetPort(),
+		Protocol: testInput.GetProtocol(),
 	}
-
-	req := ftwhttp.NewRequest(rline, *headers, nil, true)
-
 	for i := runContext.Config.MaxMarkerRetries; i > 0; i-- {
 		err := runContext.Client.NewOrReusedConnection(*dest)
 		if err != nil {
@@ -250,14 +238,42 @@ func markAndFlush(runContext *TestRunContext, dest *ftwhttp.Destination, stageID
 			return nil, fmt.Errorf("ftw/run: failed sending request to %+v: %w", dest, err)
 		}
 
-		marker := runContext.LogLines.CheckLogForMarker(stageID, runContext.Config.MaxMarkerLogLines)
+		marker := runContext.LogLines.CheckLogForMarker(stageId, runContext.Config.MaxMarkerLogLines)
 		if marker != nil {
 			return marker, nil
 		}
 	}
 	return nil, fmt.Errorf("can't find log marker. Am I reading the correct log? Log file: %s", runContext.Config.LogFile)
 }
 
+func buildMarkerRequest(runContext *TestRunContext, testInput *test.Input, stageId string) *ftwhttp.Request {
+	host := "localhost"
+	if testInput.VirtualHostMode {
+		// Use the value of the `Host` header of the test for
+		// internal requests as well, so that all requests target
+		// the same virtual host.
+		host = testInput.GetHeaders().Get("Host")
+	}
+
+	headers := &ftwhttp.Header{
+		"Accept":                              "*/*",
+		"User-Agent":                          "go-ftw test agent",
+		"Host":                                host,
+		runContext.Config.LogMarkerHeaderName: stageId,
+	}
+
+	rline := &ftwhttp.RequestLine{
+		Method: "GET",
+		// Use the `/status` endpoint of `httpbin` (http://httpbingo.org), if possible,
+		// to minimize the amount of data transferred and in the log.
+		// `httpbin` is used by the CRS test setup.
+		URI:     "/status/200",
+		Version: "HTTP/1.1",
+	}
+
+	return ftwhttp.NewRequest(rline, *headers, nil, true)
+}
+
 func needToSkipTest(runContext *TestRunContext, testCase *schema.Test) bool {
 	include := runContext.Include
 	exclude := runContext.Exclude

runner/run_test.go

@@ -14,6 +14,7 @@ import (
 	"text/template"
 
 	"github.com/coreruleset/ftw-tests-schema/v2/types"
+	"github.com/google/uuid"
 	"github.com/rs/zerolog/log"
 	"github.com/stretchr/testify/suite"
 
@@ -578,3 +579,62 @@ func (s *runTestSuite) TestIsolatedSanity() {
 	err = RunStage(rc, &check.FTWCheck{}, types.Test{}, stage)
 	s.ErrorContains(err, "'isolated' is only valid if 'expected_ids' has exactly one entry")
 }
+
+func (s *runTestSuite) TestVirtualHostMode_Default() {
+	method := "POST"
+	input := &test.Input{
+		Method: &method,
+		Headers: ftwhttp.Header{
+			"Host": "not-localhost_virtual-host",
+		},
+		DestAddr: &s.dest.DestAddr,
+		Port:     &s.dest.Port,
+		Protocol: &s.dest.Protocol,
+	}
+	context := &TestRunContext{
+		Config: config.NewDefaultConfig(),
+	}
+	request := buildMarkerRequest(context, input, uuid.NewString())
+
+	s.Equal("localhost", request.Headers().Get("Host"))
+}
+
+func (s *runTestSuite) TestVirtualHostMode_False() {
+	method := "POST"
+	input := &test.Input{
+		Method: &method,
+		Headers: ftwhttp.Header{
+			"Host": "not-localhost_virtual-host",
+		},
+		DestAddr:        &s.dest.DestAddr,
+		Port:            &s.dest.Port,
+		Protocol:        &s.dest.Protocol,
+		VirtualHostMode: false,
+	}
+	context := &TestRunContext{
+		Config: config.NewDefaultConfig(),
+	}
+	request := buildMarkerRequest(context, input, uuid.NewString())
+
+	s.Equal("localhost", request.Headers().Get("Host"))
+}
+
+func (s *runTestSuite) TestVirtualHostMode_True() {
+	method := "POST"
+	input := &test.Input{
+		Method: &method,
+		Headers: ftwhttp.Header{
+			"Host": "not-localhost_virtual-host",
+		},
+		DestAddr:        &s.dest.DestAddr,
+		Port:            &s.dest.Port,
+		Protocol:        &s.dest.Protocol,
+		VirtualHostMode: true,
+	}
+	context := &TestRunContext{
+		Config: config.NewDefaultConfig(),
+	}
+	request := buildMarkerRequest(context, input, uuid.NewString())
+
+	s.Equal("not-localhost_virtual-host", request.Headers().Get("Host"))
+}