Merge branch 'main' into myles/resort-masquerades

fronted.go

@@ -257,59 +257,70 @@ func (f *fronted) RoundTripHijack(req *http.Request) (*http.Response, net.Conn,
 			op.FailIf(err)
 			return nil, nil, err
 		}
-		provider := f.providerFor(m)
-		if provider == nil {
-			log.Debugf("Skipping masquerade with disabled/unknown provider '%s'", m.getProviderID())
-			masqueradeGood(false)
-			continue
-		}
-		frontedHost := provider.Lookup(originHost)
-		if frontedHost == "" {
-			// this error is not the masquerade's fault in particular
-			// so it is returned as good.
-			conn.Close()
-			masqueradeGood(true)
-			err := fmt.Errorf("no domain fronting mapping for '%s'. Please add it to provider_map.yaml or equivalent for %s",
-				m.getProviderID(), originHost)
-			op.FailIf(err)
-			return nil, nil, err
-		}
-		log.Debugf("Translated origin %s -> %s for provider %s...", originHost, frontedHost, m.getProviderID())
-
-		reqi, err := cloneRequestWith(req, frontedHost, getBody())
-		if err != nil {
-			return nil, nil, op.FailIf(log.Errorf("Failed to copy http request with origin translated to %v?: %v", frontedHost, err))
-		}
-
-		// don't clobber/confuse Connection header on Upgrade requests.
-		disableKeepAlives := true
-		if strings.EqualFold(reqi.Header.Get("Connection"), "upgrade") {
-			disableKeepAlives = false
-		}
-
-		tr := frontedHTTPTransport(conn, disableKeepAlives)
-		resp, err := tr.RoundTrip(reqi)
-		if err != nil {
-			log.Debugf("Could not complete request: %v", err)
-			masqueradeGood(false)
-			continue
-		}
 
-		err = provider.ValidateResponse(resp)
+		resp, conn, err := f.validateMasqueradeWithConn(req, conn, m, originHost, getBody, masqueradeGood)
 		if err != nil {
 			log.Debugf("Could not complete request: %v", err)
-			resp.Body.Close()
-			masqueradeGood(false)
 			continue
 		}
 
-		masqueradeGood(true)
 		return resp, conn, nil
 	}
 
 	return nil, nil, op.FailIf(errors.New("could not complete request even with retries"))
 }
 
+func (f *fronted) validateMasqueradeWithConn(req *http.Request, conn net.Conn, m MasqueradeInterface, originHost string, getBody func() io.ReadCloser, masqueradeGood func(bool) bool) (*http.Response, net.Conn, error) {
+	op := ops.Begin("validate_masquerade_with_conn")
+	defer op.End()
+	provider := f.providerFor(m)
+	if provider == nil {
+		log.Debugf("Skipping masquerade with disabled/unknown provider '%s'", m.getProviderID())
+		masqueradeGood(false)
+		return nil, nil, op.FailIf(log.Errorf("Skipping masquerade with disabled/unknown provider '%s'", m.getProviderID()))
+	}
+	frontedHost := provider.Lookup(originHost)
+	if frontedHost == "" {
+		// this error is not the masquerade's fault in particular
+		// so it is returned as good.
+		conn.Close()
+		masqueradeGood(true)
+		err := fmt.Errorf("no domain fronting mapping for '%s'. Please add it to provider_map.yaml or equivalent for %s",
+			m.getProviderID(), originHost)
+		op.FailIf(err)
+		return nil, nil, err
+	}
+	log.Debugf("Translated origin %s -> %s for provider %s...", originHost, frontedHost, m.getProviderID())
+
+	reqi, err := cloneRequestWith(req, frontedHost, getBody())
+	if err != nil {
+		return nil, nil, op.FailIf(log.Errorf("Failed to copy http request with origin translated to %v?: %v", frontedHost, err))
+	}
+	disableKeepAlives := true
+	if strings.EqualFold(reqi.Header.Get("Connection"), "upgrade") {
+		disableKeepAlives = false
+	}
+
+	tr := frontedHTTPTransport(conn, disableKeepAlives)
+	resp, err := tr.RoundTrip(reqi)
+	if err != nil {
+		log.Debugf("Could not complete request: %v", err)
+		masqueradeGood(false)
+		return nil, nil, err
+	}
+
+	err = provider.ValidateResponse(resp)
+	if err != nil {
+		log.Debugf("Could not complete request: %v", err)
+		resp.Body.Close()
+		masqueradeGood(false)
+		return nil, nil, err
+	}
+
+	masqueradeGood(true)
+	return resp, conn, nil
+}
+
 // Dial dials out using all available masquerades until one succeeds.
 func (f *fronted) dialAll(ctx context.Context) (net.Conn, MasqueradeInterface, func(bool) bool, error) {
 	defer func(op ops.Op) { op.End() }(ops.Begin("dial_all"))
@@ -515,6 +526,8 @@ type directTransport struct {
 }
 
 func (ddf *directTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
+	defer func(op ops.Op) { op.End() }(ops.Begin("direct_transport_roundtrip"))
+
 	// The connection is already encrypted by domain fronting.  We need to rewrite URLs starting
 	// with "https://" to "http://", lest we get an error for doubling up on TLS.
 

fronted_test.go

@@ -212,15 +212,15 @@ func TestHostAliasesBasic(t *testing.T) {
 	}{
 		{
 			"http://fff.cloudsack.biz/foo",
-			`Get "http://fff.cloudsack.biz/foo": no domain fronting mapping for 'cloudsack'. Please add it to provider_map.yaml or equivalent for fff.cloudsack.biz`,
+			`Get "http://fff.cloudsack.biz/foo": could not complete request even with retries`,
 		},
 		{
 			"http://fff.cloudsack.biz:1234/bar?x=y&z=w",
-			`Get "http://fff.cloudsack.biz:1234/bar?x=y&z=w": no domain fronting mapping for 'cloudsack'. Please add it to provider_map.yaml or equivalent for fff.cloudsack.biz`,
+			`Get "http://fff.cloudsack.biz:1234/bar?x=y&z=w": could not complete request even with retries`,
 		},
 		{
 			"https://www.google.com",
-			`Get "https://www.google.com": no domain fronting mapping for 'cloudsack'. Please add it to provider_map.yaml or equivalent for www.google.com`,
+			`Get "https://www.google.com": could not complete request even with retries`,
 		},
 	}
 
@@ -442,27 +442,27 @@ func TestPassthrough(t *testing.T) {
 	}{
 		{
 			"http://www.notok.cloudsack.biz",
-			`Get "http://www.notok.cloudsack.biz": no domain fronting mapping for 'cloudsack'. Please add it to provider_map.yaml or equivalent for www.notok.cloudsack.biz`,
+			`Get "http://www.notok.cloudsack.biz": could not complete request even with retries`,
 		},
 		{
 			"http://ok.cloudsack.biz",
-			`Get "http://ok.cloudsack.biz": no domain fronting mapping for 'cloudsack'. Please add it to provider_map.yaml or equivalent for ok.cloudsack.biz`,
+			`Get "http://ok.cloudsack.biz": could not complete request even with retries`,
 		},
 		{
 			"http://www.abc.cloudsack.biz",
-			`Get "http://www.abc.cloudsack.biz": no domain fronting mapping for 'cloudsack'. Please add it to provider_map.yaml or equivalent for www.abc.cloudsack.biz`,
+			`Get "http://www.abc.cloudsack.biz": could not complete request even with retries`,
 		},
 		{
 			"http://noabc.cloudsack.biz",
-			`Get "http://noabc.cloudsack.biz": no domain fronting mapping for 'cloudsack'. Please add it to provider_map.yaml or equivalent for noabc.cloudsack.biz`,
+			`Get "http://noabc.cloudsack.biz": could not complete request even with retries`,
 		},
 		{
 			"http://cloudsack.biz",
-			`Get "http://cloudsack.biz": no domain fronting mapping for 'cloudsack'. Please add it to provider_map.yaml or equivalent for cloudsack.biz`,
+			`Get "http://cloudsack.biz": could not complete request even with retries`,
 		},
 		{
 			"https://www.google.com",
-			`Get "https://www.google.com": no domain fronting mapping for 'cloudsack'. Please add it to provider_map.yaml or equivalent for www.google.com`,
+			`Get "https://www.google.com": could not complete request even with retries`,
 		},
 	}
 

go.mod

@@ -10,11 +10,13 @@ require (
 	github.com/getlantern/ops v0.0.0-20230424193308-26325dfed3cf
 	github.com/getlantern/tlsdialer/v3 v3.0.0
 	github.com/getlantern/waitforserver v1.0.1
-	github.com/refraction-networking/utls v0.0.0-20200729012536-186025ac7b77
+	github.com/refraction-networking/utls v1.6.7
 	github.com/stretchr/testify v1.8.2
 )
 
 require (
+	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/getlantern/byteexec v0.0.0-20170405023437-4cfb26ec74f4 // indirect
 	github.com/getlantern/context v0.0.0-20220418194847-3d5e7a086201 // indirect
@@ -27,11 +29,12 @@ require (
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/klauspost/compress v1.17.4 // indirect
 	github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.opentelemetry.io/otel v1.14.0 // indirect
 	go.opentelemetry.io/otel/trace v1.14.0 // indirect
-	golang.org/x/crypto v0.1.0 // indirect
-	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -5,6 +5,8 @@ github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sE
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
+github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks=
 github.com/aristanetworks/glog v0.0.0-20180419172825-c15b03b3054f/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA=
 github.com/aristanetworks/goarista v0.0.0-20191001182449-186a6201b8ef/go.mod h1:Z4RTxGAuYhPzcq8+EdRM+R8M48Ssle2TsWtwRKa+vns=
@@ -13,6 +15,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
+github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -96,6 +100,8 @@ github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -131,8 +137,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/refraction-networking/utls v0.0.0-20190909200633-43c36d3c1f57/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0=
-github.com/refraction-networking/utls v0.0.0-20200729012536-186025ac7b77 h1:f+9aczEfJx9WNE7NMhRmQqPCspsFM+O/XAaiz8E5O1Q=
-github.com/refraction-networking/utls v0.0.0-20200729012536-186025ac7b77/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0=
+github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM=
+github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -165,8 +171,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
-golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -188,8 +194,8 @@ golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=