Adding SNI config

.github/workflows/test.yaml

@@ -11,9 +11,9 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Set up Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v4
       with:
-        go-version: 1.18
+        go-version-file: "go.mod"
     - name: Granting private modules access
       run: |
           git config --global url."https://${{ secrets.GH_TOKEN }}:[email protected]/".insteadOf "https://github.com/"
@@ -24,4 +24,4 @@ jobs:
     - name: Send coverage
       env:
         COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: goveralls -coverprofile=profile.cov -service=github
+      run: goveralls -coverprofile=profile.cov -service=github

README.md

@@ -9,3 +9,6 @@ For docs:
 `godoc github.com/getlantern/fronted`
 
 See [ddftool](https://github.com/getlantern/ddftool) for more details on how to generate and tests fronting domains for the supported CDNs.
+
+[!NOTE]
+Since the masquerade domains and IP addresses can change, tests might fail and they need to be updated. You can basically ping some of the masquerade domains (from `default_masquerade.go`) and update the IPs accordingly.

cache_test.go

@@ -2,7 +2,6 @@ package fronted
 
 import (
 	"encoding/json"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -13,7 +12,7 @@ import (
 )
 
 func TestCaching(t *testing.T) {
-	dir, err := ioutil.TempDir("", "direct_test")
+	dir, err := os.MkdirTemp("", "direct_test")
 	if !assert.NoError(t, err, "Unable to create temp dir") {
 		return
 	}
@@ -23,8 +22,8 @@ func TestCaching(t *testing.T) {
 	cloudsackID := "cloudsack"
 
 	providers := map[string]*Provider{
-		testProviderID: NewProvider(nil, "", nil, nil, nil),
-		cloudsackID:    NewProvider(nil, "", nil, nil, nil),
+		testProviderID: NewProvider(nil, "", nil, nil, nil, nil),
+		cloudsackID:    NewProvider(nil, "", nil, nil, nil, nil),
 	}
 
 	makeDirect := func() *direct {
@@ -52,7 +51,7 @@ func TestCaching(t *testing.T) {
 
 	readCached := func() []*masquerade {
 		var result []*masquerade
-		b, err := ioutil.ReadFile(cacheFile)
+		b, err := os.ReadFile(cacheFile)
 		require.NoError(t, err, "Unable to read cache file")
 		err = json.Unmarshal(b, &result)
 		require.NoError(t, err, "Unable to unmarshal cache file")

context.go

@@ -97,7 +97,7 @@ func (fctx *FrontingContext) ConfigureWithHello(pool *x509.CertPool, providers m
 
 	// copy providers
 	for k, p := range providers {
-		d.providers[k] = NewProvider(p.HostAliases, p.TestURL, p.Masquerades, p.Validator, p.PassthroughPatterns)
+		d.providers[k] = NewProvider(p.HostAliases, p.TestURL, p.Masquerades, p.Validator, p.PassthroughPatterns, p.SNIConfig)
 	}
 
 	d.loadCandidates(d.providers)

default_masquerades.go

@@ -21,12 +21,31 @@ var DefaultTrustedCAs = []*CA{
 		CommonName: "USERTrust RSA Certification Authority",
 		Cert:       "-----BEGIN CERTIFICATE-----\nMIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB\niDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl\ncnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV\nBAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw\nMjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV\nBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU\naGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy\ndGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK\nAoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B\n3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY\ntJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/\nFp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2\nVN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT\n79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6\nc0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT\nYo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l\nc6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee\nUB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE\nHg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd\nBgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G\nA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF\nUp/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO\nVWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3\nATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs\n8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR\niQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze\nSf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ\nXHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/\nqS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB\nVXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB\nL6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG\njjxDah2nGN59PRbxYvnKkKj9\n-----END CERTIFICATE-----\n",
 	},
+	{
+		CommonName: "GlobalSign",
+		Cert:       "-----BEGIN CERTIFICATE-----\nMIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G\nA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp\nZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4\nMTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG\nA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI\nhvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8\nRgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT\ngHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm\nKPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd\nQQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ\nXriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw\nDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o\nLkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU\nRUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp\njjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK\n6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX\nmcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs\nMx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH\nWD9f\n-----END CERTIFICATE-----\n",
+	},
+	{
+		CommonName: "ISRG Root X1",
+		Cert:       "-----BEGIN CERTIFICATE-----\nMIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw\nTzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh\ncmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4\nWhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu\nZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY\nMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc\nh77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+\n0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U\nA5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW\nT8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH\nB5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC\nB5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv\nKBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn\nOlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn\njh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw\nqHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI\nrU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV\nHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq\nhkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL\nubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ\n3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK\nNFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5\nORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur\nTkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC\njNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc\noyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq\n4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA\nmRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d\nemyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=\n-----END CERTIFICATE-----\n",
+	},
+}
+
+var DefaultAkamaiMasquerades = []*Masquerade{
+	{
+		Domain:    "a248.e.akamai.net",
+		IpAddress: "23.53.122.84",
+	},
+	{
+		Domain:    "a248.e.akamai.net",
+		IpAddress: "2.19.198.29",
+	},
 }
 
 var DefaultCloudfrontMasquerades = []*Masquerade{
 	{
 		Domain:    "www.amazon.ae",
-		IpAddress: "13.224.6.43",
+		IpAddress: "3.164.6.125",
 	},
 	{
 		Domain:    "cloudfront.net",
@@ -393,8 +412,8 @@ var DefaultCloudfrontMasquerades = []*Masquerade{
 		IpAddress: "13.224.2.94",
 	},
 	{
-		Domain:    "gbf.game-a.mbga.jp",
-		IpAddress: "13.224.0.132",
+		Domain:    "www.amazon.com",
+		IpAddress: "23.199.14.80",
 	},
 	{
 		Domain:    "cloudfront.net",
@@ -450,7 +469,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{
 	},
 	{
 		Domain:    "alexa-comms-mobile-service.amazon.com",
-		IpAddress: "13.224.0.182",
+		IpAddress: "108.139.184.238",
 	},
 	{
 		Domain:    "hkcp08.com",
@@ -722,7 +741,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{
 	},
 	{
 		Domain:    "datadoghq.com",
-		IpAddress: "13.249.5.87",
+		IpAddress: "65.8.214.61",
 	},
 	{
 		Domain:    "demandbase.com",
@@ -782,7 +801,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{
 	},
 	{
 		Domain:    "mobile.mercadopago.com",
-		IpAddress: "99.86.1.210",
+		IpAddress: "108.158.166.197",
 	},
 	{
 		Domain:    "www.awsapps.com",
@@ -940,10 +959,6 @@ var DefaultCloudfrontMasquerades = []*Masquerade{
 		Domain:    "cloudfront.net",
 		IpAddress: "99.84.3.31",
 	},
-	{
-		Domain:    "customerfi.com",
-		IpAddress: "13.224.0.137",
-	},
 	{
 		Domain:    "www.linebc.jp",
 		IpAddress: "54.182.4.177",
@@ -998,11 +1013,11 @@ var DefaultCloudfrontMasquerades = []*Masquerade{
 	},
 	{
 		Domain:    "mercadopago.com",
-		IpAddress: "13.249.6.109",
+		IpAddress: "13.227.126.107",
 	},
 	{
 		Domain:    "www.stg.misumi-ec.com",
-		IpAddress: "99.86.2.175",
+		IpAddress: "52.192.248.133",
 	},
 	{
 		Domain:    "cloudfront.net",
@@ -1014,7 +1029,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{
 	},
 	{
 		Domain:    "www.amazon.sa",
-		IpAddress: "54.239.130.180",
+		IpAddress: "18.67.145.124",
 	},
 	{
 		Domain:    "workflow-stage.licenses.adobe.com",

direct.go

@@ -7,8 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
-	"math/rand"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"net/url"
@@ -65,7 +64,7 @@ func (d *direct) loadCandidates(initial map[string]*Provider) {
 		// ('inside-out' Fisher-Yates)
 		sh := make([]*Masquerade, size)
 		for i := 0; i < size; i++ {
-			j := rand.Intn(i + 1) // 0 <= j <= i
+			j := rand.IntN(i + 1) // 0 <= j <= i
 			sh[i] = sh[j]
 			sh[j] = arr[i]
 		}
@@ -181,7 +180,7 @@ func doCheck(client *http.Client, method string, expectedStatus int, u string) b
 		return false
 	}
 	if resp.Body != nil {
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 	}
 	if resp.StatusCode != expectedStatus {
@@ -219,7 +218,7 @@ func (d *direct) RoundTripHijack(req *http.Request) (*http.Response, net.Conn, e
 	var err error
 	if isIdempotent && req.Body != nil {
 		// store body in-memory to be able to replay it if necessary
-		body, err = ioutil.ReadAll(req.Body)
+		body, err = io.ReadAll(req.Body)
 		if err != nil {
 			err := fmt.Errorf("unable to read request body: %v", err)
 			op.FailIf(err)
@@ -235,7 +234,7 @@ func (d *direct) RoundTripHijack(req *http.Request) (*http.Response, net.Conn, e
 		if !isIdempotent {
 			return req.Body
 		}
-		return ioutil.NopCloser(bytes.NewReader(body))
+		return io.NopCloser(bytes.NewReader(body))
 	}
 
 	tries := 1
@@ -411,10 +410,35 @@ func (d *direct) doDial(m *Masquerade) (conn net.Conn, retriable bool, err error
 }
 
 func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) {
+	op := ops.Begin("dial_server_with")
+	defer op.End()
+
+	op.Set("masquerade_domain", m.Domain)
+	op.Set("masquerade_ip", m.IpAddress)
+
 	tlsConfig := d.frontingTLSConfig(m)
 	dialTimeout := 10 * time.Second
-	sendServerNameExtension := false
 	addr := m.IpAddress
+	var sendServerNameExtension bool
+
+	// looking for provider and using SNI if enabled
+	provider := d.findProviderFromMasquerade(m)
+	if provider != nil && provider.SNIConfig != nil && provider.SNIConfig.UseArbitrarySNIs {
+		sendServerNameExtension = true
+
+		// selecting a random SNI
+		randomSNIIndex := rand.IntN(len(provider.SNIConfig.ArbitrarySNIs))
+		sniDomain := provider.SNIConfig.ArbitrarySNIs[randomSNIIndex]
+
+		op.Set("arbitrary_sni", sniDomain)
+		tlsConfig.ServerName = sniDomain
+		tlsConfig.InsecureSkipVerify = true
+		tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+			log.Tracef("verifying peer certificate for masquerade domain %s", m.Domain)
+			return verifyPeerCertificate(rawCerts, verifiedChains, d.certPool, m.Domain)
+		}
+
+	}
 
 	_, _, err := net.SplitHostPort(addr)
 	if err != nil {
@@ -436,6 +460,50 @@ func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) {
 	return conn, err
 }
 
+func verifyPeerCertificate(rawCerts [][]byte, verifiedChains [][]*x509.Certificate, roots *x509.CertPool, domain string) error {
+	if len(rawCerts) == 0 {
+		return fmt.Errorf("no certificates presented")
+	}
+	cert, err := x509.ParseCertificate(rawCerts[0])
+	if err != nil {
+		return fmt.Errorf("unable to parse certificate: %v", err)
+	}
+
+	masqueradeOpts := x509.VerifyOptions{
+		Roots:         roots,
+		CurrentTime:   time.Now(),
+		DNSName:       domain,
+		Intermediates: x509.NewCertPool(),
+	}
+
+	for i := range rawCerts {
+		if i == 0 {
+			continue
+		}
+		crt, err := x509.ParseCertificate(rawCerts[i])
+		if err != nil {
+			return fmt.Errorf("unable to parse intermediate certificate: %v", err)
+		}
+		masqueradeOpts.Intermediates.AddCert(crt)
+	}
+
+	_, masqueradeErr := cert.Verify(masqueradeOpts)
+	if masqueradeErr != nil {
+		return fmt.Errorf("certificate verification failed for masquerade: %v", masqueradeErr)
+	}
+
+	return nil
+}
+
+func (d *direct) findProviderFromMasquerade(m *Masquerade) *Provider {
+	for _, masquerade := range d.masquerades {
+		if masquerade.Domain == m.Domain && masquerade.IpAddress == m.IpAddress {
+			return d.providers[masquerade.ProviderID]
+		}
+	}
+	return nil
+}
+
 // frontingTLSConfig builds a tls.Config for dialing the fronting domain. This is to establish the
 // initial TCP connection to the CDN.
 func (d *direct) frontingTLSConfig(m *Masquerade) *tls.Config {