diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a7d05f0..e9fd457 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,9 +11,9 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: - go-version: 1.18 + go-version-file: "go.mod" - name: Granting private modules access run: | git config --global url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github.com/".insteadOf "https://github.com/" @@ -24,4 +24,4 @@ jobs: - name: Send coverage env: COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: goveralls -coverprofile=profile.cov -service=github \ No newline at end of file + run: goveralls -coverprofile=profile.cov -service=github diff --git a/README.md b/README.md index 4614fe1..deacd49 100644 --- a/README.md +++ b/README.md @@ -9,3 +9,6 @@ For docs: `godoc github.com/getlantern/fronted` See [ddftool](https://github.com/getlantern/ddftool) for more details on how to generate and tests fronting domains for the supported CDNs. + +[!NOTE] +Since the masquerade domains and IP addresses can change, tests might fail and they need to be updated. You can basically ping some of the masquerade domains (from `default_masquerade.go`) and update the IPs accordingly. diff --git a/cache_test.go b/cache_test.go index 22f2e45..3c36192 100644 --- a/cache_test.go +++ b/cache_test.go @@ -2,7 +2,6 @@ package fronted import ( "encoding/json" - "io/ioutil" "os" "path/filepath" "testing" @@ -13,7 +12,7 @@ import ( ) func TestCaching(t *testing.T) { - dir, err := ioutil.TempDir("", "direct_test") + dir, err := os.MkdirTemp("", "direct_test") if !assert.NoError(t, err, "Unable to create temp dir") { return } @@ -23,8 +22,8 @@ func TestCaching(t *testing.T) { cloudsackID := "cloudsack" providers := map[string]*Provider{ - testProviderID: NewProvider(nil, "", nil, nil, nil), - cloudsackID: NewProvider(nil, "", nil, nil, nil), + testProviderID: NewProvider(nil, "", nil, nil, nil, nil), + cloudsackID: NewProvider(nil, "", nil, nil, nil, nil), } makeDirect := func() *direct { @@ -52,7 +51,7 @@ func TestCaching(t *testing.T) { readCached := func() []*masquerade { var result []*masquerade - b, err := ioutil.ReadFile(cacheFile) + b, err := os.ReadFile(cacheFile) require.NoError(t, err, "Unable to read cache file") err = json.Unmarshal(b, &result) require.NoError(t, err, "Unable to unmarshal cache file") diff --git a/context.go b/context.go index 6e3df2a..0b08eae 100644 --- a/context.go +++ b/context.go @@ -97,7 +97,7 @@ func (fctx *FrontingContext) ConfigureWithHello(pool *x509.CertPool, providers m // copy providers for k, p := range providers { - d.providers[k] = NewProvider(p.HostAliases, p.TestURL, p.Masquerades, p.Validator, p.PassthroughPatterns) + d.providers[k] = NewProvider(p.HostAliases, p.TestURL, p.Masquerades, p.Validator, p.PassthroughPatterns, p.SNIConfig) } d.loadCandidates(d.providers) diff --git a/default_masquerades.go b/default_masquerades.go index b7fdec8..94611fc 100644 --- a/default_masquerades.go +++ b/default_masquerades.go @@ -21,12 +21,31 @@ var DefaultTrustedCAs = []*CA{ CommonName: "USERTrust RSA Certification Authority", Cert: "-----BEGIN CERTIFICATE-----\nMIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB\niDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl\ncnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV\nBAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw\nMjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV\nBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU\naGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy\ndGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK\nAoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B\n3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY\ntJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/\nFp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2\nVN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT\n79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6\nc0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT\nYo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l\nc6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee\nUB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE\nHg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd\nBgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G\nA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF\nUp/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO\nVWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3\nATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs\n8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR\niQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze\nSf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ\nXHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/\nqS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB\nVXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB\nL6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG\njjxDah2nGN59PRbxYvnKkKj9\n-----END CERTIFICATE-----\n", }, + { + CommonName: "GlobalSign", + Cert: "-----BEGIN CERTIFICATE-----\nMIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G\nA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp\nZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4\nMTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG\nA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI\nhvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8\nRgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT\ngHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm\nKPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd\nQQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ\nXriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw\nDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o\nLkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU\nRUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp\njjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK\n6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX\nmcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs\nMx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH\nWD9f\n-----END CERTIFICATE-----\n", + }, + { + CommonName: "ISRG Root X1", + Cert: "-----BEGIN CERTIFICATE-----\nMIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw\nTzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh\ncmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4\nWhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu\nZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY\nMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc\nh77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+\n0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U\nA5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW\nT8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH\nB5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC\nB5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv\nKBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn\nOlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn\njh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw\nqHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI\nrU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV\nHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq\nhkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL\nubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ\n3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK\nNFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5\nORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur\nTkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC\njNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc\noyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq\n4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA\nmRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d\nemyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=\n-----END CERTIFICATE-----\n", + }, +} + +var DefaultAkamaiMasquerades = []*Masquerade{ + { + Domain: "a248.e.akamai.net", + IpAddress: "23.53.122.84", + }, + { + Domain: "a248.e.akamai.net", + IpAddress: "2.19.198.29", + }, } var DefaultCloudfrontMasquerades = []*Masquerade{ { Domain: "www.amazon.ae", - IpAddress: "13.224.6.43", + IpAddress: "3.164.6.125", }, { Domain: "cloudfront.net", @@ -393,8 +412,8 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ IpAddress: "13.224.2.94", }, { - Domain: "gbf.game-a.mbga.jp", - IpAddress: "13.224.0.132", + Domain: "www.amazon.com", + IpAddress: "23.199.14.80", }, { Domain: "cloudfront.net", @@ -450,7 +469,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ }, { Domain: "alexa-comms-mobile-service.amazon.com", - IpAddress: "13.224.0.182", + IpAddress: "108.139.184.238", }, { Domain: "hkcp08.com", @@ -722,7 +741,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ }, { Domain: "datadoghq.com", - IpAddress: "13.249.5.87", + IpAddress: "65.8.214.61", }, { Domain: "demandbase.com", @@ -782,7 +801,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ }, { Domain: "mobile.mercadopago.com", - IpAddress: "99.86.1.210", + IpAddress: "108.158.166.197", }, { Domain: "www.awsapps.com", @@ -940,10 +959,6 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ Domain: "cloudfront.net", IpAddress: "99.84.3.31", }, - { - Domain: "customerfi.com", - IpAddress: "13.224.0.137", - }, { Domain: "www.linebc.jp", IpAddress: "54.182.4.177", @@ -998,11 +1013,11 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ }, { Domain: "mercadopago.com", - IpAddress: "13.249.6.109", + IpAddress: "13.227.126.107", }, { Domain: "www.stg.misumi-ec.com", - IpAddress: "99.86.2.175", + IpAddress: "52.192.248.133", }, { Domain: "cloudfront.net", @@ -1014,7 +1029,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ }, { Domain: "www.amazon.sa", - IpAddress: "54.239.130.180", + IpAddress: "18.67.145.124", }, { Domain: "workflow-stage.licenses.adobe.com", diff --git a/direct.go b/direct.go index 3ab43c7..de9e330 100644 --- a/direct.go +++ b/direct.go @@ -7,8 +7,7 @@ import ( "errors" "fmt" "io" - "io/ioutil" - "math/rand" + "math/rand/v2" "net" "net/http" "net/url" @@ -65,7 +64,7 @@ func (d *direct) loadCandidates(initial map[string]*Provider) { // ('inside-out' Fisher-Yates) sh := make([]*Masquerade, size) for i := 0; i < size; i++ { - j := rand.Intn(i + 1) // 0 <= j <= i + j := rand.IntN(i + 1) // 0 <= j <= i sh[i] = sh[j] sh[j] = arr[i] } @@ -181,7 +180,7 @@ func doCheck(client *http.Client, method string, expectedStatus int, u string) b return false } if resp.Body != nil { - io.Copy(ioutil.Discard, resp.Body) + io.Copy(io.Discard, resp.Body) resp.Body.Close() } if resp.StatusCode != expectedStatus { @@ -219,7 +218,7 @@ func (d *direct) RoundTripHijack(req *http.Request) (*http.Response, net.Conn, e var err error if isIdempotent && req.Body != nil { // store body in-memory to be able to replay it if necessary - body, err = ioutil.ReadAll(req.Body) + body, err = io.ReadAll(req.Body) if err != nil { err := fmt.Errorf("unable to read request body: %v", err) op.FailIf(err) @@ -235,7 +234,7 @@ func (d *direct) RoundTripHijack(req *http.Request) (*http.Response, net.Conn, e if !isIdempotent { return req.Body } - return ioutil.NopCloser(bytes.NewReader(body)) + return io.NopCloser(bytes.NewReader(body)) } tries := 1 @@ -411,10 +410,29 @@ func (d *direct) doDial(m *Masquerade) (conn net.Conn, retriable bool, err error } func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) { + op := ops.Begin("dial_server_with") + defer op.End() + + op.Set("masquerade_domain", m.Domain) + op.Set("masquerade_ip", m.IpAddress) + tlsConfig := d.frontingTLSConfig(m) dialTimeout := 10 * time.Second - sendServerNameExtension := false addr := m.IpAddress + var sendServerNameExtension bool + + if m.SNI != "" { + sendServerNameExtension = true + + op.Set("arbitrary_sni", m.SNI) + tlsConfig.ServerName = m.SNI + tlsConfig.InsecureSkipVerify = true + tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + log.Tracef("verifying peer certificate for masquerade domain %s", m.Domain) + return verifyPeerCertificate(rawCerts, d.certPool, m.Domain) + } + + } _, _, err := net.SplitHostPort(addr) if err != nil { @@ -436,6 +454,50 @@ func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) { return conn, err } +func verifyPeerCertificate(rawCerts [][]byte, roots *x509.CertPool, domain string) error { + if len(rawCerts) == 0 { + return fmt.Errorf("no certificates presented") + } + cert, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return fmt.Errorf("unable to parse certificate: %w", err) + } + + masqueradeOpts := x509.VerifyOptions{ + Roots: roots, + CurrentTime: time.Now(), + DNSName: domain, + Intermediates: x509.NewCertPool(), + } + + for i := range rawCerts { + if i == 0 { + continue + } + crt, err := x509.ParseCertificate(rawCerts[i]) + if err != nil { + return fmt.Errorf("unable to parse intermediate certificate: %w", err) + } + masqueradeOpts.Intermediates.AddCert(crt) + } + + _, masqueradeErr := cert.Verify(masqueradeOpts) + if masqueradeErr != nil { + return fmt.Errorf("certificate verification failed for masquerade: %w", masqueradeErr) + } + + return nil +} + +func (d *direct) findProviderFromMasquerade(m *Masquerade) *Provider { + for _, masquerade := range d.masquerades { + if masquerade.Domain == m.Domain && masquerade.IpAddress == m.IpAddress { + return d.providers[masquerade.ProviderID] + } + } + return nil +} + // frontingTLSConfig builds a tls.Config for dialing the fronting domain. This is to establish the // initial TCP connection to the CDN. func (d *direct) frontingTLSConfig(m *Masquerade) *tls.Config { diff --git a/direct_test.go b/direct_test.go index 06f2d42..66c1ad2 100644 --- a/direct_test.go +++ b/direct_test.go @@ -4,7 +4,7 @@ import ( "crypto/x509" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "net/http/httputil" @@ -23,7 +23,7 @@ import ( ) func TestDirectDomainFronting(t *testing.T) { - dir, err := ioutil.TempDir("", "direct_test") + dir, err := os.MkdirTemp("", "direct_test") require.NoError(t, err, "Unable to create temp dir") defer os.RemoveAll(dir) cacheFile := filepath.Join(dir, "cachefile.2") @@ -34,11 +34,38 @@ func TestDirectDomainFronting(t *testing.T) { doTestDomainFronting(t, cacheFile, numberToVetInitially) } -func doTestDomainFronting(t *testing.T, cacheFile string, expectedMasqueradesAtEnd int) int { +func TestDirectDomainFrontingWithSNIConfig(t *testing.T) { + dir, err := os.MkdirTemp("", "direct_test") + require.NoError(t, err, "Unable to create temp dir") + defer os.RemoveAll(dir) + cacheFile := filepath.Join(dir, "cachefile.3") + + getURL := "https://config.example.com/global.yaml.gz" + getHost := "config.example.com" + getFrontedHost := "globalconfig.dsa.akamai.getiantem.org" + + hosts := map[string]string{ + getHost: getFrontedHost, + } + certs := trustedCACerts(t) + p := testAkamaiProvidersWithHosts(hosts, &SNIConfig{ + UseArbitrarySNIs: true, + ArbitrarySNIs: []string{"mercadopago.com", "amazon.com.br", "facebook.com", "google.com", "twitter.com", "youtube.com", "instagram.com", "linkedin.com", "whatsapp.com", "netflix.com", "microsoft.com", "yahoo.com", "bing.com", "wikipedia.org", "github.com"}, + }) + Configure(certs, p, testProviderID, cacheFile) - getURL := "http://config.example.com/proxies.yaml.gz" + transport, ok := NewDirect(0) + require.True(t, ok) + client := &http.Client{ + Transport: transport, + } + require.True(t, doCheck(client, http.MethodGet, http.StatusOK, getURL)) +} + +func doTestDomainFronting(t *testing.T, cacheFile string, expectedMasqueradesAtEnd int) int { + getURL := "https://config.example.com/global.yaml.gz" getHost := "config.example.com" - getFrontedHost := "d2wi0vwulmtn99.cloudfront.net" + getFrontedHost := "d24ykmup0867cj.cloudfront.net" pingHost := "ping.example.com" pu, err := url.Parse(pingTestURL) @@ -117,7 +144,7 @@ func TestLoadCandidates(t *testing.T) { actual := make(map[Masquerade]bool) count := 0 for _, m := range d.masquerades { - actual[Masquerade{m.Domain, m.IpAddress}] = true + actual[Masquerade{Domain: m.Domain, IpAddress: m.IpAddress}] = true count++ } @@ -202,7 +229,7 @@ func TestHostAliasesBasic(t *testing.T) { "abc.forbidden.com": "abc.cloudsack.biz", "def.forbidden.com": "def.cloudsack.biz", } - p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, nil) + p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, nil, nil) certs := x509.NewCertPool() certs.AddCert(cloudSack.Certificate()) @@ -232,7 +259,7 @@ func TestHostAliasesBasic(t *testing.T) { } var result CDNResult - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if !assert.NoError(t, err) { continue } @@ -300,14 +327,14 @@ func TestHostAliasesMulti(t *testing.T) { "abc.forbidden.com": "abc.cloudsack.biz", "def.forbidden.com": "def.cloudsack.biz", } - p1 := NewProvider(alias1, "https://ttt.cloudsack.biz/ping", masq1, nil, nil) + p1 := NewProvider(alias1, "https://ttt.cloudsack.biz/ping", masq1, nil, nil, nil) masq2 := []*Masquerade{{Domain: "example.com", IpAddress: sadCloudAddr}} alias2 := map[string]string{ "abc.forbidden.com": "abc.sadcloud.io", "def.forbidden.com": "def.sadcloud.io", } - p2 := NewProvider(alias2, "https://ttt.sadcloud.io/ping", masq2, nil, nil) + p2 := NewProvider(alias2, "https://ttt.sadcloud.io/ping", masq2, nil, nil, nil) certs := x509.NewCertPool() certs.AddCert(cloudSack.Certificate()) @@ -339,7 +366,7 @@ func TestHostAliasesMulti(t *testing.T) { } var result CDNResult - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if !assert.NoError(t, err) { continue } @@ -439,7 +466,7 @@ func TestPassthrough(t *testing.T) { masq := []*Masquerade{{Domain: "example.com", IpAddress: cloudSackAddr}} alias := map[string]string{} passthrough := []string{"*.ok.cloudsack.biz", "abc.cloudsack.biz"} - p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, passthrough) + p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, passthrough, nil) certs := x509.NewCertPool() certs.AddCert(cloudSack.Certificate()) @@ -469,7 +496,7 @@ func TestPassthrough(t *testing.T) { } var result CDNResult - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if !assert.NoError(t, err) { continue } @@ -506,7 +533,7 @@ func TestCustomValidators(t *testing.T) { alias := map[string]string{ "abc.forbidden.com": "abc.sadcloud.io", } - p := NewProvider(alias, "https://ttt.sadcloud.io/ping", masq, validator, nil) + p := NewProvider(alias, "https://ttt.sadcloud.io/ping", masq, validator, nil, nil) certs := x509.NewCertPool() certs.AddCert(sadCloud.Certificate()) @@ -674,7 +701,7 @@ func newCDN(providerID, domain string) (*httptest.Server, string, error) { func corruptMasquerades(cacheFile string) { log.Debug("Corrupting masquerades") - data, err := ioutil.ReadFile(cacheFile) + data, err := os.ReadFile(cacheFile) if err != nil { log.Error(err) return @@ -699,5 +726,5 @@ func corruptMasquerades(cacheFile string) { if err != nil { return } - ioutil.WriteFile(cacheFile, messedUp, 0644) + os.WriteFile(cacheFile, messedUp, 0644) } diff --git a/go.mod b/go.mod index 814b760..fc503af 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/getlantern/fronted -go 1.18 +go 1.22.3 require ( github.com/getlantern/eventual v1.0.0 diff --git a/go.sum b/go.sum index f16868c..bf7c839 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= diff --git a/masquerade.go b/masquerade.go index 041b370..0abade8 100644 --- a/masquerade.go +++ b/masquerade.go @@ -2,6 +2,7 @@ package fronted import ( "fmt" + "hash/crc32" "net" "net/http" "sort" @@ -33,6 +34,9 @@ type Masquerade struct { // IpAddress: pre-resolved ip address to use instead of Domain (if // available) IpAddress string + + // SNI: the SNI to use for this masquerade + SNI string } type masquerade struct { @@ -79,6 +83,11 @@ type Provider struct { // Url used to vet masquerades for this provider TestURL string Masquerades []*Masquerade + + // SNIConfig has the configuration that sets if we should or not use arbitrary SNIs + // and which SNIs to use. + SNIConfig *SNIConfig + // Optional response validator used to determine whether // fronting succeeded for this provider. If the validator // detects a failure for a given masquerade, it is discarded. @@ -86,20 +95,33 @@ type Provider struct { Validator ResponseValidator } +type SNIConfig struct { + UseArbitrarySNIs bool + ArbitrarySNIs []string +} + // Create a Provider with the given details -func NewProvider(hosts map[string]string, testURL string, masquerades []*Masquerade, validator ResponseValidator, passthrough []string) *Provider { +func NewProvider(hosts map[string]string, testURL string, masquerades []*Masquerade, validator ResponseValidator, passthrough []string, sniConfig *SNIConfig) *Provider { d := &Provider{ HostAliases: make(map[string]string), TestURL: testURL, Masquerades: make([]*Masquerade, 0, len(masquerades)), Validator: validator, PassthroughPatterns: make([]string, 0, len(passthrough)), + SNIConfig: sniConfig, } for k, v := range hosts { d.HostAliases[strings.ToLower(k)] = v } + for _, m := range masquerades { - d.Masquerades = append(d.Masquerades, &Masquerade{Domain: m.Domain, IpAddress: m.IpAddress}) + var sni string + if d.SNIConfig != nil && d.SNIConfig.UseArbitrarySNIs { + // Ensure that we use a consistent SNI for a given combination of IP address and SNI set + crc32Hash := int(crc32.ChecksumIEEE([]byte(m.IpAddress))) + sni = d.SNIConfig.ArbitrarySNIs[crc32Hash%len(d.SNIConfig.ArbitrarySNIs)] + } + d.Masquerades = append(d.Masquerades, &Masquerade{Domain: m.Domain, IpAddress: m.IpAddress, SNI: sni}) } d.PassthroughPatterns = append(d.PassthroughPatterns, passthrough...) return d diff --git a/test_support.go b/test_support.go index a1fb8ad..7726ed5 100644 --- a/test_support.go +++ b/test_support.go @@ -9,7 +9,7 @@ import ( var ( testProviderID = "cloudfront" - pingTestURL = "http://d157vud77ygy87.cloudfront.net/ping" + pingTestURL = "https://d157vud77ygy87.cloudfront.net/ping" testHosts = map[string]string(nil) testMasquerades = DefaultCloudfrontMasquerades ) @@ -47,12 +47,17 @@ func trustedCACerts(t *testing.T) *x509.CertPool { func testProviders() map[string]*Provider { return map[string]*Provider{ - testProviderID: NewProvider(testHosts, pingTestURL, testMasquerades, nil, nil), + testProviderID: NewProvider(testHosts, pingTestURL, testMasquerades, nil, nil, nil), } } func testProvidersWithHosts(hosts map[string]string) map[string]*Provider { return map[string]*Provider{ - testProviderID: NewProvider(hosts, pingTestURL, testMasquerades, nil, nil), + testProviderID: NewProvider(hosts, pingTestURL, testMasquerades, nil, nil, nil), + } +} +func testAkamaiProvidersWithHosts(hosts map[string]string, sniConfig *SNIConfig) map[string]*Provider { + return map[string]*Provider{ + testProviderID: NewProvider(hosts, pingTestURL, DefaultAkamaiMasquerades, nil, nil, sniConfig), } }