From 3062fd0c5ea2e219adc7ec91462f032f11763c19 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:39:37 -0300 Subject: [PATCH 01/17] feat: adding SNIConfig to provider and load it --- context.go | 2 +- masquerade.go | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/context.go b/context.go index 6e3df2a..0b08eae 100644 --- a/context.go +++ b/context.go @@ -97,7 +97,7 @@ func (fctx *FrontingContext) ConfigureWithHello(pool *x509.CertPool, providers m // copy providers for k, p := range providers { - d.providers[k] = NewProvider(p.HostAliases, p.TestURL, p.Masquerades, p.Validator, p.PassthroughPatterns) + d.providers[k] = NewProvider(p.HostAliases, p.TestURL, p.Masquerades, p.Validator, p.PassthroughPatterns, p.SNIConfig) } d.loadCandidates(d.providers) diff --git a/masquerade.go b/masquerade.go index 041b370..8231571 100644 --- a/masquerade.go +++ b/masquerade.go @@ -79,6 +79,12 @@ type Provider struct { // Url used to vet masquerades for this provider TestURL string Masquerades []*Masquerade + + // SNIConfig is a map of SNIs with configurations per region. + // The key can be a region initial or the default value, used when + // the region wants to use arbitrary SNIs but doesn't provide the SNI list. + SNIConfig *SNIConfig + // Optional response validator used to determine whether // fronting succeeded for this provider. If the validator // detects a failure for a given masquerade, it is discarded. @@ -86,14 +92,20 @@ type Provider struct { Validator ResponseValidator } +type SNIConfig struct { + UseArbitrarySNIs bool + ArbitrarySNIs []string +} + // Create a Provider with the given details -func NewProvider(hosts map[string]string, testURL string, masquerades []*Masquerade, validator ResponseValidator, passthrough []string) *Provider { +func NewProvider(hosts map[string]string, testURL string, masquerades []*Masquerade, validator ResponseValidator, passthrough []string, sniConfig *SNIConfig) *Provider { d := &Provider{ HostAliases: make(map[string]string), TestURL: testURL, Masquerades: make([]*Masquerade, 0, len(masquerades)), Validator: validator, PassthroughPatterns: make([]string, 0, len(passthrough)), + SNIConfig: sniConfig, } for k, v := range hosts { d.HostAliases[strings.ToLower(k)] = v From 0cebc1631ce29d856dcd63eefab46459ec381d6d Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:40:43 -0300 Subject: [PATCH 02/17] feat: check if there's a SNIConfig available for the provider and use it if there's a enabled SNIConfig; also adding function for verifying the certificate --- direct.go | 52 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/direct.go b/direct.go index 3ab43c7..890876a 100644 --- a/direct.go +++ b/direct.go @@ -8,7 +8,7 @@ import ( "fmt" "io" "io/ioutil" - "math/rand" + "math/rand/v2" "net" "net/http" "net/url" @@ -356,7 +356,7 @@ dialLoop: // We do the full TLS connection here because in practice the domains at a given IP // address can change frequently on CDNs, so the certificate may not match what // we expect. - conn, retriable, err := d.doDial(&m.Masquerade) + conn, retriable, err := d.doDial(m) masqueradeGood := func(good bool) bool { if good { m.markSucceeded() @@ -378,7 +378,7 @@ dialLoop: return nil, nil, nil, errors.New("could not dial any masquerade?") } -func (d *direct) doDial(m *Masquerade) (conn net.Conn, retriable bool, err error) { +func (d *direct) doDial(m *masquerade) (conn net.Conn, retriable bool, err error) { op := ops.Begin("dial_masquerade") defer op.End() op.Set("masquerade_domain", m.Domain) @@ -410,7 +410,7 @@ func (d *direct) doDial(m *Masquerade) (conn net.Conn, retriable bool, err error return } -func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) { +func (d *direct) dialServerWith(m *masquerade) (net.Conn, error) { tlsConfig := d.frontingTLSConfig(m) dialTimeout := 10 * time.Second sendServerNameExtension := false @@ -436,9 +436,51 @@ func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) { return conn, err } +func (d *direct) verifyPeerCertificate(domain string, rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return errors.New("no certificates provided") + } + cert, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return err + } + opts := x509.VerifyOptions{ + Roots: d.certPool, + CurrentTime: time.Now(), + DNSName: domain, + Intermediates: x509.NewCertPool(), + } + for i := 1; i < len(rawCerts); i++ { + intermediate, err := x509.ParseCertificate(rawCerts[i]) + if err != nil { + return err + } + opts.Intermediates.AddCert(intermediate) + } + _, err = cert.Verify(opts) + if err != nil { + return err + } + + return nil +} + // frontingTLSConfig builds a tls.Config for dialing the fronting domain. This is to establish the // initial TCP connection to the CDN. -func (d *direct) frontingTLSConfig(m *Masquerade) *tls.Config { +func (d *direct) frontingTLSConfig(m *masquerade) *tls.Config { + provider := d.providers[m.ProviderID] + if provider.SNIConfig.UseArbitrarySNIs { + randomSNIIndex := rand.IntN(len(provider.SNIConfig.ArbitrarySNIs)) + sniDomain := provider.SNIConfig.ArbitrarySNIs[randomSNIIndex] + return &tls.Config{ + InsecureSkipVerify: true, + VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + return d.verifyPeerCertificate(m.Domain, rawCerts, verifiedChains) + }, + ServerName: sniDomain, + RootCAs: d.certPool, + } + } return &tls.Config{ ServerName: m.Domain, RootCAs: d.certPool, From 4a5c36e6e56bbd30bcefdabdde7508d3c5dea0f9 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:44:38 -0300 Subject: [PATCH 03/17] feat: updating go version to 1.22.3; go mod tidy and updating workflow for using the go version from go.mod --- .github/workflows/test.yaml | 4 ++-- go.mod | 2 +- go.sum | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a7d05f0..927425c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -13,7 +13,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.18 + go-version-file: "go.mod" - name: Granting private modules access run: | git config --global url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github.com/".insteadOf "https://github.com/" @@ -24,4 +24,4 @@ jobs: - name: Send coverage env: COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: goveralls -coverprofile=profile.cov -service=github \ No newline at end of file + run: goveralls -coverprofile=profile.cov -service=github diff --git a/go.mod b/go.mod index 814b760..fc503af 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/getlantern/fronted -go 1.18 +go 1.22.3 require ( github.com/getlantern/eventual v1.0.0 diff --git a/go.sum b/go.sum index f16868c..bf7c839 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= From 547471bb73a8f6875207f66ae1e76f171c331f0c Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:45:15 -0300 Subject: [PATCH 04/17] chore: updating workflow for using actions/setup-go@v4 --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 927425c..e9fd457 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: go-version-file: "go.mod" - name: Granting private modules access From 878cb7b6b04c2302c28d43df0e28c53ef94c3f88 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Wed, 17 Jul 2024 16:00:27 -0300 Subject: [PATCH 05/17] fix: replacing IntN old reference, reverting expected argument type to *Masquerade instead of masquerade and add function for finding provider from a given masquerade --- direct.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/direct.go b/direct.go index 890876a..d9e167b 100644 --- a/direct.go +++ b/direct.go @@ -65,7 +65,7 @@ func (d *direct) loadCandidates(initial map[string]*Provider) { // ('inside-out' Fisher-Yates) sh := make([]*Masquerade, size) for i := 0; i < size; i++ { - j := rand.Intn(i + 1) // 0 <= j <= i + j := rand.IntN(i + 1) // 0 <= j <= i sh[i] = sh[j] sh[j] = arr[i] } @@ -356,7 +356,7 @@ dialLoop: // We do the full TLS connection here because in practice the domains at a given IP // address can change frequently on CDNs, so the certificate may not match what // we expect. - conn, retriable, err := d.doDial(m) + conn, retriable, err := d.doDial(&m.Masquerade) masqueradeGood := func(good bool) bool { if good { m.markSucceeded() @@ -378,7 +378,7 @@ dialLoop: return nil, nil, nil, errors.New("could not dial any masquerade?") } -func (d *direct) doDial(m *masquerade) (conn net.Conn, retriable bool, err error) { +func (d *direct) doDial(m *Masquerade) (conn net.Conn, retriable bool, err error) { op := ops.Begin("dial_masquerade") defer op.End() op.Set("masquerade_domain", m.Domain) @@ -410,7 +410,7 @@ func (d *direct) doDial(m *masquerade) (conn net.Conn, retriable bool, err error return } -func (d *direct) dialServerWith(m *masquerade) (net.Conn, error) { +func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) { tlsConfig := d.frontingTLSConfig(m) dialTimeout := 10 * time.Second sendServerNameExtension := false @@ -465,11 +465,20 @@ func (d *direct) verifyPeerCertificate(domain string, rawCerts [][]byte, _ [][]* return nil } +func (d *direct) findProviderFromMasquerade(m *Masquerade) *Provider { + for _, masquerade := range d.masquerades { + if masquerade.Domain == m.Domain && masquerade.IpAddress == m.IpAddress { + return d.providers[masquerade.ProviderID] + } + } + return nil +} + // frontingTLSConfig builds a tls.Config for dialing the fronting domain. This is to establish the // initial TCP connection to the CDN. -func (d *direct) frontingTLSConfig(m *masquerade) *tls.Config { - provider := d.providers[m.ProviderID] - if provider.SNIConfig.UseArbitrarySNIs { +func (d *direct) frontingTLSConfig(m *Masquerade) *tls.Config { + provider := d.findProviderFromMasquerade(m) + if provider != nil && provider.SNIConfig != nil && provider.SNIConfig.UseArbitrarySNIs { randomSNIIndex := rand.IntN(len(provider.SNIConfig.ArbitrarySNIs)) sniDomain := provider.SNIConfig.ArbitrarySNIs[randomSNIIndex] return &tls.Config{ From bf437da68fef7396214f3c96058d06ec8ee9e0a3 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Wed, 17 Jul 2024 16:12:13 -0300 Subject: [PATCH 06/17] fix: removing deprecated references and updating NewProvider references for using a nil SNIConfig --- cache_test.go | 9 ++++----- direct.go | 7 +++---- direct_test.go | 24 ++++++++++++------------ test_support.go | 4 ++-- 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/cache_test.go b/cache_test.go index 22f2e45..3c36192 100644 --- a/cache_test.go +++ b/cache_test.go @@ -2,7 +2,6 @@ package fronted import ( "encoding/json" - "io/ioutil" "os" "path/filepath" "testing" @@ -13,7 +12,7 @@ import ( ) func TestCaching(t *testing.T) { - dir, err := ioutil.TempDir("", "direct_test") + dir, err := os.MkdirTemp("", "direct_test") if !assert.NoError(t, err, "Unable to create temp dir") { return } @@ -23,8 +22,8 @@ func TestCaching(t *testing.T) { cloudsackID := "cloudsack" providers := map[string]*Provider{ - testProviderID: NewProvider(nil, "", nil, nil, nil), - cloudsackID: NewProvider(nil, "", nil, nil, nil), + testProviderID: NewProvider(nil, "", nil, nil, nil, nil), + cloudsackID: NewProvider(nil, "", nil, nil, nil, nil), } makeDirect := func() *direct { @@ -52,7 +51,7 @@ func TestCaching(t *testing.T) { readCached := func() []*masquerade { var result []*masquerade - b, err := ioutil.ReadFile(cacheFile) + b, err := os.ReadFile(cacheFile) require.NoError(t, err, "Unable to read cache file") err = json.Unmarshal(b, &result) require.NoError(t, err, "Unable to unmarshal cache file") diff --git a/direct.go b/direct.go index d9e167b..fbfe839 100644 --- a/direct.go +++ b/direct.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "math/rand/v2" "net" "net/http" @@ -181,7 +180,7 @@ func doCheck(client *http.Client, method string, expectedStatus int, u string) b return false } if resp.Body != nil { - io.Copy(ioutil.Discard, resp.Body) + io.Copy(io.Discard, resp.Body) resp.Body.Close() } if resp.StatusCode != expectedStatus { @@ -219,7 +218,7 @@ func (d *direct) RoundTripHijack(req *http.Request) (*http.Response, net.Conn, e var err error if isIdempotent && req.Body != nil { // store body in-memory to be able to replay it if necessary - body, err = ioutil.ReadAll(req.Body) + body, err = io.ReadAll(req.Body) if err != nil { err := fmt.Errorf("unable to read request body: %v", err) op.FailIf(err) @@ -235,7 +234,7 @@ func (d *direct) RoundTripHijack(req *http.Request) (*http.Response, net.Conn, e if !isIdempotent { return req.Body } - return ioutil.NopCloser(bytes.NewReader(body)) + return io.NopCloser(bytes.NewReader(body)) } tries := 1 diff --git a/direct_test.go b/direct_test.go index 06f2d42..793f632 100644 --- a/direct_test.go +++ b/direct_test.go @@ -4,7 +4,7 @@ import ( "crypto/x509" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "net/http/httputil" @@ -23,7 +23,7 @@ import ( ) func TestDirectDomainFronting(t *testing.T) { - dir, err := ioutil.TempDir("", "direct_test") + dir, err := os.MkdirTemp("", "direct_test") require.NoError(t, err, "Unable to create temp dir") defer os.RemoveAll(dir) cacheFile := filepath.Join(dir, "cachefile.2") @@ -202,7 +202,7 @@ func TestHostAliasesBasic(t *testing.T) { "abc.forbidden.com": "abc.cloudsack.biz", "def.forbidden.com": "def.cloudsack.biz", } - p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, nil) + p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, nil, nil) certs := x509.NewCertPool() certs.AddCert(cloudSack.Certificate()) @@ -232,7 +232,7 @@ func TestHostAliasesBasic(t *testing.T) { } var result CDNResult - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if !assert.NoError(t, err) { continue } @@ -300,14 +300,14 @@ func TestHostAliasesMulti(t *testing.T) { "abc.forbidden.com": "abc.cloudsack.biz", "def.forbidden.com": "def.cloudsack.biz", } - p1 := NewProvider(alias1, "https://ttt.cloudsack.biz/ping", masq1, nil, nil) + p1 := NewProvider(alias1, "https://ttt.cloudsack.biz/ping", masq1, nil, nil, nil) masq2 := []*Masquerade{{Domain: "example.com", IpAddress: sadCloudAddr}} alias2 := map[string]string{ "abc.forbidden.com": "abc.sadcloud.io", "def.forbidden.com": "def.sadcloud.io", } - p2 := NewProvider(alias2, "https://ttt.sadcloud.io/ping", masq2, nil, nil) + p2 := NewProvider(alias2, "https://ttt.sadcloud.io/ping", masq2, nil, nil, nil) certs := x509.NewCertPool() certs.AddCert(cloudSack.Certificate()) @@ -339,7 +339,7 @@ func TestHostAliasesMulti(t *testing.T) { } var result CDNResult - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if !assert.NoError(t, err) { continue } @@ -439,7 +439,7 @@ func TestPassthrough(t *testing.T) { masq := []*Masquerade{{Domain: "example.com", IpAddress: cloudSackAddr}} alias := map[string]string{} passthrough := []string{"*.ok.cloudsack.biz", "abc.cloudsack.biz"} - p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, passthrough) + p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, passthrough, nil) certs := x509.NewCertPool() certs.AddCert(cloudSack.Certificate()) @@ -469,7 +469,7 @@ func TestPassthrough(t *testing.T) { } var result CDNResult - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if !assert.NoError(t, err) { continue } @@ -506,7 +506,7 @@ func TestCustomValidators(t *testing.T) { alias := map[string]string{ "abc.forbidden.com": "abc.sadcloud.io", } - p := NewProvider(alias, "https://ttt.sadcloud.io/ping", masq, validator, nil) + p := NewProvider(alias, "https://ttt.sadcloud.io/ping", masq, validator, nil, nil) certs := x509.NewCertPool() certs.AddCert(sadCloud.Certificate()) @@ -674,7 +674,7 @@ func newCDN(providerID, domain string) (*httptest.Server, string, error) { func corruptMasquerades(cacheFile string) { log.Debug("Corrupting masquerades") - data, err := ioutil.ReadFile(cacheFile) + data, err := os.ReadFile(cacheFile) if err != nil { log.Error(err) return @@ -699,5 +699,5 @@ func corruptMasquerades(cacheFile string) { if err != nil { return } - ioutil.WriteFile(cacheFile, messedUp, 0644) + os.WriteFile(cacheFile, messedUp, 0644) } diff --git a/test_support.go b/test_support.go index a1fb8ad..122b278 100644 --- a/test_support.go +++ b/test_support.go @@ -47,12 +47,12 @@ func trustedCACerts(t *testing.T) *x509.CertPool { func testProviders() map[string]*Provider { return map[string]*Provider{ - testProviderID: NewProvider(testHosts, pingTestURL, testMasquerades, nil, nil), + testProviderID: NewProvider(testHosts, pingTestURL, testMasquerades, nil, nil, nil), } } func testProvidersWithHosts(hosts map[string]string) map[string]*Provider { return map[string]*Provider{ - testProviderID: NewProvider(hosts, pingTestURL, testMasquerades, nil, nil), + testProviderID: NewProvider(hosts, pingTestURL, testMasquerades, nil, nil, nil), } } From 36cf77f50f45502d1a3ea41b50efe212aa878a6d Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:27:00 -0300 Subject: [PATCH 07/17] chore: updating some old domains --- default_masquerades.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/default_masquerades.go b/default_masquerades.go index b7fdec8..49affe6 100644 --- a/default_masquerades.go +++ b/default_masquerades.go @@ -393,7 +393,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ IpAddress: "13.224.2.94", }, { - Domain: "gbf.game-a.mbga.jp", + Domain: "ambia-onboarding.goaptive.com", IpAddress: "13.224.0.132", }, { @@ -941,7 +941,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ IpAddress: "99.84.3.31", }, { - Domain: "customerfi.com", + Domain: "hansarangcare.com", IpAddress: "13.224.0.137", }, { From df005363ca26a191a44af88a96fad81ad53c66d6 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Mon, 22 Jul 2024 12:50:55 -0300 Subject: [PATCH 08/17] chore: updating default masquerade IPs --- default_masquerades.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/default_masquerades.go b/default_masquerades.go index 49affe6..7d12460 100644 --- a/default_masquerades.go +++ b/default_masquerades.go @@ -26,7 +26,7 @@ var DefaultTrustedCAs = []*CA{ var DefaultCloudfrontMasquerades = []*Masquerade{ { Domain: "www.amazon.ae", - IpAddress: "13.224.6.43", + IpAddress: "3.164.6.125", }, { Domain: "cloudfront.net", @@ -393,8 +393,8 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ IpAddress: "13.224.2.94", }, { - Domain: "ambia-onboarding.goaptive.com", - IpAddress: "13.224.0.132", + Domain: "www.amazon.com", + IpAddress: "23.199.14.80", }, { Domain: "cloudfront.net", @@ -450,7 +450,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ }, { Domain: "alexa-comms-mobile-service.amazon.com", - IpAddress: "13.224.0.182", + IpAddress: "108.139.184.238", }, { Domain: "hkcp08.com", @@ -722,7 +722,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ }, { Domain: "datadoghq.com", - IpAddress: "13.249.5.87", + IpAddress: "65.8.214.61", }, { Domain: "demandbase.com", @@ -782,7 +782,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ }, { Domain: "mobile.mercadopago.com", - IpAddress: "99.86.1.210", + IpAddress: "108.158.166.197", }, { Domain: "www.awsapps.com", @@ -940,10 +940,6 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ Domain: "cloudfront.net", IpAddress: "99.84.3.31", }, - { - Domain: "hansarangcare.com", - IpAddress: "13.224.0.137", - }, { Domain: "www.linebc.jp", IpAddress: "54.182.4.177", @@ -998,11 +994,11 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ }, { Domain: "mercadopago.com", - IpAddress: "13.249.6.109", + IpAddress: "13.227.126.107", }, { Domain: "www.stg.misumi-ec.com", - IpAddress: "99.86.2.175", + IpAddress: "52.192.248.133", }, { Domain: "cloudfront.net", @@ -1014,7 +1010,7 @@ var DefaultCloudfrontMasquerades = []*Masquerade{ }, { Domain: "www.amazon.sa", - IpAddress: "54.239.130.180", + IpAddress: "18.67.145.124", }, { Domain: "workflow-stage.licenses.adobe.com", From c5f6958c4838830c53ec21dfe94128bbeb2e7004 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Mon, 22 Jul 2024 12:52:54 -0300 Subject: [PATCH 09/17] chore: returning custom errors when verifying peer certificates --- direct.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/direct.go b/direct.go index fbfe839..519a4d6 100644 --- a/direct.go +++ b/direct.go @@ -441,7 +441,7 @@ func (d *direct) verifyPeerCertificate(domain string, rawCerts [][]byte, _ [][]* } cert, err := x509.ParseCertificate(rawCerts[0]) if err != nil { - return err + return fmt.Errorf("failed to parse certificate: %v", err) } opts := x509.VerifyOptions{ Roots: d.certPool, @@ -452,13 +452,13 @@ func (d *direct) verifyPeerCertificate(domain string, rawCerts [][]byte, _ [][]* for i := 1; i < len(rawCerts); i++ { intermediate, err := x509.ParseCertificate(rawCerts[i]) if err != nil { - return err + return fmt.Errorf("failed to parse intermediate certificate: %v", err) } opts.Intermediates.AddCert(intermediate) } _, err = cert.Verify(opts) if err != nil { - return err + return fmt.Errorf("failed to verify certificate: %v", err) } return nil From 4b8fdd6b9b590c81a9b39545df683b1fb310adab Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Mon, 22 Jul 2024 12:55:09 -0300 Subject: [PATCH 10/17] fix: updating old test URLs and references so tests can work --- direct_test.go | 5 ++--- test_support.go | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/direct_test.go b/direct_test.go index 793f632..cc927bc 100644 --- a/direct_test.go +++ b/direct_test.go @@ -35,10 +35,9 @@ func TestDirectDomainFronting(t *testing.T) { } func doTestDomainFronting(t *testing.T, cacheFile string, expectedMasqueradesAtEnd int) int { - - getURL := "http://config.example.com/proxies.yaml.gz" + getURL := "https://config.example.com/global.yaml.gz" getHost := "config.example.com" - getFrontedHost := "d2wi0vwulmtn99.cloudfront.net" + getFrontedHost := "d24ykmup0867cj.cloudfront.net" pingHost := "ping.example.com" pu, err := url.Parse(pingTestURL) diff --git a/test_support.go b/test_support.go index 122b278..b3e6e5a 100644 --- a/test_support.go +++ b/test_support.go @@ -9,7 +9,7 @@ import ( var ( testProviderID = "cloudfront" - pingTestURL = "http://d157vud77ygy87.cloudfront.net/ping" + pingTestURL = "https://d157vud77ygy87.cloudfront.net/ping" testHosts = map[string]string(nil) testMasquerades = DefaultCloudfrontMasquerades ) From 9ba01d0002469b819cca22f0fcd67470a3923153 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Tue, 23 Jul 2024 17:05:05 -0300 Subject: [PATCH 11/17] chore: adding more test certificates and akamai default masquerades used for testing purposes --- default_masquerades.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/default_masquerades.go b/default_masquerades.go index 7d12460..94611fc 100644 --- a/default_masquerades.go +++ b/default_masquerades.go @@ -21,6 +21,25 @@ var DefaultTrustedCAs = []*CA{ CommonName: "USERTrust RSA Certification Authority", Cert: "-----BEGIN CERTIFICATE-----\nMIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB\niDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl\ncnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV\nBAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw\nMjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV\nBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU\naGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy\ndGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK\nAoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B\n3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY\ntJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/\nFp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2\nVN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT\n79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6\nc0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT\nYo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l\nc6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee\nUB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE\nHg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd\nBgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G\nA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF\nUp/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO\nVWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3\nATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs\n8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR\niQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze\nSf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ\nXHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/\nqS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB\nVXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB\nL6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG\njjxDah2nGN59PRbxYvnKkKj9\n-----END CERTIFICATE-----\n", }, + { + CommonName: "GlobalSign", + Cert: "-----BEGIN CERTIFICATE-----\nMIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G\nA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp\nZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4\nMTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG\nA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI\nhvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8\nRgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT\ngHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm\nKPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd\nQQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ\nXriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw\nDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o\nLkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU\nRUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp\njjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK\n6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX\nmcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs\nMx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH\nWD9f\n-----END CERTIFICATE-----\n", + }, + { + CommonName: "ISRG Root X1", + Cert: "-----BEGIN CERTIFICATE-----\nMIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw\nTzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh\ncmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4\nWhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu\nZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY\nMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc\nh77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+\n0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U\nA5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW\nT8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH\nB5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC\nB5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv\nKBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn\nOlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn\njh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw\nqHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI\nrU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV\nHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq\nhkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL\nubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ\n3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK\nNFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5\nORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur\nTkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC\njNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc\noyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq\n4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA\nmRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d\nemyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=\n-----END CERTIFICATE-----\n", + }, +} + +var DefaultAkamaiMasquerades = []*Masquerade{ + { + Domain: "a248.e.akamai.net", + IpAddress: "23.53.122.84", + }, + { + Domain: "a248.e.akamai.net", + IpAddress: "2.19.198.29", + }, } var DefaultCloudfrontMasquerades = []*Masquerade{ From b26777e5d90c427035a16ce3ddb7bf4083cf268b Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Tue, 23 Jul 2024 17:15:55 -0300 Subject: [PATCH 12/17] feat: adding feat for sending the provided SNI verify the domain if it maches with masquerade --- direct.go | 70 +++++++++++++++++++++++++++++++------------------ direct_test.go | 29 ++++++++++++++++++++ test_support.go | 5 ++++ 3 files changed, 78 insertions(+), 26 deletions(-) diff --git a/direct.go b/direct.go index 519a4d6..0eac8f2 100644 --- a/direct.go +++ b/direct.go @@ -410,10 +410,35 @@ func (d *direct) doDial(m *Masquerade) (conn net.Conn, retriable bool, err error } func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) { + op := ops.Begin("dial_server_with") + defer op.End() + + op.Set("masquerade_domain", m.Domain) + op.Set("masquerade_ip", m.IpAddress) + tlsConfig := d.frontingTLSConfig(m) dialTimeout := 10 * time.Second - sendServerNameExtension := false addr := m.IpAddress + var sendServerNameExtension bool + + // looking for provider and using SNI if enabled + provider := d.findProviderFromMasquerade(m) + if provider != nil && provider.SNIConfig != nil && provider.SNIConfig.UseArbitrarySNIs { + sendServerNameExtension = true + + // selecting a random SNI + randomSNIIndex := rand.IntN(len(provider.SNIConfig.ArbitrarySNIs)) + sniDomain := provider.SNIConfig.ArbitrarySNIs[randomSNIIndex] + + op.Set("arbitrary_sni", sniDomain) + tlsConfig.ServerName = sniDomain + tlsConfig.InsecureSkipVerify = true + tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + log.Tracef("verifying peer certificate for masquerade domain %s", m.Domain) + return verifyPeerCertificate(rawCerts, verifiedChains, d.certPool, m.Domain) + } + + } _, _, err := net.SplitHostPort(addr) if err != nil { @@ -435,30 +460,36 @@ func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) { return conn, err } -func (d *direct) verifyPeerCertificate(domain string, rawCerts [][]byte, _ [][]*x509.Certificate) error { +func verifyPeerCertificate(rawCerts [][]byte, verifiedChains [][]*x509.Certificate, roots *x509.CertPool, domain string) error { if len(rawCerts) == 0 { - return errors.New("no certificates provided") + return fmt.Errorf("no certificates presented") } cert, err := x509.ParseCertificate(rawCerts[0]) if err != nil { - return fmt.Errorf("failed to parse certificate: %v", err) + return fmt.Errorf("unable to parse certificate: %v", err) } - opts := x509.VerifyOptions{ - Roots: d.certPool, + + masqueradeOpts := x509.VerifyOptions{ + Roots: roots, CurrentTime: time.Now(), DNSName: domain, Intermediates: x509.NewCertPool(), } - for i := 1; i < len(rawCerts); i++ { - intermediate, err := x509.ParseCertificate(rawCerts[i]) + + for i := range rawCerts { + if i == 0 { + continue + } + crt, err := x509.ParseCertificate(rawCerts[i]) if err != nil { - return fmt.Errorf("failed to parse intermediate certificate: %v", err) + return fmt.Errorf("unable to parse intermediate certificate: %v", err) } - opts.Intermediates.AddCert(intermediate) + masqueradeOpts.Intermediates.AddCert(crt) } - _, err = cert.Verify(opts) - if err != nil { - return fmt.Errorf("failed to verify certificate: %v", err) + + _, masqueradeErr := cert.Verify(masqueradeOpts) + if masqueradeErr != nil { + return fmt.Errorf("certificate verification failed for masquerade: %v", masqueradeErr) } return nil @@ -476,19 +507,6 @@ func (d *direct) findProviderFromMasquerade(m *Masquerade) *Provider { // frontingTLSConfig builds a tls.Config for dialing the fronting domain. This is to establish the // initial TCP connection to the CDN. func (d *direct) frontingTLSConfig(m *Masquerade) *tls.Config { - provider := d.findProviderFromMasquerade(m) - if provider != nil && provider.SNIConfig != nil && provider.SNIConfig.UseArbitrarySNIs { - randomSNIIndex := rand.IntN(len(provider.SNIConfig.ArbitrarySNIs)) - sniDomain := provider.SNIConfig.ArbitrarySNIs[randomSNIIndex] - return &tls.Config{ - InsecureSkipVerify: true, - VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { - return d.verifyPeerCertificate(m.Domain, rawCerts, verifiedChains) - }, - ServerName: sniDomain, - RootCAs: d.certPool, - } - } return &tls.Config{ ServerName: m.Domain, RootCAs: d.certPool, diff --git a/direct_test.go b/direct_test.go index cc927bc..657d9c7 100644 --- a/direct_test.go +++ b/direct_test.go @@ -34,6 +34,35 @@ func TestDirectDomainFronting(t *testing.T) { doTestDomainFronting(t, cacheFile, numberToVetInitially) } +func TestDirectDomainFrontingWithSNIConfig(t *testing.T) { + dir, err := os.MkdirTemp("", "direct_test") + require.NoError(t, err, "Unable to create temp dir") + defer os.RemoveAll(dir) + cacheFile := filepath.Join(dir, "cachefile.3") + + getURL := "https://config.example.com/global.yaml.gz" + getHost := "config.example.com" + getFrontedHost := "globalconfig.dsa.akamai.getiantem.org" + + hosts := map[string]string{ + getHost: getFrontedHost, + } + certs := trustedCACerts(t) + p := testAkamaiProvidersWithHosts(hosts, &SNIConfig{ + UseArbitrarySNIs: true, + ArbitrarySNIs: []string{"mercadopago.com", "amazon.com.br", "facebook.com", "google.com", "twitter.com", "youtube.com", "instagram.com", "linkedin.com", "whatsapp.com", "netflix.com", "microsoft.com", "yahoo.com", "bing.com", "wikipedia.org", "github.com"}, + }) + Configure(certs, p, testProviderID, cacheFile) + + transport, ok := NewDirect(0) + require.True(t, ok) + client := &http.Client{ + Transport: transport, + } + require.True(t, doCheck(client, http.MethodGet, http.StatusOK, getURL)) + t.Logf("SNIConfig test passed") +} + func doTestDomainFronting(t *testing.T, cacheFile string, expectedMasqueradesAtEnd int) int { getURL := "https://config.example.com/global.yaml.gz" getHost := "config.example.com" diff --git a/test_support.go b/test_support.go index b3e6e5a..7726ed5 100644 --- a/test_support.go +++ b/test_support.go @@ -56,3 +56,8 @@ func testProvidersWithHosts(hosts map[string]string) map[string]*Provider { testProviderID: NewProvider(hosts, pingTestURL, testMasquerades, nil, nil, nil), } } +func testAkamaiProvidersWithHosts(hosts map[string]string, sniConfig *SNIConfig) map[string]*Provider { + return map[string]*Provider{ + testProviderID: NewProvider(hosts, pingTestURL, DefaultAkamaiMasquerades, nil, nil, sniConfig), + } +} From 0e2046586a71c39e20046896c74536f1c89c55f3 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Tue, 23 Jul 2024 17:38:17 -0300 Subject: [PATCH 13/17] chore: adding note about default_masquerades.go with old IP addresses --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 4614fe1..deacd49 100644 --- a/README.md +++ b/README.md @@ -9,3 +9,6 @@ For docs: `godoc github.com/getlantern/fronted` See [ddftool](https://github.com/getlantern/ddftool) for more details on how to generate and tests fronting domains for the supported CDNs. + +[!NOTE] +Since the masquerade domains and IP addresses can change, tests might fail and they need to be updated. You can basically ping some of the masquerade domains (from `default_masquerade.go`) and update the IPs accordingly. From 1385d8cd4e7146b10d67c0003c6a7d15dee7111e Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Fri, 26 Jul 2024 10:57:16 -0300 Subject: [PATCH 14/17] fix: updating SNIConfig comment --- masquerade.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/masquerade.go b/masquerade.go index 8231571..98cc541 100644 --- a/masquerade.go +++ b/masquerade.go @@ -80,9 +80,8 @@ type Provider struct { TestURL string Masquerades []*Masquerade - // SNIConfig is a map of SNIs with configurations per region. - // The key can be a region initial or the default value, used when - // the region wants to use arbitrary SNIs but doesn't provide the SNI list. + // SNIConfig has the configuration that sets if we should or not use arbitrary SNIs + // and which SNIs to use. SNIConfig *SNIConfig // Optional response validator used to determine whether From 5ecdafa2de7529d9c58ed20f500d610a83ad3d34 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Fri, 26 Jul 2024 12:11:38 -0300 Subject: [PATCH 15/17] fix: hashing IP addresses and setting SNI to masquerades --- direct.go | 12 +++--------- direct_test.go | 2 +- masquerade.go | 12 +++++++++++- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/direct.go b/direct.go index 0eac8f2..4357bec 100644 --- a/direct.go +++ b/direct.go @@ -421,17 +421,11 @@ func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) { addr := m.IpAddress var sendServerNameExtension bool - // looking for provider and using SNI if enabled - provider := d.findProviderFromMasquerade(m) - if provider != nil && provider.SNIConfig != nil && provider.SNIConfig.UseArbitrarySNIs { + if m.SNI != "" { sendServerNameExtension = true - // selecting a random SNI - randomSNIIndex := rand.IntN(len(provider.SNIConfig.ArbitrarySNIs)) - sniDomain := provider.SNIConfig.ArbitrarySNIs[randomSNIIndex] - - op.Set("arbitrary_sni", sniDomain) - tlsConfig.ServerName = sniDomain + op.Set("arbitrary_sni", m.SNI) + tlsConfig.ServerName = m.SNI tlsConfig.InsecureSkipVerify = true tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { log.Tracef("verifying peer certificate for masquerade domain %s", m.Domain) diff --git a/direct_test.go b/direct_test.go index 657d9c7..619307a 100644 --- a/direct_test.go +++ b/direct_test.go @@ -145,7 +145,7 @@ func TestLoadCandidates(t *testing.T) { actual := make(map[Masquerade]bool) count := 0 for _, m := range d.masquerades { - actual[Masquerade{m.Domain, m.IpAddress}] = true + actual[Masquerade{Domain: m.Domain, IpAddress: m.IpAddress}] = true count++ } diff --git a/masquerade.go b/masquerade.go index 98cc541..2dceda6 100644 --- a/masquerade.go +++ b/masquerade.go @@ -2,6 +2,7 @@ package fronted import ( "fmt" + "hash/crc32" "net" "net/http" "sort" @@ -33,6 +34,9 @@ type Masquerade struct { // IpAddress: pre-resolved ip address to use instead of Domain (if // available) IpAddress string + + // SNI: the SNI to use for this masquerade + SNI string } type masquerade struct { @@ -109,8 +113,14 @@ func NewProvider(hosts map[string]string, testURL string, masquerades []*Masquer for k, v := range hosts { d.HostAliases[strings.ToLower(k)] = v } + for _, m := range masquerades { - d.Masquerades = append(d.Masquerades, &Masquerade{Domain: m.Domain, IpAddress: m.IpAddress}) + var sni string + if d.SNIConfig != nil && d.SNIConfig.UseArbitrarySNIs { + crc32Hash := int(crc32.ChecksumIEEE([]byte(m.IpAddress))) + sni = d.SNIConfig.ArbitrarySNIs[crc32Hash%len(d.SNIConfig.ArbitrarySNIs)] + } + d.Masquerades = append(d.Masquerades, &Masquerade{Domain: m.Domain, IpAddress: m.IpAddress, SNI: sni}) } d.PassthroughPatterns = append(d.PassthroughPatterns, passthrough...) return d From 58b31713fcbc8723a769b3c217d2e04f63e0bc75 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Mon, 29 Jul 2024 11:59:00 -0300 Subject: [PATCH 16/17] fix: removing unused verifiedChains parameter, replacing fmt errors %v by %w and unused test log --- direct.go | 12 ++++++------ direct_test.go | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/direct.go b/direct.go index 4357bec..de9e330 100644 --- a/direct.go +++ b/direct.go @@ -427,9 +427,9 @@ func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) { op.Set("arbitrary_sni", m.SNI) tlsConfig.ServerName = m.SNI tlsConfig.InsecureSkipVerify = true - tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error { log.Tracef("verifying peer certificate for masquerade domain %s", m.Domain) - return verifyPeerCertificate(rawCerts, verifiedChains, d.certPool, m.Domain) + return verifyPeerCertificate(rawCerts, d.certPool, m.Domain) } } @@ -454,13 +454,13 @@ func (d *direct) dialServerWith(m *Masquerade) (net.Conn, error) { return conn, err } -func verifyPeerCertificate(rawCerts [][]byte, verifiedChains [][]*x509.Certificate, roots *x509.CertPool, domain string) error { +func verifyPeerCertificate(rawCerts [][]byte, roots *x509.CertPool, domain string) error { if len(rawCerts) == 0 { return fmt.Errorf("no certificates presented") } cert, err := x509.ParseCertificate(rawCerts[0]) if err != nil { - return fmt.Errorf("unable to parse certificate: %v", err) + return fmt.Errorf("unable to parse certificate: %w", err) } masqueradeOpts := x509.VerifyOptions{ @@ -476,14 +476,14 @@ func verifyPeerCertificate(rawCerts [][]byte, verifiedChains [][]*x509.Certifica } crt, err := x509.ParseCertificate(rawCerts[i]) if err != nil { - return fmt.Errorf("unable to parse intermediate certificate: %v", err) + return fmt.Errorf("unable to parse intermediate certificate: %w", err) } masqueradeOpts.Intermediates.AddCert(crt) } _, masqueradeErr := cert.Verify(masqueradeOpts) if masqueradeErr != nil { - return fmt.Errorf("certificate verification failed for masquerade: %v", masqueradeErr) + return fmt.Errorf("certificate verification failed for masquerade: %w", masqueradeErr) } return nil diff --git a/direct_test.go b/direct_test.go index 619307a..66c1ad2 100644 --- a/direct_test.go +++ b/direct_test.go @@ -60,7 +60,6 @@ func TestDirectDomainFrontingWithSNIConfig(t *testing.T) { Transport: transport, } require.True(t, doCheck(client, http.MethodGet, http.StatusOK, getURL)) - t.Logf("SNIConfig test passed") } func doTestDomainFronting(t *testing.T, cacheFile string, expectedMasqueradesAtEnd int) int { From 3da002e5cd71890e42ac5210bc7f08bcfaa8fdb9 Mon Sep 17 00:00:00 2001 From: WendelHime <6754291+WendelHime@users.noreply.github.com> Date: Tue, 30 Jul 2024 10:35:28 -0300 Subject: [PATCH 17/17] fix: adding comment suggestion explaining about ensuring the use of hashing for consistently retrieving a SNI for the masquerade IP address --- masquerade.go | 1 + 1 file changed, 1 insertion(+) diff --git a/masquerade.go b/masquerade.go index 2dceda6..0abade8 100644 --- a/masquerade.go +++ b/masquerade.go @@ -117,6 +117,7 @@ func NewProvider(hosts map[string]string, testURL string, masquerades []*Masquer for _, m := range masquerades { var sni string if d.SNIConfig != nil && d.SNIConfig.UseArbitrarySNIs { + // Ensure that we use a consistent SNI for a given combination of IP address and SNI set crc32Hash := int(crc32.ChecksumIEEE([]byte(m.IpAddress))) sni = d.SNIConfig.ArbitrarySNIs[crc32Hash%len(d.SNIConfig.ArbitrarySNIs)] }