From 00e3a47d9f51b3d88e2a1e8722cd302fc0acdb99 Mon Sep 17 00:00:00 2001 From: nicksherron Date: Mon, 3 Feb 2020 11:24:49 -0500 Subject: [PATCH] added us-proxy.org source --- internal/download.go | 15 +++++++-- internal/providers.go | 68 ++++++++++++++++++++++++++++++++------ internal/providers_test.go | 50 +++++++++++++++++++--------- 3 files changed, 106 insertions(+), 27 deletions(-) diff --git a/internal/download.go b/internal/download.go index 7c4f8a9..934321f 100644 --- a/internal/download.go +++ b/internal/download.go @@ -147,11 +147,22 @@ func getKuaidaili(u string) (string, error) { // DownloadProxies downloads proxies from providers. func DownloadProxies() Proxies { log.Println("\rStarting proxy downloads...") - wgD.Add(16) + wgD.Add(17) var providerProxies Proxies ctxTimeout := DownloadTimeout // Download from providers + + go func() { + defer wgD.Done() + ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout) + defer cancel() + results := usProxyP(ctx) + mutex.Lock() + providerProxies = append(providerProxies, results...) + mutex.Unlock() + }() + go func() { defer wgD.Done() ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout) @@ -174,7 +185,7 @@ func DownloadProxies() Proxies { defer wgD.Done() ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout) defer cancel() - results := CheckerproxyP(ctx) + results := checkerproxyP(ctx) mutex.Lock() providerProxies = append(providerProxies, results...) mutex.Unlock() diff --git a/internal/providers.go b/internal/providers.go index 6bc9f6f..b4a1af0 100644 --- a/internal/providers.go +++ b/internal/providers.go @@ -164,7 +164,7 @@ func webanetlabsP(ctx context.Context) Proxies { } } -func CheckerproxyP(ctx context.Context) Proxies { +func checkerproxyP(ctx context.Context) Proxies { defer ctx.Done() start := time.Now() @@ -709,22 +709,69 @@ func proxylistDownloadP(ctx context.Context) Proxies { var ( foundProxies Proxies source = "proxy-list.download" + urls = []string{ + "https://www.proxy-list.download/api/v1/get?type=http", + "https://www.proxy-list.download/api/v0/get?l=en&t=http", + "https://www.proxy-list.download/api/v0/get?l=en&t=https", + } ) done := make(chan bool) go func() { - body, err := get("https://www.proxy-list.download/api/v1/get?type=http") - if err != nil { - return + for _, u := range urls { + body, err := get(u) + if err != nil { + return + } + for _, proxy := range findAllTemplate(reProxy, body, templateProxy) { + if proxy == "" { + continue + } + p := Proxy{Proxy: proxy, Source: source} + foundProxies = append(foundProxies, &p) + } + done <- true } - for _, proxy := range findAllTemplate(reProxy, body, templateProxy) { - if proxy == "" { - continue + }() + for { + select { + case <-ctx.Done(): + if os.Getenv("PROXI_PROVIDER_DEBUG") == "1" { + fmt.Printf("\n%v\t%v\t%v\n", time.Since(start), source, len(foundProxies)) } - p := Proxy{Proxy: proxy, Source: source} - foundProxies = append(foundProxies, &p) + return foundProxies + case <-done: + if os.Getenv("PROXI_PROVIDER_DEBUG") == "1" { + fmt.Printf("\n%v\t%v\t%v\n", time.Since(start), source, len(foundProxies)) + } + return foundProxies } - done <- true + } +} + + +func usProxyP(ctx context.Context) Proxies { + defer ctx.Done() + start := time.Now() + var ( + foundProxies Proxies + source = "us-proxy.org" + ) + + done := make(chan bool) + go func() { + body, err := get("https://us-proxy.org/") + if err != nil { + return + } + for _, proxy := range findAllTemplate(reProxy, body, templateProxy) { + if proxy == "" { + continue + } + p := Proxy{Proxy: proxy, Source: source} + foundProxies = append(foundProxies, &p) + } + done <- true }() for { select { @@ -742,6 +789,7 @@ func proxylistDownloadP(ctx context.Context) Proxies { } } + func blogspotP(ctx context.Context) Proxies { defer ctx.Done() start := time.Now() diff --git a/internal/providers_test.go b/internal/providers_test.go index e1362e2..116332a 100644 --- a/internal/providers_test.go +++ b/internal/providers_test.go @@ -30,7 +30,27 @@ var ( // TODO: Not sure if these are very idiomatic. Maybe use test table for providers instead of separate functions but still perform each test regardless of success. -func TestkuaidailiP(t *testing.T) { + +func TestUsProxyP(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + results := usProxyP(ctx) + name := "usProxyP(ctx)" + if len(results) == 0 { + if *testRresults { + t.Errorf("%s didn't return results.", name) + } + return + } + if !re.MatchString(results[0].Proxy) { + t.Errorf("%s sample = %v; expected url pattern matching http://121.139.218.165:31409", name, results[0].Proxy) + } else { + t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results)) + } +} + + +func TestKuaidailiP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := kuaidailiP(ctx) @@ -48,7 +68,7 @@ func TestkuaidailiP(t *testing.T) { } } -func TestfeiyiproxyP(t *testing.T) { +func TestFeiyiproxyP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := feiyiproxyP(ctx) @@ -66,7 +86,7 @@ func TestfeiyiproxyP(t *testing.T) { } } -func TestyipP(t *testing.T) { +func TestYipP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := yipP(ctx) @@ -84,7 +104,7 @@ func TestyipP(t *testing.T) { } } -func Testip3366P(t *testing.T) { +func TestIp3366P(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := ip3366P(ctx) @@ -102,7 +122,7 @@ func Testip3366P(t *testing.T) { } } -func TestgithubClarketmP(t *testing.T) { +func TestGithubClarketmP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := githubClarketmP(ctx) @@ -119,7 +139,7 @@ func TestgithubClarketmP(t *testing.T) { t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results)) } } -func TestproxP(t *testing.T) { +func TestProxP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := proxP(ctx) @@ -136,7 +156,7 @@ func TestproxP(t *testing.T) { t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results)) } } -func TestproxyListP(t *testing.T) { +func TestProxyListP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := proxyListP(ctx) @@ -153,7 +173,7 @@ func TestproxyListP(t *testing.T) { t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results)) } } -func TestmyProxyP(t *testing.T) { +func TestMyProxyP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := myProxyP(ctx) @@ -170,7 +190,7 @@ func TestmyProxyP(t *testing.T) { t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results)) } } -func TestxseoP(t *testing.T) { +func TestXseoP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := xseoP(ctx) @@ -187,7 +207,7 @@ func TestxseoP(t *testing.T) { t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results)) } } -func TestproxylistDownloadP(t *testing.T) { +func TestProxylistDownloadP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := proxylistDownloadP(ctx) @@ -204,7 +224,7 @@ func TestproxylistDownloadP(t *testing.T) { t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results)) } } -func TestfreeproxylistsP(t *testing.T) { +func TestFreeproxylistsP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := freeproxylistsP(ctx) @@ -221,7 +241,7 @@ func TestfreeproxylistsP(t *testing.T) { t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results)) } } -func TestaliveproxyP(t *testing.T) { +func TestAliveproxyP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := aliveproxyP(ctx) @@ -239,7 +259,7 @@ func TestaliveproxyP(t *testing.T) { } } -func TestblogspotP(t *testing.T) { +func TestBlogspotP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() results := blogspotP(ctx) @@ -257,7 +277,7 @@ func TestblogspotP(t *testing.T) { } } -func TestwebanetlabsP(t *testing.T) { +func TestWebanetlabsP(t *testing.T) { flag.Parse() ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() @@ -279,7 +299,7 @@ func TestwebanetlabsP(t *testing.T) { func TestCheckerproxyP(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() - results := CheckerproxyP(ctx) + results := checkerproxyP(ctx) name := "CheckerproxyP(ctx)" if len(results) == 0 { if *testRresults {