Skip to content
This repository has been archived by the owner on Mar 5, 2021. It is now read-only.

Commit

Permalink
added us-proxy.org source
Browse files Browse the repository at this point in the history
  • Loading branch information
@nicksherron
nicksherron committed Feb 3, 2020
1 parent b3b4027 commit 00e3a47
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 27 deletions.
15 changes: 13 additions & 2 deletions internal/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,22 @@ func getKuaidaili(u string) (string, error) {
// DownloadProxies downloads proxies from providers.
func DownloadProxies() Proxies {
log.Println("\rStarting proxy downloads...")
wgD.Add(16)
wgD.Add(17)
var providerProxies Proxies

ctxTimeout := DownloadTimeout
// Download from providers

go func() {
defer wgD.Done()
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout)
defer cancel()
results := usProxyP(ctx)
mutex.Lock()
providerProxies = append(providerProxies, results...)
mutex.Unlock()
}()

go func() {
defer wgD.Done()
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout)
Expand All @@ -174,7 +185,7 @@ func DownloadProxies() Proxies {
defer wgD.Done()
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout)
defer cancel()
results := CheckerproxyP(ctx)
results := checkerproxyP(ctx)
mutex.Lock()
providerProxies = append(providerProxies, results...)
mutex.Unlock()
Expand Down
68 changes: 58 additions & 10 deletions internal/providers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ func webanetlabsP(ctx context.Context) Proxies {
}
}

func CheckerproxyP(ctx context.Context) Proxies {
func checkerproxyP(ctx context.Context) Proxies {

defer ctx.Done()
start := time.Now()
Expand Down Expand Up @@ -709,22 +709,69 @@ func proxylistDownloadP(ctx context.Context) Proxies {
var (
foundProxies Proxies
source = "proxy-list.download"
urls = []string{
"https://www.proxy-list.download/api/v1/get?type=http",
"https://www.proxy-list.download/api/v0/get?l=en&t=http",
"https://www.proxy-list.download/api/v0/get?l=en&t=https",
}
)

done := make(chan bool)
go func() {
body, err := get("https://www.proxy-list.download/api/v1/get?type=http")
if err != nil {
return
for _, u := range urls {
body, err := get(u)
if err != nil {
return
}
for _, proxy := range findAllTemplate(reProxy, body, templateProxy) {
if proxy == "" {
continue
}
p := Proxy{Proxy: proxy, Source: source}
foundProxies = append(foundProxies, &p)
}
done <- true
}
for _, proxy := range findAllTemplate(reProxy, body, templateProxy) {
if proxy == "" {
continue
}()
for {
select {
case <-ctx.Done():
if os.Getenv("PROXI_PROVIDER_DEBUG") == "1" {
fmt.Printf("\n%v\t%v\t%v\n", time.Since(start), source, len(foundProxies))
}
p := Proxy{Proxy: proxy, Source: source}
foundProxies = append(foundProxies, &p)
return foundProxies
case <-done:
if os.Getenv("PROXI_PROVIDER_DEBUG") == "1" {
fmt.Printf("\n%v\t%v\t%v\n", time.Since(start), source, len(foundProxies))
}
return foundProxies
}
done <- true
}
}


func usProxyP(ctx context.Context) Proxies {
defer ctx.Done()
start := time.Now()
var (
foundProxies Proxies
source = "us-proxy.org"
)

done := make(chan bool)
go func() {
body, err := get("https://us-proxy.org/")
if err != nil {
return
}
for _, proxy := range findAllTemplate(reProxy, body, templateProxy) {
if proxy == "" {
continue
}
p := Proxy{Proxy: proxy, Source: source}
foundProxies = append(foundProxies, &p)
}
done <- true
}()
for {
select {
Expand All @@ -742,6 +789,7 @@ func proxylistDownloadP(ctx context.Context) Proxies {
}
}


func blogspotP(ctx context.Context) Proxies {
defer ctx.Done()
start := time.Now()
Expand Down
50 changes: 35 additions & 15 deletions internal/providers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,27 @@ var (

// TODO: Not sure if these are very idiomatic. Maybe use test table for providers instead of separate functions but still perform each test regardless of success.

func TestkuaidailiP(t *testing.T) {

func TestUsProxyP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := usProxyP(ctx)
name := "usProxyP(ctx)"
if len(results) == 0 {
if *testRresults {
t.Errorf("%s didn't return results.", name)
}
return
}
if !re.MatchString(results[0].Proxy) {
t.Errorf("%s sample = %v; expected url pattern matching http://121.139.218.165:31409", name, results[0].Proxy)
} else {
t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results))
}
}


func TestKuaidailiP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := kuaidailiP(ctx)
Expand All @@ -48,7 +68,7 @@ func TestkuaidailiP(t *testing.T) {
}
}

func TestfeiyiproxyP(t *testing.T) {
func TestFeiyiproxyP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := feiyiproxyP(ctx)
Expand All @@ -66,7 +86,7 @@ func TestfeiyiproxyP(t *testing.T) {
}
}

func TestyipP(t *testing.T) {
func TestYipP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := yipP(ctx)
Expand All @@ -84,7 +104,7 @@ func TestyipP(t *testing.T) {
}
}

func Testip3366P(t *testing.T) {
func TestIp3366P(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := ip3366P(ctx)
Expand All @@ -102,7 +122,7 @@ func Testip3366P(t *testing.T) {
}
}

func TestgithubClarketmP(t *testing.T) {
func TestGithubClarketmP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := githubClarketmP(ctx)
Expand All @@ -119,7 +139,7 @@ func TestgithubClarketmP(t *testing.T) {
t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results))
}
}
func TestproxP(t *testing.T) {
func TestProxP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := proxP(ctx)
Expand All @@ -136,7 +156,7 @@ func TestproxP(t *testing.T) {
t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results))
}
}
func TestproxyListP(t *testing.T) {
func TestProxyListP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := proxyListP(ctx)
Expand All @@ -153,7 +173,7 @@ func TestproxyListP(t *testing.T) {
t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results))
}
}
func TestmyProxyP(t *testing.T) {
func TestMyProxyP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := myProxyP(ctx)
Expand All @@ -170,7 +190,7 @@ func TestmyProxyP(t *testing.T) {
t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results))
}
}
func TestxseoP(t *testing.T) {
func TestXseoP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := xseoP(ctx)
Expand All @@ -187,7 +207,7 @@ func TestxseoP(t *testing.T) {
t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results))
}
}
func TestproxylistDownloadP(t *testing.T) {
func TestProxylistDownloadP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := proxylistDownloadP(ctx)
Expand All @@ -204,7 +224,7 @@ func TestproxylistDownloadP(t *testing.T) {
t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results))
}
}
func TestfreeproxylistsP(t *testing.T) {
func TestFreeproxylistsP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := freeproxylistsP(ctx)
Expand All @@ -221,7 +241,7 @@ func TestfreeproxylistsP(t *testing.T) {
t.Logf("%s sample = %v \t found = %v", name, results[0].Proxy, len(results))
}
}
func TestaliveproxyP(t *testing.T) {
func TestAliveproxyP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := aliveproxyP(ctx)
Expand All @@ -239,7 +259,7 @@ func TestaliveproxyP(t *testing.T) {
}
}

func TestblogspotP(t *testing.T) {
func TestBlogspotP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := blogspotP(ctx)
Expand All @@ -257,7 +277,7 @@ func TestblogspotP(t *testing.T) {
}
}

func TestwebanetlabsP(t *testing.T) {
func TestWebanetlabsP(t *testing.T) {
flag.Parse()
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
Expand All @@ -279,7 +299,7 @@ func TestwebanetlabsP(t *testing.T) {
func TestCheckerproxyP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
results := CheckerproxyP(ctx)
results := checkerproxyP(ctx)
name := "CheckerproxyP(ctx)"
if len(results) == 0 {
if *testRresults {
Expand Down

0 comments on commit 00e3a47

Please sign in to comment.