Skip to content

Commit

Permalink
Merge pull request #54 from lc/lc/fix-commoncrawl
Browse files Browse the repository at this point in the history
fix(gau): change error handling for commoncrawl, bump retry count
  • Loading branch information
lc authored Nov 15, 2021
2 parents beadb20 + 92781a6 commit aa85d75
Show file tree
Hide file tree
Showing 4 changed files with 5 additions and 5 deletions.
2 changes: 1 addition & 1 deletion .gau.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
threads = 2
verbose = false
retries = 5
retries = 15
subdomains = false
providers = ["gau","commoncrawl","otx","urlscan"]
blacklist = ["ttf","woff","svg","png","jpg"]
Expand Down
2 changes: 1 addition & 1 deletion cmd/gau/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func main() {
gau := &runner.Runner{}

if err = gau.Init(config, pMap); err != nil {
log.Fatal(err)
log.Warn(err)
}

results := make(chan string)
Expand Down
2 changes: 1 addition & 1 deletion pkg/providers/providers.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"github.com/valyala/fasthttp"
)

const Version = `2.0.5`
const Version = `2.0.6`

// Provider is a generic interface for all archive fetchers
type Provider interface {
Expand Down
4 changes: 2 additions & 2 deletions runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func (r *Runner) Init(c *providers.Config, providerMap ProvidersMap) error {
case "commoncrawl":
cc, err := commoncrawl.New(c, filters)
if err != nil {
return fmt.Errorf("error instantiating client: %v\n", err)
return fmt.Errorf("error instantiating commoncrawl: %v\n", err)
}
r.providers = append(r.providers, cc)
}
Expand All @@ -51,7 +51,7 @@ func (r *Runner) Init(c *providers.Config, providerMap ProvidersMap) error {

// Starts starts the worker
func (r *Runner) Start(domains chan string, results chan string) {
for i := uint(0); i < r.config.Threads; i++ { // TODO: get thread count dynamically
for i := uint(0); i < r.config.Threads; i++ {
r.wg.Add(1)
go func() {
defer r.wg.Done()
Expand Down

0 comments on commit aa85d75

Please sign in to comment.