From b4bcd8cf5cd3226dc1dc3057492e94f7c442d9c0 Mon Sep 17 00:00:00 2001 From: g3ol4d0 Date: Fri, 11 Feb 2022 15:05:04 -0300 Subject: [PATCH 1/3] Adding new option --- .gau.toml | 1 + README.md | 1 + cmd/gau/main.go | 9 +++++++-- pkg/output/output.go | 18 ++++++++++++++++-- pkg/providers/providers.go | 1 + runner/flags/flags.go | 9 +++++++++ 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/.gau.toml b/.gau.toml index 0b4847d..fc06ea8 100644 --- a/.gau.toml +++ b/.gau.toml @@ -2,6 +2,7 @@ threads = 2 verbose = false retries = 15 subdomains = false +parameters = false providers = ["gau","commoncrawl","otx","urlscan"] blacklist = ["ttf","woff","svg","png","jpg"] json = false diff --git a/README.md b/README.md index e591b16..ef89327 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ $ gau -h |`--fc`| list of status codes to filter | gau --fc 404,302 | |`--from`| fetch urls from date (format: YYYYMM) | gau --from 202101 | |`--ft`| list of mime-types to filter | gau --ft text/plain| +|`--fp`| remove different parameters of the same endpoint | gau --fp| |`--json`| output as json | gau --json | |`--mc`| list of status codes to match | gau --mc 200,500 | |`--mt`| list of mime-types to match |gau --mt text/html,application/json| diff --git a/cmd/gau/main.go b/cmd/gau/main.go index 2344b80..f84db6a 100644 --- a/cmd/gau/main.go +++ b/cmd/gau/main.go @@ -8,6 +8,8 @@ import ( log "github.com/sirupsen/logrus" "io" "os" + "net/http" + _ "net/http/pprof" "sync" ) @@ -20,6 +22,9 @@ func main() { } } + + log.Println(http.ListenAndServe("localhost:8989", nil)) + pMap := make(runner.ProvidersMap) for _, provider := range cfg.Providers { pMap[provider] = cfg.Filters @@ -56,12 +61,12 @@ func main() { if config.JSON { go func() { defer writeWg.Done() - output.WriteURLsJSON(out, results, config.Blacklist) + output.WriteURLsJSON(out, results, config.Blacklist, config.RemoveParameters) }() } else { go func() { defer writeWg.Done() - if err = output.WriteURLs(out, results, config.Blacklist); err != nil { + if err = output.WriteURLs(out, results, config.Blacklist, config.RemoveParameters); err != nil { log.Fatalf("error writing results: %v\n", err) } }() diff --git a/pkg/output/output.go b/pkg/output/output.go index 6a9b852..190faad 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -13,7 +13,8 @@ type JSONResult struct { Url string `json:"url"` } -func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}) error { +func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}, RemoveParameters bool) error { + lastURL := make(map[string]bool) for result := range results { buf := bytebufferpool.Get() if len(blacklistMap) != 0 { @@ -30,6 +31,19 @@ func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string] } } } + if RemoveParameters { + u, err := url.Parse(result) + if err != nil { + continue + } + if lastURL[u.Host+u.Path] { + continue + } else { + lastURL[u.Host+u.Path] = true ; + } + + } + buf.B = append(buf.B, []byte(result)...) buf.B = append(buf.B, "\n"...) _, err := writer.Write(buf.B) @@ -41,7 +55,7 @@ func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string] return nil } -func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}) { +func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}, RemoveParameters bool) { var jr JSONResult enc := jsoniter.NewEncoder(writer) for result := range results { diff --git a/pkg/providers/providers.go b/pkg/providers/providers.go index 2cedf1a..f3d1471 100644 --- a/pkg/providers/providers.go +++ b/pkg/providers/providers.go @@ -23,6 +23,7 @@ type Config struct { Verbose bool MaxRetries uint IncludeSubdomains bool + RemoveParameters bool Client *fasthttp.Client Providers []string Blacklist map[string]struct{} diff --git a/runner/flags/flags.go b/runner/flags/flags.go index 8068a1b..0d1eeec 100644 --- a/runner/flags/flags.go +++ b/runner/flags/flags.go @@ -29,6 +29,7 @@ type Config struct { Verbose bool `mapstructure:"verbose"` MaxRetries uint `mapstructure:"retries"` IncludeSubdomains bool `mapstructure:"subdomains"` + RemoveParameters bool `mapstructure:"parameters"` Providers []string `mapstructure:"providers"` Blacklist []string `mapstructure:"blacklist"` JSON bool `mapstructure:"json"` @@ -60,6 +61,7 @@ func (c *Config) ProviderConfig() (*providers.Config, error) { Verbose: c.Verbose, MaxRetries: c.MaxRetries, IncludeSubdomains: c.IncludeSubdomains, + RemoveParameters: c.RemoveParameters, Client: &fasthttp.Client{ TLSConfig: &tls.Config{ InsecureSkipVerify: true, @@ -98,6 +100,7 @@ func New() *Options { pflag.StringSlice("blacklist", []string{}, "list of extensions to skip") pflag.StringSlice("providers", []string{}, "list of providers to use (wayback,commoncrawl,otx,urlscan)") pflag.Bool("subs", false, "include subdomains of target domain") + pflag.Bool("fp", false, "remove different parameters of the same endpoint") pflag.Bool("verbose", false, "show verbose output") pflag.Bool("json", false, "output as json") @@ -160,6 +163,7 @@ func (o *Options) DefaultConfig() *Config { Verbose: false, MaxRetries: 5, IncludeSubdomains: false, + RemoveParameters: false, Providers: []string{"wayback", "commoncrawl", "otx", "urlscan"}, Blacklist: []string{}, JSON: false, @@ -182,6 +186,7 @@ func (o *Options) getFlagValues(c *Config) { threads := o.viper.GetUint("threads") blacklist := o.viper.GetStringSlice("blacklist") subs := o.viper.GetBool("subs") + fp := o.viper.GetBool("fp") if version { fmt.Printf("gau version: %s\n", providers.Version) @@ -218,6 +223,10 @@ func (o *Options) getFlagValues(c *Config) { c.IncludeSubdomains = subs } + if fp { + c.RemoveParameters = fp + } + if json { c.JSON = true } From 81580e42158e8e28bd6a16bb57f4ad0249f21a0d Mon Sep 17 00:00:00 2001 From: g3ol4d0 Date: Sun, 13 Feb 2022 20:11:10 -0300 Subject: [PATCH 2/3] Removing profiler --- cmd/gau/main.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/gau/main.go b/cmd/gau/main.go index f84db6a..f1182bc 100644 --- a/cmd/gau/main.go +++ b/cmd/gau/main.go @@ -8,8 +8,6 @@ import ( log "github.com/sirupsen/logrus" "io" "os" - "net/http" - _ "net/http/pprof" "sync" ) @@ -22,9 +20,6 @@ func main() { } } - - log.Println(http.ListenAndServe("localhost:8989", nil)) - pMap := make(runner.ProvidersMap) for _, provider := range cfg.Providers { pMap[provider] = cfg.Filters From 802222d0e151e04fa9e2b828cc55448579c95d30 Mon Sep 17 00:00:00 2001 From: lc Date: Wed, 2 Mar 2022 09:39:02 -0600 Subject: [PATCH 3/3] feat(gau): use map[string]struct{} and bump version --- pkg/output/output.go | 6 +++--- pkg/providers/providers.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/output/output.go b/pkg/output/output.go index 190faad..a5bdbe7 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -14,7 +14,7 @@ type JSONResult struct { } func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}, RemoveParameters bool) error { - lastURL := make(map[string]bool) + lastURL := make(map[string]struct{}) for result := range results { buf := bytebufferpool.Get() if len(blacklistMap) != 0 { @@ -36,10 +36,10 @@ func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string] if err != nil { continue } - if lastURL[u.Host+u.Path] { + if _, ok := lastURL[u.Host+u.Path]; ok { continue } else { - lastURL[u.Host+u.Path] = true ; + lastURL[u.Host+u.Path] = struct{}{} ; } } diff --git a/pkg/providers/providers.go b/pkg/providers/providers.go index f3d1471..4324413 100644 --- a/pkg/providers/providers.go +++ b/pkg/providers/providers.go @@ -5,7 +5,7 @@ import ( "github.com/valyala/fasthttp" ) -const Version = `2.0.8` +const Version = `2.0.9` // Provider is a generic interface for all archive fetchers type Provider interface {