Merge pull request #24 from edoardottt/devel

Devel - v1.1.1

crawler/colly.go

@@ -88,44 +88,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	var FinalSecrets []scanner.SecretMatched
 	var FinalEndpoints []scanner.EndpointMatched
 	var FinalExtensions []scanner.FileTypeMatched
-	c := colly.NewCollector(
-		colly.Async(true),
-	)
-
-	c.Limit(
-		&colly.LimitRule{
-			Parallelism: concurrency,
-			Delay:       time.Duration(delayTime) * time.Second,
-		},
-	)
-	c.AllowURLRevisit = false
-
-	// Using timeout if needed
-	if timeout != 10 {
-		c.SetRequestTimeout(time.Second * time.Duration(timeout))
-	}
-
-	// Using cache if needed
-	if cache {
-		c.CacheDir = ".cariddi_cache"
-	}
-
-	// Use a Random User Agent for each request if needed
-	if rua {
-		extensions.RandomUserAgent(c)
-	} else {
-		// Avoid using the default colly user agent
-		c.UserAgent = GenerateRandomUserAgent()
-	}
 
-	// Use a Proxy if needed
-	if proxy != "" {
-		err := c.SetProxy(proxy)
-		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
-		}
-	}
+	//crawler creation
+	c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy)
 
 	// On every a element which has href attribute call callback
 	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
@@ -303,6 +268,53 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	return FinalResults, FinalSecrets, FinalEndpoints, FinalExtensions
 }
 
+//CreateColly takes as input all the settings needed to instantiate
+//a new Colly Collector object and it returns this object.
+func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
+	intensive bool, rua bool, proxy string) *colly.Collector {
+
+	c := colly.NewCollector(
+		colly.Async(true),
+	)
+
+	c.Limit(
+		&colly.LimitRule{
+			Parallelism: concurrency,
+			Delay:       time.Duration(delayTime) * time.Second,
+		},
+	)
+	c.AllowURLRevisit = false
+
+	// Using timeout if needed
+	if timeout != 10 {
+		c.SetRequestTimeout(time.Second * time.Duration(timeout))
+	}
+
+	// Using cache if needed
+	if cache {
+		c.CacheDir = ".cariddi_cache"
+	}
+
+	// Use a Random User Agent for each request if needed
+	if rua {
+		extensions.RandomUserAgent(c)
+	} else {
+		// Avoid using the default colly user agent
+		c.UserAgent = GenerateRandomUserAgent()
+	}
+
+	// Use a Proxy if needed
+	if proxy != "" {
+		err := c.SetProxy(proxy)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+	}
+
+	return c
+}
+
 //huntSecrets hunts for secrets
 func huntSecrets(secretsFile []string, target string, body string) []scanner.SecretMatched {
 	secrets := SecretsMatch(target, body, secretsFile)

go.mod

@@ -17,8 +17,8 @@ require (
 	github.com/mattn/go-colorable v0.1.11 // indirect
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/temoto/robotstxt v1.1.2 // indirect
-	golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 // indirect
-	golang.org/x/sys v0.0.0-20211001092434-39dca1131b70 // indirect
+	golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b // indirect
+	golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef // indirect
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect

go.sum

@@ -60,8 +60,8 @@ golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 h1:Z04ewVs7JhXaYkmDhBERPi41gnltfQpMWDnTnQbaCqk=
-golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b h1:SXy8Ld8oKlcogOvUAh0J5Pm5RKzgYBMMxLxt6n5XW50=
+golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -72,8 +72,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZOR
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211001092434-39dca1131b70 h1:pGleJoyD1yA5HfvuaksHxD0404gsEkNDerKsQ0N0y1s=
-golang.org/x/sys v0.0.0-20211001092434-39dca1131b70/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef h1:fPxZ3Umkct3LZ8gK9nbk+DWDJ9fstZa2grBn+lWVKPs=
+golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

main.go

@@ -1,6 +1,6 @@
 /*
 ==========
-Cariddi v1.1
+Cariddi v1.1.1
 ==========
 
 This program is free software: you can redistribute it and/or modify

output/beautify.go

@@ -35,7 +35,7 @@ func Beautify() {
 	banner2 := "   ___ __ _ _ __(_) __| | __| (_)\n"
 	banner3 := "  / __/ _` | '__| |/ _` |/ _` | |\n"
 	banner4 := " | (_| (_| | |  | | (_| | (_| | |\n"
-	banner5 := "  \\___\\__,_|_|  |_|\\__,_|\\__,_|_| v1.1\n"
+	banner5 := "  \\___\\__,_|_|  |_|\\__,_|\\__,_|_| v1.1.1\n"
 	banner6 := ""
 	banner7 := " > github.com/edoardottt/cariddi\n"
 	banner8 := " > edoardoottavianelli.it\n"