feat: matcher

main.go

@@ -1,13 +1,17 @@
 package main
 
 import (
+	"crypto/tls"
 	"flag"
 	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 )
@@ -21,7 +25,10 @@ const (
 /_/_/ /_/\__,_/\___/_/|_|\__/_/   \___/\___/ %s
 
 `
-	version = `v1.0.0`
+	version = `v1.0.2`
+
+	treeBranch = `├── `
+	treeEnd    = `└── `
 )
 
 type (
@@ -41,13 +48,32 @@ func main() {
 		showBanner()
 	}
 
+	wg := &sync.WaitGroup{}
+
+	line := make(chan string)
+
+	go func() {
+		for l := range line {
+			fmt.Println(l)
+		}
+	}()
+
 	err := crawl(
+		line,
+		wg,
 		options.URL,
 		options.Extensions,
+		options.Matchers,
 		"")
+
 	if err != nil {
 		log.Fatal(err)
 	}
+
+	wg.Wait()
+	time.Sleep(1 * time.Second)
+
+	close(line)
 }
 
 func showBanner() {
@@ -56,11 +82,15 @@ func showBanner() {
 
 func parseOptions() *Options {
 	options := &Options{}
+
 	extensions := ""
+	matchers := ""
 
 	flag.StringVar(&options.URL, "u", "", "url to parse index")
 	flag.StringVar(&extensions, "e", "", "extensions to filter, example: -e jpg,png,gif")
+	flag.StringVar(&matchers, "m", "", "match in url, example: -mu admin,login")
 	flag.BoolVar(&options.Banner, "b", true, "show banner")
+
 	flag.Parse()
 
 	if options.URL == "" {
@@ -77,54 +107,93 @@ func parseOptions() *Options {
 		options.Extensions = strings.Split(extensions, ",")
 	}
 
+	if matchers != "" {
+		options.Matchers = strings.Split(matchers, ",")
+	}
+
 	return options
 }
 
-func crawl(url string, extensions []string, prefix string) error {
+func crawl(line chan string, wg *sync.WaitGroup, url string, extensions, matchers []string, prefix string) error {
+	wg.Add(1)
+	defer wg.Done()
+
 	body, err := get(url)
 	if err != nil {
 		return fmt.Errorf("error getting index: %w", err)
 	}
 	defer body.Close()
 
-	urls, err := parseIndex(url, prefix, extensions, body)
+	urls, err := parseIndex(body)
 	if err != nil {
 		return fmt.Errorf("error parsing index: %w", err)
 	}
 
 	for i, u := range urls {
+		u = url + u
+
+		suffix := "│   "
 		if i == len(urls)-1 {
-			fmt.Println(prefix + "└── " + u)
-			if strings.HasSuffix(u, "/") {
-				err = crawl(u, extensions, prefix+"    ")
-				if err != nil {
-					log.Print(err)
+			suffix = "    "
+		}
+
+		//is dir
+		if strings.HasSuffix(u, "/") {
+			line <- prefix + treeBranch + u
+
+			err = crawl(line, wg, u, extensions, matchers, prefix+suffix)
+			if err != nil {
+				log.Print(err)
+			}
+			continue
+		}
+
+		//is match with matchers
+		if len(matchers) > 0 {
+			for _, m := range matchers {
+				if strings.Contains(strings.ToLower(u), strings.ToLower(m)) {
+					line <- prefix + treeBranch + u
+					continue
 				}
 			}
-		} else {
-			fmt.Println(prefix + "├── " + u)
-			if strings.HasSuffix(u, "/") {
-				err = crawl(u, extensions, prefix+"│   ")
-				if err != nil {
-					log.Print(err)
+			continue
+		}
+
+		//is not dir
+		if !strings.HasSuffix(u, "/") && len(extensions) > 0 {
+			ext := filepath.Ext(u)
+			if len(ext) > 0 {
+				ext = ext[1:]
+				for _, e := range extensions {
+					if e == ext {
+						line <- prefix + treeEnd + u
+						continue
+					}
 				}
 			}
+			continue
 		}
+
+		line <- prefix + treeBranch + u
 	}
 
 	return nil
 }
 
 func get(url string) (io.ReadCloser, error) {
-	resp, err := http.Get(url)
+	client := &http.Client{Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}}
+
+	resp, err := client.Get(url)
 	if err != nil {
 		return nil, fmt.Errorf("error getting index: %w", err)
 	}
 
 	return resp.Body, nil
 }
 
-func parseIndex(url, prefix string, extensions []string, body io.Reader) ([]string, error) {
+func parseIndex(body io.Reader) ([]string, error) {
 	doc, err := goquery.NewDocumentFromReader(body)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing index: %w", err)
@@ -133,43 +202,13 @@ func parseIndex(url, prefix string, extensions []string, body io.Reader) ([]stri
 	urls := make([]string, 0)
 
 	doc.Find("a").Each(func(i int, s *goquery.Selection) {
-		if url[len(url)-1:] != "/" {
-			url += "/"
-		}
-
 		//skip parent directory
 		href, _ := s.Attr("href")
-		if href == "../" { // parent directory
-			fmt.Println(prefix + "└── " + url + href)
-			return
-		}
-
-		//handle files
-		if len(s.Text()) > 0 && s.Text()[len(s.Text())-1:] != "/" {
-			//filter by extensions
-			if len(extensions) > 0 {
-				pos := strings.LastIndex(url+href, ".")
-				if pos == -1 {
-					return
-				}
-
-				ext := (url + href)[pos+1 : len(url+href)]
-				if len(ext) > 0 {
-					for _, e := range extensions {
-						if e == ext {
-							fmt.Println(prefix + "└── " + url + href)
-							return
-						}
-					}
-				}
-				return
-			}
-
-			fmt.Println(prefix + "└── " + url + href)
+		if strings.HasPrefix(href, "?") || href == "../" || href == "/" || strings.Contains(strings.ToLower(s.Text()), "parent") { // parent directory
 			return
 		}
 
-		urls = append(urls, url+href)
+		urls = append(urls, href)
 	})
 
 	return urls, nil