Fix duplicate visits (#108)

* Fix duplicate visits

* Bump version

* Improve format

arguments.go

@@ -45,15 +45,13 @@ func getArguments(ss []string) (*arguments, error) {
 	args.URL = ss[0]
 
 	rs, err := compileRegexps(args.RawExcludedPatterns)
-
 	if err != nil {
 		return nil, err
 	}
 
 	args.ExcludedPatterns = rs
 
 	hs, err := parseHeaders(args.RawHeaders)
-
 	if err != nil {
 		return nil, err
 	}

checker.go

@@ -61,10 +61,7 @@ func (c *checker) checkPage(p *page) {
 				ec <- &errorLinkResult{u, err}
 			}
 
-			if !c.onePageOnly &&
-				p != nil &&
-				c.linkValidator.Validate(p.URL()) &&
-				!c.donePages.Add(p.URL().String()) {
+			if !c.onePageOnly && p != nil && c.linkValidator.Validate(p.URL()) {
 				c.addPage(p)
 			}
 		}(u)
@@ -91,5 +88,7 @@ func (c *checker) checkPage(p *page) {
 }
 
 func (c *checker) addPage(p *page) {
-	c.daemonManager.Add(func() { c.checkPage(p) })
+	if !c.donePages.Add(p.URL().String()) {
+		c.daemonManager.Add(func() { c.checkPage(p) })
+	}
 }

checker_test.go

@@ -92,3 +92,23 @@ func TestCheckerFailToCheckPage(t *testing.T) {
 
 	assert.False(t, (<-c.Results()).OK())
 }
+
+func TestCheckerDoNotCheckSamePageTwice(t *testing.T) {
+	c := newTestChecker(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				return newFakeHTTPResponse(200, "http://foo.com", "text/html", nil), nil
+			},
+		),
+	)
+
+	go c.Check(newTestPage(t, nil, map[string]error{"http://foo.com": nil}))
+
+	i := 0
+
+	for range c.Results() {
+		i++
+	}
+
+	assert.Equal(t, 1, i)
+}

command.go

@@ -22,7 +22,6 @@ func newCommand(stdout, stderr io.Writer, terminal bool, f httpClientFactory) *c
 
 func (c *command) Run(args []string) bool {
 	ok, err := c.runWithError(args)
-
 	if err != nil {
 		c.printError(err)
 	}

configuration.go

@@ -3,7 +3,7 @@ package main
 import "time"
 
 const (
-	version               = "2.0.0"
+	version               = "2.0.1"
 	agentName             = "muffet"
 	concurrency           = 1024
 	tcpTimeout            = time.Minute

fasthttp_http_client_factory.go

@@ -24,7 +24,8 @@ func (*fasthttpHTTPClientFactory) Create(o httpClientOptions) httpClient {
 			},
 			Dial: func(addr string) (net.Conn, error) {
 				return fasthttp.DialTimeout(addr, tcpTimeout)
-			}},
+			},
+		},
 		o.MaxRedirections,
 		o.Timeout,
 	)

robots_txt_fetcher.go

@@ -19,7 +19,6 @@ func (f *robotsTxtFetcher) Fetch(uu *url.URL) (*robotstxt.RobotsData, error) {
 	u := *uu
 	u.Path = "robots.txt"
 	r, err := f.client.Get(&u, nil)
-
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch robots.txt: %v", err)
 	}

sitemap_fetcher.go

@@ -21,7 +21,6 @@ func (f *sitemapFetcher) Fetch(uu *url.URL) (map[string]struct{}, error) {
 	u.Path = "sitemap.xml"
 
 	r, err := f.client.Get(&u, nil)
-
 	if err != nil {
 		return nil, fmt.Errorf("failed to GET sitemap.xml: %v", err)
 	}