From d183b024a34f223141e7bae54b33153116c7ee12 Mon Sep 17 00:00:00 2001
From: edoardottt <edoardott@gmail.com>
Date: Tue, 3 May 2022 15:59:56 +0200
Subject: [PATCH 1/2] Add make lint

---
 Makefile | 3 +++
 make.bat | 9 +++++++++
 2 files changed, 12 insertions(+)

diff --git a/Makefile b/Makefile
index a71a49f..3b3673d 100644
--- a/Makefile
+++ b/Makefile
@@ -18,6 +18,9 @@ update:
 	@make linux
 	@echo "Done."
 
+lint:
+	@golangci-lint run
+
 linux:
 	@go build -o ./cariddi
 	@sudo mv ./cariddi /usr/bin/
diff --git a/make.bat b/make.bat
index 4caefed..195ead8 100644
--- a/make.bat
+++ b/make.bat
@@ -22,6 +22,11 @@ IF "%ARG%"=="fmt" (
   GOTO Done
 )
 
+IF "%ARG%"=="lint" (
+  CALL :Lint
+  GOTO Done
+)
+
 IF "%ARG%"=="remod" (
   del go.mod
   del go.sum
@@ -52,6 +57,10 @@ go fmt ./...
 echo Done.
 EXIT /B 0
 
+:Lint
+golangci-lint run
+EXIT /B 0
+
 :Update
 set GO111MODULE=on
 echo Updating ...

From fe14f924f3dbad4cf85756c5ebf07684141594ee Mon Sep 17 00:00:00 2001
From: edoardottt <edoardott@gmail.com>
Date: Tue, 3 May 2022 16:03:46 +0200
Subject: [PATCH 2/2] Remove ignorerobotstxt

---
 crawler/colly.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crawler/colly.go b/crawler/colly.go
index 8f0e64e..cd47f41 100644
--- a/crawler/colly.go
+++ b/crawler/colly.go
@@ -364,7 +364,7 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
 	c := colly.NewCollector(
 		colly.Async(true),
 	)
-	c.IgnoreRobotsTxt = false
+	c.IgnoreRobotsTxt = true
 	c.AllowURLRevisit = false
 
 	err := c.Limit(