Merge pull request #54 from edoardottt/devel

Devel
edoardottt · May 3, 2022 · ca6d5bf · ca6d5bf
2 parents cfcd63c + fe14f92
commit ca6d5bf
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 1 deletion.
diff --git a/Makefile b/Makefile
@@ -18,6 +18,9 @@ update:
 	@make linux
 	@echo "Done."
 
+lint:
+	@golangci-lint run
+
 linux:
 	@go build -o ./cariddi
 	@sudo mv ./cariddi /usr/bin/

diff --git a/crawler/colly.go b/crawler/colly.go
@@ -364,7 +364,7 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
 	c := colly.NewCollector(
 		colly.Async(true),
 	)
-	c.IgnoreRobotsTxt = false
+	c.IgnoreRobotsTxt = true
 	c.AllowURLRevisit = false
 
 	err := c.Limit(

diff --git a/make.bat b/make.bat
@@ -22,6 +22,11 @@ IF "%ARG%"=="fmt" (
   GOTO Done
 )
 
+IF "%ARG%"=="lint" (
+  CALL :Lint
+  GOTO Done
+)
+
 IF "%ARG%"=="remod" (
   del go.mod
   del go.sum
@@ -52,6 +57,10 @@ go fmt ./...
 echo Done.
 EXIT /B 0
 
+:Lint
+golangci-lint run
+EXIT /B 0
+
 :Update
 set GO111MODULE=on
 echo Updating ...