Remove ignorerobotstxt

edoardottt · May 3, 2022 · fe14f92 · fe14f92
1 parent d183b02
commit fe14f92
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/crawler/colly.go b/crawler/colly.go
@@ -364,7 +364,7 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
 	c := colly.NewCollector(
 		colly.Async(true),
 	)
-	c.IgnoreRobotsTxt = false
+	c.IgnoreRobotsTxt = true
 	c.AllowURLRevisit = false
 
 	err := c.Limit(