do not follow links that have attribute set to

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to `spatie/crawler` will be documented in this file.
 
+## 4.0.4 - 2018-03-20
+
+- do not follow links that have attribute `rel` set to `nofollow`
+
 ## 4.0.3 - 2018-03-02
 
 - Support both `Illuminate`'s and `Tighten`'s `Collection`. 

src/Crawler.php

@@ -414,6 +414,9 @@ protected function extractAllLinks(string $html, UriInterface $foundOnUrl)
         $domCrawler = new DomCrawler($html, $foundOnUrl);
 
         return collect($domCrawler->filterXpath('//a')->links())
+            ->reject(function(Link $link) {
+                return $link->getNode()->getAttribute('rel') === 'nofollow';
+            })
             ->map(function (Link $link) {
                 try {
                     return new Uri($link->getUri());

tests/CrawlerTest.php

@@ -342,6 +342,17 @@ public function it_crawls_subdomains()
         ]);
     }
 
+    /** @test */
+    public function it_should_not_follow_nofollow_links()
+    {
+        Crawler::create()
+            ->setCrawlObserver(new CrawlLogger())
+            ->setMaximumDepth(1)
+            ->startCrawling('http://localhost:8080');
+
+        $this->assertNotCrawled([['url' => 'http://localhost:8080/nofollow', 'foundOn' => 'http://localhost:8080/']]);
+    }
+
     protected function regularUrls(): array
     {
         return [

tests/server/server.js

@@ -3,7 +3,7 @@
 let app = require('express')();
 
 app.get('/', function (request, response) {
-    response.end('<a href="/link1">Link1</a><a href="/link2">Link2</a><a href="dir/link4">Link4</a><a href="mailto:[email protected]">Email</a><a href="tel:123">Telephone</a>');
+    response.end('<a href="/link1">Link1</a><a href="/link2">Link2</a><a href="dir/link4">Link4</a><a href="mailto:[email protected]">Email</a><a href="tel:123">Telephone</a><a href="/nofollow" rel="nofollow">No follow</a>');
 });
 
 app.get('/link1', function (request, response) {
@@ -14,6 +14,10 @@ app.get('/javascript', function (request, response) {
     response.end('This page can only be reached if JavaScript is being executed');
 });
 
+app.get('/nofollow', function (request, response) {
+    response.end('This page should not be crawler');
+});
+
 app.get('/link2', function (request, response) {
     response.end('You are on link2<a href="/link3">Link3</a><a href="http://sub.localhost:8080/subdomainpage">Subdomain</a><a href="http://subdomain.sub.localhost:8080/subdomainpage">Subdomain2</a>');
 });