Merge pull request #38 from cogniteev/bug/prefix_wildcard_with_fullpath

Merging fix for leading asterisk in robots.txt directive.
seomoz · May 28, 2019 · f406b76 · f406b76
2 parents 8afac6e + 90901e7
commit f406b76
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 2 deletions.
diff --git a/src/agent.cpp b/src/agent.cpp
@@ -25,7 +25,7 @@ namespace Rep
         {
             return *this;
         }
-        directives_.push_back(Directive(escape_url(url), true));
+        directives_.push_back(Directive(url.defrag().escape().str(), true));
         sorted_ = false;
         return *this;
     }
@@ -45,7 +45,8 @@ namespace Rep
             {
                 return *this;
             }
-            directives_.push_back(Directive(escape_url(url), false));
+
+            directives_.push_back(Directive(url.defrag().escape().str(), false));
         }
         sorted_ = false;
         return *this;

diff --git a/test/test-robots.cpp b/test/test-robots.cpp
@@ -319,3 +319,21 @@ TEST(RobotsTest, NeverExternalAllowed)
     Rep::Robots robot("", "http://a.com/robots.txt");
     EXPECT_FALSE(robot.allowed("http://b.com/", "one"));
 }
+
+TEST(RobotsTest, PrefixStarExample)
+{
+    std::string content =
+        "# /robots.txt for fun and profit\n"
+        "\n"
+        "User-agent: ohmagad\n"
+        "Allow: /\n"
+        "Disallow: */dir\n"
+        "Disallow: /*/dir\n";
+    Rep::Robots robot(content);
+
+    // The ohmagad bot
+    EXPECT_TRUE(robot.allowed("/", "ohmagad"));
+    EXPECT_TRUE(robot.allowed("/a/b/page.html", "ohmagad"));
+    EXPECT_FALSE(robot.allowed("/dir/page.html", "ohmagad"));
+    EXPECT_FALSE(robot.allowed("/some/dir/page.html", "ohmagad"));
+}