From dc71859c9ef596ad09f7ba88fe125e66d4aa72d6 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 16 Nov 2016 07:28:23 -0800 Subject: [PATCH 1/2] Failing test about robots.txt agent case insensitivity. --- test/test-robots.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-robots.cpp b/test/test-robots.cpp index bedea8e..4593845 100644 --- a/test/test-robots.cpp +++ b/test/test-robots.cpp @@ -122,7 +122,7 @@ TEST(RobotsTest, ExposesSitemaps) TEST(RobotsTest, CaseInsensitivity) { std::string content = - "User-agent: agent\n" + "User-agent: Agent\n" "Disallow: /path\n"; Rep::Robots robot(content); EXPECT_FALSE(robot.allowed("/path", "agent")); From b829e56de194eb95b8510723410f1962b28c3c49 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 16 Nov 2016 07:29:49 -0800 Subject: [PATCH 2/2] Passing test about robots.txt agent case insensitivity. --- src/robots.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/robots.cpp b/src/robots.cpp index c764c35..6382e8d 100644 --- a/src/robots.cpp +++ b/src/robots.cpp @@ -65,6 +65,9 @@ namespace Rep { if (key.compare("user-agent") == 0) { + // Store the user agent string as lowercased + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + if (last_agent) { group.push_back(value);