Updated RobotsExclusionTools support

TurnerSoftware · Dec 28, 2018 · 7e7a7f6 · 7e7a7f6
1 parent c49a2d7
commit 7e7a7f6
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 8 deletions.
diff --git a/src/TurnerSoftware.SitemapTools/SitemapQuery.cs b/src/TurnerSoftware.SitemapTools/SitemapQuery.cs
@@ -61,18 +61,16 @@ public async Task<IEnumerable<Uri>> DiscoverSitemaps(string domainName)
 			{
 				defaultSitemapUri
 			};
-
-			//TODO: Need to release new version of RobotsParser with ability to pass a HttpClient
-			//var robotsFile = await new RobotsParser().FromUriAsync(baseUri);
-			//sitemapUris.AddRange(robotsFile.SitemapEntries.Select(s => s.Sitemap));
-			//sitemapUris = sitemapUris.Distinct().ToList();
+
+			var robotsFile = await new RobotsParser(HttpClient).FromUriAsync(baseUri);
+			sitemapUris.AddRange(robotsFile.SitemapEntries.Select(s => s.Sitemap));
+			sitemapUris = sitemapUris.Distinct().ToList();
 
 			var result = new HashSet<Uri>();
 			foreach (var uri in sitemapUris)
 			{
 				try
 				{
-					//We perform a head request because we don't care about the content here
 					var requestMessage = new HttpRequestMessage(HttpMethod.Head, uri);
 					var response = await HttpClient.SendAsync(requestMessage);
 

diff --git a/src/TurnerSoftware.SitemapTools/TurnerSoftware.SitemapTools.csproj b/src/TurnerSoftware.SitemapTools/TurnerSoftware.SitemapTools.csproj
@@ -19,7 +19,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="TurnerSoftware.RobotsExclusionTools" Version="0.1.0" />
+    <PackageReference Include="TurnerSoftware.RobotsExclusionTools" Version="0.2.0" />
   </ItemGroup>
 
 </Project>
diff --git a/tests/TurnerSoftware.SitemapTools.Tests/SitemapQueryTests.cs b/tests/TurnerSoftware.SitemapTools.Tests/SitemapQueryTests.cs
@@ -21,7 +21,7 @@ public async Task GetSitemap()
 			var sitemap = await sitemapQuery.GetSitemap(uriBuilder.Uri);
 
 			Assert.AreEqual(0, sitemap.Sitemaps.Count());
-			Assert.AreEqual(5, sitemap.Urls.Count());
+			Assert.AreEqual(12, sitemap.Urls.Count());
 		}
 
 		[TestMethod]