diff --git a/src/TurnerSoftware.SitemapTools/SitemapQuery.cs b/src/TurnerSoftware.SitemapTools/SitemapQuery.cs index 2a350dd..3a5173f 100644 --- a/src/TurnerSoftware.SitemapTools/SitemapQuery.cs +++ b/src/TurnerSoftware.SitemapTools/SitemapQuery.cs @@ -61,18 +61,16 @@ public async Task> DiscoverSitemaps(string domainName) { defaultSitemapUri }; - - //TODO: Need to release new version of RobotsParser with ability to pass a HttpClient - //var robotsFile = await new RobotsParser().FromUriAsync(baseUri); - //sitemapUris.AddRange(robotsFile.SitemapEntries.Select(s => s.Sitemap)); - //sitemapUris = sitemapUris.Distinct().ToList(); + + var robotsFile = await new RobotsParser(HttpClient).FromUriAsync(baseUri); + sitemapUris.AddRange(robotsFile.SitemapEntries.Select(s => s.Sitemap)); + sitemapUris = sitemapUris.Distinct().ToList(); var result = new HashSet(); foreach (var uri in sitemapUris) { try { - //We perform a head request because we don't care about the content here var requestMessage = new HttpRequestMessage(HttpMethod.Head, uri); var response = await HttpClient.SendAsync(requestMessage); diff --git a/src/TurnerSoftware.SitemapTools/TurnerSoftware.SitemapTools.csproj b/src/TurnerSoftware.SitemapTools/TurnerSoftware.SitemapTools.csproj index 1698517..0f44ac1 100644 --- a/src/TurnerSoftware.SitemapTools/TurnerSoftware.SitemapTools.csproj +++ b/src/TurnerSoftware.SitemapTools/TurnerSoftware.SitemapTools.csproj @@ -19,7 +19,7 @@ all runtime; build; native; contentfiles; analyzers - + diff --git a/tests/TurnerSoftware.SitemapTools.Tests/SitemapQueryTests.cs b/tests/TurnerSoftware.SitemapTools.Tests/SitemapQueryTests.cs index c3a7a52..e5c1fd3 100644 --- a/tests/TurnerSoftware.SitemapTools.Tests/SitemapQueryTests.cs +++ b/tests/TurnerSoftware.SitemapTools.Tests/SitemapQueryTests.cs @@ -21,7 +21,7 @@ public async Task GetSitemap() var sitemap = await sitemapQuery.GetSitemap(uriBuilder.Uri); Assert.AreEqual(0, sitemap.Sitemaps.Count()); - Assert.AreEqual(5, sitemap.Urls.Count()); + Assert.AreEqual(12, sitemap.Urls.Count()); } [TestMethod]