From 7161f4f6eefaed16af087819b65f3ecbf2b9967b Mon Sep 17 00:00:00 2001 From: David Lambert Date: Thu, 10 Oct 2024 20:30:31 -0400 Subject: [PATCH] Added content encoding to the function that detects compressed sitemaps. I ran into a site that was gzipping sitemap entries and they weren't being detected properly. --- src/TurnerSoftware.SitemapTools/SitemapQuery.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/TurnerSoftware.SitemapTools/SitemapQuery.cs b/src/TurnerSoftware.SitemapTools/SitemapQuery.cs index c0d2b41..71158f4 100644 --- a/src/TurnerSoftware.SitemapTools/SitemapQuery.cs +++ b/src/TurnerSoftware.SitemapTools/SitemapQuery.cs @@ -131,9 +131,10 @@ public async Task> DiscoverSitemapsAsync(string domainName, Can return result; } - private static bool IsCompressedStream(string contentType) => + private static bool IsCompressedStream(string contentType, ICollection contentEncoding) => contentType.Equals("application/x-gzip", StringComparison.InvariantCultureIgnoreCase) || - contentType.Equals("application/octet-stream", StringComparison.InvariantCultureIgnoreCase); + contentType.Equals("application/octet-stream", StringComparison.InvariantCultureIgnoreCase) || + contentEncoding.Contains("x-gzip"); /// /// Retrieves a sitemap at the given URI, converting it to a . @@ -149,9 +150,10 @@ private static bool IsCompressedStream(string contentType) => if (response.IsSuccessStatusCode) { var contentType = response.Content.Headers.ContentType.MediaType; + var contentEncoding = response.Content.Headers.ContentEncoding; var requiresManualDecompression = false; - - if (IsCompressedStream(contentType)) + + if (IsCompressedStream(contentType, contentEncoding)) { requiresManualDecompression = true; var baseFileName = Path.GetFileNameWithoutExtension(sitemapUrl.AbsolutePath);