Stop generating sitemap.xml.gz (#6561)

We generate sitemap-index.xml which is also put correctly into robots.txt. However we still provide the old sitemap.xml.gz.

But, while this serves no purpose in addition to the index, it can cause problems if Google somehow parses it (for example it's submitted to the Google Search console). For this reason, we  stop providing sitemap.xml.gz.

- also serve the batched sitemap under the old name sitemap.xml.gz
- update comment in robots.txt

packages/volto/news/6561.bugfix

@@ -0,0 +1 @@
+Stop generating sitemap.xml.gz @reebalazs

packages/volto/src/express-middleware/sitemap.js

@@ -47,11 +47,24 @@ export const sitemapIndex = function (req, res, next) {
   });
 };
 
+export const sitemapIndexCompatibility = function (req, res, next) {
+  generateSitemapIndex(req, true).then((sitemapIndex) => {
+    res.set('Content-Type', 'application/x-gzip');
+    res.set('Content-Disposition', 'attachment; filename="sitemap.xml.gz"');
+    res.send(sitemapIndex);
+  });
+};
+
 export default function sitemapMiddleware() {
   const middleware = express.Router();
 
-  middleware.all('**/sitemap.xml.gz', sitemap);
+  // For backwards compatibility, and allow a graceful transition for
+  // sites that are already set up on the Google Search Console, we continue delivering
+  // the new batched sitemap under the old sitemap.xml.gz name.
+  middleware.all('**/sitemap.xml.gz', sitemapIndexCompatibility);
   middleware.all('**/sitemap:batch.xml.gz', sitemap);
+  // For new setups, `sitemap-index.xml` should be added to the
+  // Google Search Console.
   middleware.all('**/sitemap-index.xml', sitemapIndex);
   middleware.id = 'sitemap.xml.gz';
   return middleware;

packages/volto/src/helpers/Sitemap/Sitemap.js

@@ -61,7 +61,7 @@ export const generateSitemap = (_req, start = 0, size = undefined) =>
  * @param {Object} _req Request object
  * @return {string} Generated sitemap index
  */
-export const generateSitemapIndex = (_req) =>
+export const generateSitemapIndex = (_req, gzip = false) =>
   new Promise((resolve) => {
     const { settings } = config;
     const APISUFIX = settings.legacyTraverse ? '' : '/++api++';
@@ -88,7 +88,14 @@ export const generateSitemapIndex = (_req) =>
         const result = `<?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 ${items.join('\n')}\n</sitemapindex>`;
-        resolve(result);
+
+        if (gzip) {
+          zlib.gzip(Buffer.from(result, 'utf8'), (_err, buffer) => {
+            resolve(buffer);
+          });
+        } else {
+          resolve(result);
+        }
       }
     });
   });