Split big sitemaps & generate proper index (closes #79, #106)

UPGRADE-2.0.md

@@ -12,8 +12,9 @@
 ## New features
 
 * Generation of sitemaps is done via the CLI, schedule them in a cronjob:
-    * Sitemap Index: `bin/console sylius:sitemap:generate-index`
+    * Sitemap Index: `bin/console sylius:sitemap:generate`
 * Sitemap URLs now support adding images. The default providers do this where possible. It can be disabled using the `images` configuration key.
+* Sitemaps are split automatically per 50.000 URL's. This can be adjusted by providing `--limit={int}` to the CLI command.
 
 ## Removed features
 

src/Builder/SitemapIndexBuilder.php

@@ -20,6 +20,9 @@ final class SitemapIndexBuilder implements SitemapIndexBuilderInterface
     /** @var array */
     private $indexProviders = [];
 
+    /** @var array */
+    private $paths = [];
+
     public function __construct(SitemapIndexFactoryInterface $sitemapIndexFactory)
     {
         $this->sitemapIndexFactory = $sitemapIndexFactory;
@@ -41,6 +44,18 @@ public function addIndexProvider(IndexUrlProviderInterface $provider): void
         $this->indexProviders[] = $provider;
     }
 
+    /**
+     * {@inheritdoc}
+     */
+    public function addPath(UrlProviderInterface $provider, string $path): void
+    {
+        if (!array_key_exists($provider->getName(), $this->paths)) {
+            $this->paths[$provider->getName()] = [];
+        }
+
+        $this->paths[$provider->getName()][] = $path;
+    }
+
     /**
      * {@inheritdoc}
      */
@@ -53,7 +68,7 @@ public function build(): SitemapInterface
         foreach ($this->indexProviders as $indexProvider) {
             /** @var UrlProviderInterface $provider */
             foreach ($this->providers as $provider) {
-                $indexProvider->addProvider($provider);
+                $indexProvider->addProvider($provider, $this->paths[$provider->getName()]);
             }
 
             $urls[] = $indexProvider->generate();

src/Command/GenerateSitemapCommand.php

@@ -56,43 +56,50 @@ public function __construct(
     protected function configure(): void
     {
         $this->addOption('channel', 'c', InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL, 'Channel codes to generate. If none supplied, all channels will generated.');
+        $this->addOption('limit', 'l', InputOption::VALUE_OPTIONAL, 'Limit amount of URLs per sitemap', 50000);
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
         foreach ($this->channels($input) as $channel) {
-            $this->executeChannel($channel, $output);
+            $this->executeChannel($channel, $input, $output);
         }
     }
 
-    private function executeChannel(ChannelInterface $channel, OutputInterface $output)
+    private function executeChannel(ChannelInterface $channel, InputInterface $input, OutputInterface $output)
     {
-        // TODO make sure providers are every time emptied (reset call or smth?)
         foreach ($this->sitemapBuilder->getProviders() as $provider) {
             $output->writeln(\sprintf('Start generating sitemap "%s" for channel "%s"', $provider->getName(), $channel->getCode()));
 
             $sitemap = $this->sitemapBuilder->build($provider, $channel); // TODO use provider instance, not the name
-            $xml = $this->sitemapRenderer->render($sitemap);
-            $path = $path = $this->path($channel, \sprintf('%s.xml', $provider->getName()));
+            $xml = $this->sitemapRenderer->render($sitemap, (int)$input->getOption('limit'));
+            foreach($xml as $index => $data) {
+                $path = $path = $this->path($channel, \sprintf('%s_%d.xml', $provider->getName(), $index));
 
-            $this->writer->write(
-                $path,
-                $xml
-            );
+                $this->writer->write(
+                    $path,
+                    $data
+                );
 
-            $output->writeln(\sprintf('Finished generating sitemap "%s" for channel "%s" at path "%s"', $provider->getName(), $channel->getCode(), $path));
+                $output->writeln(\sprintf('Finished generating sitemap "%s" (%d) for channel "%s" at path "%s"', $provider->getName(), $index, $channel->getCode(), $path));
+
+                $this->sitemapIndexBuilder->addPath($provider, $path);
+            }
         }
 
         $output->writeln(\sprintf('Start generating sitemap index for channel "%s"', $channel->getCode()));
 
         $sitemap = $this->sitemapIndexBuilder->build();
         $xml = $this->sitemapIndexRenderer->render($sitemap);
-        $path = $this->path($channel, 'sitemap_index.xml');
 
-        $this->writer->write(
-            $path,
-            $xml
-        );
+        foreach($xml as $index => $data) {
+            $path = $this->path($channel, 'sitemap_index.xml');
+
+            $this->writer->write(
+                $path,
+                $data
+            );
+        }
 
         $output->writeln(\sprintf('Finished generating sitemap index for channel "%s" at path "%s"', $channel->getCode(), $path));
     }

src/Controller/SitemapController.php

@@ -22,9 +22,9 @@ public function __construct(
         parent::__construct($reader);
     }
 
-    public function showAction(string $name): Response
+    public function showAction(string $name, int $index): Response
     {
-        $path = \sprintf('%s/%s', $this->channelContext->getChannel()->getCode(), \sprintf('%s.xml', $name));
+        $path = \sprintf('%s/%s', $this->channelContext->getChannel()->getCode(), \sprintf('%s_%d.xml', $name, $index));
 
         return $this->createResponse($path);
     }

src/Provider/IndexUrlProvider.php

@@ -21,6 +21,9 @@ final class IndexUrlProvider implements IndexUrlProviderInterface
     /** @var array */
     private $urls = [];
 
+    /** @var array */
+    private $paths = [];
+
     public function __construct(
         RouterInterface $router,
         IndexUrlFactoryInterface $sitemapIndexUrlFactory
@@ -29,17 +32,24 @@ public function __construct(
         $this->sitemapIndexUrlFactory = $sitemapIndexUrlFactory;
     }
 
-    public function addProvider(UrlProviderInterface $provider): void
+    public function addProvider(UrlProviderInterface $provider, array $paths = []): void
     {
         $this->providers[] = $provider;
+        $this->paths[$provider->getName()] = $paths;
     }
 
     public function generate(): iterable
     {
         foreach ($this->providers as $provider) {
-            $location = $this->router->generate('sylius_sitemap_' . $provider->getName());
+            $pathCount = count($this->paths[$provider->getName()]);
+
+            for ($i = 0; $i < $pathCount; $i++) {
+                $params = ['index' => $i];
+
+                $location = $this->router->generate('sylius_sitemap_'.$provider->getName(), $params);
 
-            $this->urls[] = $this->sitemapIndexUrlFactory->createNew($location);
+                $this->urls[] = $this->sitemapIndexUrlFactory->createNew($location);
+            }
         }
 
         return $this->urls;

src/Provider/IndexUrlProviderInterface.php

@@ -11,5 +11,5 @@ interface IndexUrlProviderInterface
      */
     public function generate(): iterable;
 
-    public function addProvider(UrlProviderInterface $provider): void;
+    public function addProvider(UrlProviderInterface $provider, array $paths = []): void;
 }

src/Renderer/SitemapRenderer.php

@@ -19,8 +19,19 @@ public function __construct(RendererAdapterInterface $adapter)
     /**
      * {@inheritdoc}
      */
-    public function render(SitemapInterface $sitemap): string
+    public function render(SitemapInterface $sitemap, ?int $limit = null): iterable
     {
-        return $this->adapter->render($sitemap);
+        $urls = $sitemap->getUrls();
+        $total = count($urls);
+
+        if (null === $limit || $limit < 0) {
+            $limit = $total;
+        }
+
+        foreach(array_chunk($urls, $limit) as $slice) {
+            $sitemap->setUrls($slice);
+
+            yield $this->adapter->render($sitemap);
+        }
     }
 }

src/Renderer/SitemapRendererInterface.php

@@ -8,5 +8,9 @@
 
 interface SitemapRendererInterface
 {
-    public function render(SitemapInterface $sitemap): string;
+
+    /**
+     * @return string[]
+     */
+    public function render(SitemapInterface $sitemap, ?int $limit = null): iterable;
 }

src/Routing/SitemapLoader.php

@@ -34,7 +34,7 @@ public function __construct(SitemapBuilderInterface $sitemapBuilder)
     public function load($resource, $type = null): RouteCollection
     {
         $routes = new RouteCollection();
-        
+
         if (true === $this->loaded) {
             return $routes;
         }
@@ -50,12 +50,14 @@ public function load($resource, $type = null): RouteCollection
             $routes->add(
                 $name,
                 new Route(
-                    '/sitemap/' . $provider->getName() . '.xml',
+                    '/sitemap/' . $provider->getName() . '/{index}.xml',
                     [
                         '_controller' => 'sylius.controller.sitemap:showAction',
                         'name' => $provider->getName(),
                     ],
-                    [],
+                    [
+                        'index' => '\d+',
+                    ],
                     [],
                     '',
                     [],