From c4a2842b42afe8b6673395e10f119aadf91693df Mon Sep 17 00:00:00 2001 From: psdigital Date: Thu, 21 Jan 2021 12:15:15 +1300 Subject: [PATCH 1/4] Add robots noindex switch for global and individual pages --- _config/config.yml | 5 ++- src/Extensions/SitemapPageExtension.php | 31 +++++++++++++++++-- src/Extensions/SitemapSiteConfigExtension.php | 26 ++++++++++++++++ 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 src/Extensions/SitemapSiteConfigExtension.php diff --git a/_config/config.yml b/_config/config.yml index 52df732..5e4d713 100644 --- a/_config/config.yml +++ b/_config/config.yml @@ -1,3 +1,6 @@ Page: extensions: - - PlasticStudio\Sitemap\Extensions\SitemapPageExtension \ No newline at end of file + - PlasticStudio\Sitemap\Extensions\SitemapPageExtension +SilverStripe\SiteConfig\SiteConfig: + extensions: + - PlasticStudio\Sitemap\Extensions\SitemapSiteConfigExtension \ No newline at end of file diff --git a/src/Extensions/SitemapPageExtension.php b/src/Extensions/SitemapPageExtension.php index 1e490d9..e2f84a8 100644 --- a/src/Extensions/SitemapPageExtension.php +++ b/src/Extensions/SitemapPageExtension.php @@ -6,28 +6,53 @@ use Silverstripe\Forms\FieldList; use Silverstripe\Forms\CheckBoxField; use SilverStripe\CMS\Model\SiteTree; +use SilverStripe\SiteConfig\SiteConfig; class SitemapPageExtension extends DataExtension { private static $db = [ - 'ExcludeFromSitemap' => 'Boolean' + 'ExcludeFromSitemap' => 'Boolean', + 'DoNotIndex' => 'Boolean' ]; public function updateSettingsFields(FieldList $fields) { - $fields->addFieldToTab( - 'Root.Settings', + $fields->insertBefore( + 'ShowInSearch', CheckBoxField::create( 'ExcludeFromSitemap', 'Exclude from sitemap?' ), 'ShowInSearch' ); + $fields->insertAfter( + 'ShowInSearch', + CheckBoxField::create( + 'DoNotIndex', + 'Exclude this page from indexing robots (like Google etc)' + ) + ); } public function Sitemap() { return SiteTree::get()->Filter(['ParentID' => 0]); } + + /** + * Append robots meta tag to SiteTree's ExtraMeta db attribute + * + * @return string meta tags + */ + public function ExtraMeta() + { + $tags = $this->owner->ExtraMeta; + if (SiteConfig::current_site_config()->DoNotIndex) { + $tags .= ''; + } else if ($this->owner->DoNotIndex) { + $tags .= ''; + } + return $tags; + } } \ No newline at end of file diff --git a/src/Extensions/SitemapSiteConfigExtension.php b/src/Extensions/SitemapSiteConfigExtension.php new file mode 100644 index 0000000..7940381 --- /dev/null +++ b/src/Extensions/SitemapSiteConfigExtension.php @@ -0,0 +1,26 @@ + 'Boolean' + ]; + + public function updateCMSFields(FieldList $fields) + { + $fields->addFieldToTab( + 'Root.Indexing', + CheckBoxField::create( + 'DoNotIndex', + 'Exclude entire site from indexing robots (like Google etc)' + ) + ); + } + +} \ No newline at end of file From 29f7355cdf9ff352d733e24f68fa35e0ec9661bc Mon Sep 17 00:00:00 2001 From: psdigital Date: Thu, 21 Jan 2021 12:17:18 +1300 Subject: [PATCH 2/4] Simplify tag appending condition --- src/Extensions/SitemapPageExtension.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Extensions/SitemapPageExtension.php b/src/Extensions/SitemapPageExtension.php index e2f84a8..8b541bb 100644 --- a/src/Extensions/SitemapPageExtension.php +++ b/src/Extensions/SitemapPageExtension.php @@ -47,9 +47,7 @@ public function Sitemap() public function ExtraMeta() { $tags = $this->owner->ExtraMeta; - if (SiteConfig::current_site_config()->DoNotIndex) { - $tags .= ''; - } else if ($this->owner->DoNotIndex) { + if (SiteConfig::current_site_config()->DoNotIndex || $this->owner->DoNotIndex) { $tags .= ''; } return $tags; From 5e6695a13816d47cfc1271a1987b29da8ad7dfc0 Mon Sep 17 00:00:00 2001 From: psdigital Date: Thu, 21 Jan 2021 13:37:49 +1300 Subject: [PATCH 3/4] Field descriptions for noindex checkboxes --- src/Extensions/SitemapPageExtension.php | 2 +- src/Extensions/SitemapSiteConfigExtension.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Extensions/SitemapPageExtension.php b/src/Extensions/SitemapPageExtension.php index 8b541bb..f635766 100644 --- a/src/Extensions/SitemapPageExtension.php +++ b/src/Extensions/SitemapPageExtension.php @@ -30,7 +30,7 @@ public function updateSettingsFields(FieldList $fields) CheckBoxField::create( 'DoNotIndex', 'Exclude this page from indexing robots (like Google etc)' - ) + )->setDescription('Checking this box will add a "robots no-index" tag to the head of this page, preventing search engine bots from crawling and indexing it (assuming those bots are well-behaved and respect the tag)') ); } diff --git a/src/Extensions/SitemapSiteConfigExtension.php b/src/Extensions/SitemapSiteConfigExtension.php index 7940381..325f5b8 100644 --- a/src/Extensions/SitemapSiteConfigExtension.php +++ b/src/Extensions/SitemapSiteConfigExtension.php @@ -19,7 +19,7 @@ public function updateCMSFields(FieldList $fields) CheckBoxField::create( 'DoNotIndex', 'Exclude entire site from indexing robots (like Google etc)' - ) + )->setDescription('Checking this box will add a "robots no-index" tag to the head of every page, preventing search engine bots from crawling and indexing this site (assuming those bots are well-behaved and respect the tag)') ); } From d5eda0039cf06582d08fa3287c45df4907cfc339 Mon Sep 17 00:00:00 2001 From: psdigital Date: Thu, 21 Jan 2021 13:42:29 +1300 Subject: [PATCH 4/4] Update readme --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eed8897..5cc19c2 100644 --- a/README.md +++ b/README.md @@ -21,11 +21,14 @@ Features * Exclude any page from sitemaps with CMS checkbox (on Settings tab) * Uses partial caching, refreshed when any page in the sitetree is updated * Creates/updates an actual sitemap.xml file in the root of the site on load of xml sitemap page +* Adds the ability to prevent a single page or the entire site from being crawled and indexed by (well-behaved) bots Installation ============ -1. Install module +1. Install module via composer `composer require plasticstudio/sitemap` 2. Run /dev/build?flush=1 3. Pages can be excluded from sitemap using the checkbox in the Settings CMS tab (under Visibility) +4. Pages can be excluded from being crawled/indexed by bots in the Settings CMS tab (under Visibility) +5. The entire site can be excluded from being crawled/indexed by bots in Site Settings (under the Indexing tab)