Send X-Robots-Tag header in all admin/api controllers and in pulse …

…error (#423) Pages that should not be indexed, should use the header or a meta tag, not robots.txt, but we'll try both at least for now. https://support.google.com/webmasters/answer/7440203#indexed_though_blocked_by_robots_txt https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag
spaze · Oct 30, 2024 · 495eafa · 495eafa
2 parents cb23505 + f1aefa6
commit 495eafa
Show file tree

Hide file tree

Showing 14 changed files with 138 additions and 5 deletions.
diff --git a/app/config/services.neon b/app/config/services.neon
@@ -85,6 +85,7 @@ services:
 	- MichalSpacekCz\Http\FetchMetadata\ResourceIsolationPolicy(reportOnly: true)
 	- MichalSpacekCz\Http\HttpInput
 	- MichalSpacekCz\Http\Redirections
+	- MichalSpacekCz\Http\Robots\Robots
 	- MichalSpacekCz\Http\SecurityHeaders(permissionsPolicy: %permissionsPolicy%)
 	- MichalSpacekCz\Interviews\InterviewInputsFactory(videoThumbnails: @interviewVideoThumbnails)
 	- MichalSpacekCz\Interviews\Interviews(videoFactory: @interviewVideoFactory)

diff --git a/app/src/Admin/Presenters/BasePresenter.php b/app/src/Admin/Presenters/BasePresenter.php
@@ -3,6 +3,8 @@
 
 namespace MichalSpacekCz\Admin\Presenters;
 
+use MichalSpacekCz\Http\Robots\Robots;
+use MichalSpacekCz\Http\Robots\RobotsRule;
 use MichalSpacekCz\Www\Presenters\BasePresenter as WwwBasePresenter;
 use Nette\Security\User;
 use Override;
@@ -13,6 +15,7 @@ abstract class BasePresenter extends WwwBasePresenter
 
 	private MysqlSessionHandler $sessionHandler;
 	private User $user;
+	private Robots $robots;
 
 	protected bool $haveBacklink = true;
 
@@ -37,10 +40,20 @@ public function injectUser(User $user): void
 	}
 
 
+	/**
+	 * @internal
+	 */
+	public function injectRobots(Robots $robots): void
+	{
+		$this->robots = $robots;
+	}
+
+
 	#[Override]
 	protected function startup(): void
 	{
 		parent::startup();
+		$this->robots->setHeader([RobotsRule::NoIndex, RobotsRule::NoFollow]);
 		if (!$this->user->isLoggedIn()) {
 			$params = ($this->haveBacklink ? ['backlink' => $this->storeRequest()] : []);
 			$this->redirect(':Admin:Sign:in', $params);

diff --git a/app/src/Admin/Presenters/templates/@layout.latte b/app/src/Admin/Presenters/templates/@layout.latte
@@ -1,7 +1,6 @@
 {layout "../../../Www/Presenters/templates/@layout.latte"}
 {var $containerExtraClass = 'admin'}
 {var $headerLinks = false}
-{var $robots = 'noindex, nofollow'}
 {define #feeds}{/define}
 {define #scriptsReplace}
 {script app + admin async, defer}

diff --git a/app/src/Api/Presenters/BasePresenter.php b/app/src/Api/Presenters/BasePresenter.php
@@ -0,0 +1,33 @@
+<?php
+declare(strict_types = 1);
+
+namespace MichalSpacekCz\Api\Presenters;
+
+use MichalSpacekCz\Http\Robots\Robots;
+use MichalSpacekCz\Http\Robots\RobotsRule;
+use MichalSpacekCz\Www\Presenters\BasePresenter as WwwBasePresenter;
+use Override;
+
+abstract class BasePresenter extends WwwBasePresenter
+{
+
+	private Robots $robots;
+
+
+	/**
+	 * @internal
+	 */
+	public function injectRobots(Robots $robots): void
+	{
+		$this->robots = $robots;
+	}
+
+
+	#[Override]
+	protected function startup(): void
+	{
+		parent::startup();
+		$this->robots->setHeader([RobotsRule::NoIndex, RobotsRule::NoFollow]);
+	}
+
+}
diff --git a/app/src/Api/Presenters/CertificatesPresenter.php b/app/src/Api/Presenters/CertificatesPresenter.php
@@ -9,7 +9,6 @@
 use MichalSpacekCz\Tls\Certificates;
 use MichalSpacekCz\Tls\Exceptions\CertificateException;
 use MichalSpacekCz\Tls\Exceptions\SomeCertificatesLoggedToFileException;
-use MichalSpacekCz\Www\Presenters\BasePresenter;
 use Nette\Security\AuthenticationException;
 use Override;
 use Tracy\Debugger;

diff --git a/app/src/Api/Presenters/CompanyPresenter.php b/app/src/Api/Presenters/CompanyPresenter.php
@@ -6,7 +6,6 @@
 use MichalSpacekCz\CompanyInfo\CompanyInfo;
 use MichalSpacekCz\Http\FetchMetadata\ResourceIsolationPolicyCrossSite;
 use MichalSpacekCz\Http\SecurityHeaders;
-use MichalSpacekCz\Www\Presenters\BasePresenter;
 use Nette\Application\BadRequestException;
 
 class CompanyPresenter extends BasePresenter

diff --git a/app/src/EasterEgg/FourOhFourButFound.php b/app/src/EasterEgg/FourOhFourButFound.php
@@ -3,6 +3,8 @@
 
 namespace MichalSpacekCz\EasterEgg;
 
+use MichalSpacekCz\Http\Robots\Robots;
+use MichalSpacekCz\Http\Robots\RobotsRule;
 use Nette\Application\Responses\TextResponse;
 use Nette\Application\UI\Presenter;
 use Nette\Http\IRequest;
@@ -17,6 +19,7 @@
 
 	public function __construct(
 		private IRequest $httpRequest,
+		private readonly Robots $robots,
 	) {
 	}
 
@@ -39,6 +42,7 @@ public function sendItMaybe(Presenter $presenter): void
 
 	private function sendIt(Presenter $presenter, string $template): never
 	{
+		$this->robots->setHeader([RobotsRule::NoIndex, RobotsRule::NoFollow]);
 		$presenter->sendResponse(new TextResponse(file_get_contents($template)));
 	}
 

diff --git a/app/src/EasterEgg/templates/etcPasswd.html b/app/src/EasterEgg/templates/etcPasswd.html
@@ -2,7 +2,6 @@
 <html lang="en">
 <head>
 	<meta charset="utf-8">
-	<meta name="robots" content="noindex, nofollow">
 	<meta name="viewport" content="width=device-width, initial-scale=1">
 	<title>/etc/passwd</title>
 </head>

diff --git a/app/src/Http/Robots/Robots.php b/app/src/Http/Robots/Robots.php
@@ -0,0 +1,26 @@
+<?php
+declare(strict_types = 1);
+
+namespace MichalSpacekCz\Http\Robots;
+
+use Nette\Http\IResponse;
+
+class Robots
+{
+
+	public function __construct(
+		private readonly IResponse $httpResponse,
+	) {
+	}
+
+
+	/**
+	 * @param list<RobotsRule> $rules
+	 */
+	public function setHeader(array $rules): void
+	{
+		$value = implode(', ', array_map(fn(RobotsRule $rule): string => $rule->value, $rules));
+		$this->httpResponse->addHeader('X-Robots-Tag', $value);
+	}
+
+}
diff --git a/app/src/Http/Robots/RobotsRule.php b/app/src/Http/Robots/RobotsRule.php
@@ -0,0 +1,12 @@
+<?php
+declare(strict_types = 1);
+
+namespace MichalSpacekCz\Http\Robots;
+
+enum RobotsRule: string
+{
+
+	case NoFollow = 'nofollow';
+	case NoIndex = 'noindex';
+
+}
diff --git a/app/src/Pulse/Presenters/ErrorPresenter.php b/app/src/Pulse/Presenters/ErrorPresenter.php
@@ -3,6 +3,8 @@
 
 namespace MichalSpacekCz\Pulse\Presenters;
 
+use MichalSpacekCz\Http\Robots\Robots;
+use MichalSpacekCz\Http\Robots\RobotsRule;
 use MichalSpacekCz\Www\Presenters\BaseErrorPresenter;
 use Nette\Application\Responses\TextResponse;
 
@@ -12,8 +14,16 @@ class ErrorPresenter extends BaseErrorPresenter
 	protected bool $logAccess = false;
 
 
+	public function __construct(
+		private readonly Robots $robots,
+	) {
+		parent::__construct();
+	}
+
+
 	public function actionDefault(): never
 	{
+		$this->robots->setHeader([RobotsRule::NoIndex, RobotsRule::NoFollow]);
 		$this->sendResponse(new TextResponse(file_get_contents(__DIR__ . '/templates/Error/notFound.html')));
 	}
 

diff --git a/app/src/Www/Presenters/TrainingsPresenter.php b/app/src/Www/Presenters/TrainingsPresenter.php
@@ -9,6 +9,8 @@
 use MichalSpacekCz\Form\TrainingApplicationPreliminaryFormFactory;
 use MichalSpacekCz\Form\UiForm;
 use MichalSpacekCz\Formatter\TexyFormatter;
+use MichalSpacekCz\Http\Robots\Robots;
+use MichalSpacekCz\Http\Robots\RobotsRule;
 use MichalSpacekCz\ShouldNotHappenException;
 use MichalSpacekCz\Training\Applications\TrainingApplications;
 use MichalSpacekCz\Training\Applications\TrainingApplicationSessionSection;
@@ -61,6 +63,7 @@ public function __construct(
 		private readonly TrainingFilesDownload $trainingFilesDownload,
 		private readonly Translator $translator,
 		private readonly Session $sessionHandler,
+		private readonly Robots $robots,
 	) {
 		parent::__construct();
 	}
@@ -228,6 +231,7 @@ public function actionFiles(string $name, ?string $param): void
 		if (count($application->getFiles()) === 0) {
 			throw new BadRequestException('No files for application id ' . $application->getId());
 		}
+		$this->robots->setHeader([RobotsRule::NoIndex, RobotsRule::NoFollow]);
 
 		$this->template->trainingTitle = $training->getName();
 		$this->template->trainingName = ($training->isCustom() ? null : $training->getAction());

diff --git a/app/src/Www/Presenters/templates/Trainings/files.latte b/app/src/Www/Presenters/templates/Trainings/files.latte
@@ -1,5 +1,4 @@
 {varType MichalSpacekCz\Training\Files\TrainingFile[] $files}
-{var $robots = 'noindex, nofollow'}
 
 {define #menu}
 &raquo; <a n:href="Homepage:">Michal Špaček</a>

diff --git a/app/tests/Http/Robots/RobotsTest.phpt b/app/tests/Http/Robots/RobotsTest.phpt
@@ -0,0 +1,35 @@
+<?php
+declare(strict_types = 1);
+
+namespace MichalSpacekCz\Http\Robots;
+
+use MichalSpacekCz\Test\Http\Response;
+use MichalSpacekCz\Test\TestCaseRunner;
+use Tester\Assert;
+use Tester\TestCase;
+
+require __DIR__ . '/../../bootstrap.php';
+
+/** @testCase */
+class RobotsTest extends TestCase
+{
+
+	public function __construct(
+		private readonly Response $httpResponse,
+		private readonly Robots $robots,
+	) {
+	}
+
+
+	public function testSetHeader(): void
+	{
+		$this->robots->setHeader([RobotsRule::NoIndex]);
+		Assert::same('noindex', $this->httpResponse->getHeader('X-Robots-Tag'));
+
+		$this->robots->setHeader([RobotsRule::NoIndex, RobotsRule::NoFollow]);
+		Assert::same('noindex, nofollow', $this->httpResponse->getHeader('X-Robots-Tag'));
+	}
+
+}
+
+TestCaseRunner::run(RobotsTest::class);