merge: Set X-Robots-Tag to disable indexing API endpoints (resolves #756) (!1119)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/1119

Closes #756

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Marie <github@yuugi.dev>

5 files changed, 23 insertions, 0 deletions

diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts
index 41beadb56d..a362308b17 100644
--- a/packages/backend/src/server/ActivityPubServerService.ts
+++ b/packages/backend/src/server/ActivityPubServerService.ts
@@ -791,6 +791,10 @@ export class ActivityPubServerService {
 			reply.header('Access-Control-Allow-Origin', '*');
 			reply.header('Access-Control-Expose-Headers', 'Vary');
 
+			// Tell crawlers not to index AP endpoints.
+			// https://developers.google.com/search/docs/crawling-indexing/block-indexing
+			reply.header('X-Robots-Tag', 'noindex');
+
 			/* tell any caching proxy that they should not cache these
 				 responses: we wouldn't want the proxy to return a 403 to
 				 someone presenting a valid signature, or return a cached
diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts
index 1a372cb789..0910c0d36b 100644
--- a/packages/backend/src/server/FileServerService.ts
+++ b/packages/backend/src/server/FileServerService.ts
@@ -70,6 +70,10 @@ export class FileServerService {
 		fastify.addHook('onRequest', (request, reply, done) => {
 			reply.header('Content-Security-Policy', 'default-src \'none\'; img-src \'self\'; media-src \'self\'; style-src \'unsafe-inline\'');
 			reply.header('Access-Control-Allow-Origin', '*');
+
+			// Tell crawlers not to index files endpoints.
+			// https://developers.google.com/search/docs/crawling-indexing/block-indexing
+			reply.header('X-Robots-Tag', 'noindex');
 			done();
 		});
 
diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts
index 6d6c86bb82..66d968224a 100644
--- a/packages/backend/src/server/api/ApiCallService.ts
+++ b/packages/backend/src/server/api/ApiCallService.ts
@@ -148,6 +148,10 @@ export class ApiCallService implements OnApplicationShutdown {
 		request: FastifyRequest<{ Body: Record<string, unknown> | undefined, Querystring: Record<string, unknown> }>,
 		reply: FastifyReply,
 	): void {
+		// Tell crawlers not to index API endpoints.
+		// https://developers.google.com/search/docs/crawling-indexing/block-indexing
+		reply.header('X-Robots-Tag', 'noindex');
+
 		const body = request.method === 'GET'
 			? request.query
 			: request.body;
diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts
index 74fd9d7d59..072dacf708 100644
--- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts
+++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts
@@ -71,6 +71,13 @@ export class MastodonApiServerService {
 			done();
 		});
 
+		// Tell crawlers not to index API endpoints.
+		// https://developers.google.com/search/docs/crawling-indexing/block-indexing
+		fastify.addHook('onRequest', (request, reply, done) => {
+			reply.header('X-Robots-Tag', 'noindex');
+			done();
+		});
+
 		// External endpoints
 		this.apiAccountMastodon.register(fastify);
 		this.apiAppsMastodon.register(fastify);
diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts
index ed5d87d15d..71a142fc6f 100644
--- a/packages/backend/src/server/web/UrlPreviewService.ts
+++ b/packages/backend/src/server/web/UrlPreviewService.ts
@@ -125,6 +125,10 @@ export class UrlPreviewService {
 		reply: FastifyReply,
 	): Promise<void> {
 		if (!this.meta.urlPreviewEnabled) {
+			// Tell crawlers not to index URL previews.
+			// https://developers.google.com/search/docs/crawling-indexing/block-indexing
+			reply.header('X-Robots-Tag', 'noindex');
+
 			return reply.code(403).send({
 				error: {
 					message: 'URL preview is disabled',