merge: Adding robots.txt override via admin control panel for #418 (!877)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/877

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Hazelnoot <acomputerdog@gmail.com>

7 files changed, 42 insertions, 1 deletions

diff --git a/packages/backend/migration/1738346484187-robotsTxt.js b/packages/backend/migration/1738346484187-robotsTxt.js
new file mode 100644
index 0000000000..00ea1fb030
--- /dev/null
+++ b/packages/backend/migration/1738346484187-robotsTxt.js
@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: marie and other Sharkey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+export class RobotsTxt1738346484187 {
+    name = 'RobotsTxt1738346484187'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" ADD "robotsTxt" text`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "robotsTxt"`);
+    }
+}
diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts
index 7d7b4cbd81..857e8f5a7b 100644
--- a/packages/backend/src/core/entities/MetaEntityService.ts
+++ b/packages/backend/src/core/entities/MetaEntityService.ts
@@ -95,6 +95,7 @@ export class MetaEntityService {
 			mcaptchaInstanceUrl: instance.mcaptchaInstanceUrl,
 			enableRecaptcha: instance.enableRecaptcha,
 			enableAchievements: instance.enableAchievements,
+			robotsTxt: instance.robotsTxt,
 			recaptchaSiteKey: instance.recaptchaSiteKey,
 			enableTurnstile: instance.enableTurnstile,
 			turnstileSiteKey: instance.turnstileSiteKey,
diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts
index 3fc3f273dd..a224117676 100644
--- a/packages/backend/src/models/Meta.ts
+++ b/packages/backend/src/models/Meta.ts
@@ -599,6 +599,11 @@ export class MiMeta {
 	})
 	public enableAchievements: boolean;
 
+	@Column('varchar', {
+		length: 2048, nullable: true,
+	})
+	public robotsTxt: string | null;
+
 	@Column('jsonb', {
 		default: { },
 	})
diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts
index 5179e5d51c..29fdb4f6be 100644
--- a/packages/backend/src/models/json-schema/meta.ts
+++ b/packages/backend/src/models/json-schema/meta.ts
@@ -139,6 +139,10 @@ export const packedMetaLiteSchema = {
 			type: 'boolean',
 			optional: false, nullable: true,
 		},
+		robotsTxt: {
+			type: 'string',
+			optional: false, nullable: true,
+		},
 		enableTestcaptcha: {
 			type: 'boolean',
 			optional: false, nullable: false,
diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts
index 6495e3b7da..436dcf27cb 100644
--- a/packages/backend/src/server/api/endpoints/admin/meta.ts
+++ b/packages/backend/src/server/api/endpoints/admin/meta.ts
@@ -391,6 +391,10 @@ export const meta = {
 				type: 'boolean',
 				optional: false, nullable: false,
 			},
+			robotsTxt: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 			enableIdenticonGeneration: {
 				type: 'boolean',
 				optional: false, nullable: false,
@@ -708,6 +712,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				enableStatsForFederatedInstances: instance.enableStatsForFederatedInstances,
 				enableServerMachineStats: instance.enableServerMachineStats,
 				enableAchievements: instance.enableAchievements,
+				robotsTxt: instance.robotsTxt,
 				enableIdenticonGeneration: instance.enableIdenticonGeneration,
 				bannedEmailDomains: instance.bannedEmailDomains,
 				policies: { ...DEFAULT_POLICIES, ...instance.policies },
diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts
index 72f428d85f..b3733d3d39 100644
--- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts
+++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts
@@ -149,6 +149,7 @@ export const paramDef = {
 		enableStatsForFederatedInstances: { type: 'boolean' },
 		enableServerMachineStats: { type: 'boolean' },
 		enableAchievements: { type: 'boolean' },
+		robotsTxt: { type: 'string', nullable: true },
 		enableIdenticonGeneration: { type: 'boolean' },
 		serverRules: { type: 'array', items: { type: 'string' } },
 		bannedEmailDomains: { type: 'array', items: { type: 'string' } },
@@ -636,6 +637,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.enableAchievements = ps.enableAchievements;
 			}
 
+			if (ps.robotsTxt !== undefined) {
+				set.robotsTxt = ps.robotsTxt;
+			}
+
 			if (ps.enableIdenticonGeneration !== undefined) {
 				set.enableIdenticonGeneration = ps.enableIdenticonGeneration;
 			}
diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts
index e59314bf55..ae923ada1f 100644
--- a/packages/backend/src/server/web/ClientServerService.ts
+++ b/packages/backend/src/server/web/ClientServerService.ts
@@ -488,7 +488,12 @@ export class ClientServerService {
 		});
 
 		fastify.get('/robots.txt', async (request, reply) => {
-			return await reply.sendFile('/robots.txt', staticAssets);
+			if (this.meta.robotsTxt) {
+				reply.header('Content-Type', 'text/plain');
+				return await reply.send(this.meta.robotsTxt);
+			} else {
+				return await reply.sendFile('/robots.txt', staticAssets);
+			}
 		});
 
 		// OpenSearch XML