Adding robots.txt override via admin control panel

This is a requested low priority feature in #418 - I created the changes
to follow similarly to how the Instance Description is handled.

8 files changed, 51 insertions, 1 deletions

diff --git a/packages/backend/migration/1738098171990-robotsTxt.js b/packages/backend/migration/1738098171990-robotsTxt.js
new file mode 100644
index 0000000000..947f21cc46
--- /dev/null
+++ b/packages/backend/migration/1738098171990-robotsTxt.js
@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: marie and other Sharkey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+export class RobotsTxt1738098171990 {
+    name = 'RobotsTxt1738098171990'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" ADD "robotsTxt" character varying(2048)`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "robotsTxt"`);
+    }
+}
diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts
index 7d7b4cbd81..857e8f5a7b 100644
--- a/packages/backend/src/core/entities/MetaEntityService.ts
+++ b/packages/backend/src/core/entities/MetaEntityService.ts
@@ -95,6 +95,7 @@ export class MetaEntityService {
 			mcaptchaInstanceUrl: instance.mcaptchaInstanceUrl,
 			enableRecaptcha: instance.enableRecaptcha,
 			enableAchievements: instance.enableAchievements,
+			robotsTxt: instance.robotsTxt,
 			recaptchaSiteKey: instance.recaptchaSiteKey,
 			enableTurnstile: instance.enableTurnstile,
 			turnstileSiteKey: instance.turnstileSiteKey,
diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts
index 3fc3f273dd..a224117676 100644
--- a/packages/backend/src/models/Meta.ts
+++ b/packages/backend/src/models/Meta.ts
@@ -599,6 +599,11 @@ export class MiMeta {
 	})
 	public enableAchievements: boolean;
 
+	@Column('varchar', {
+		length: 2048, nullable: true,
+	})
+	public robotsTxt: string | null;
+
 	@Column('jsonb', {
 		default: { },
 	})
diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts
index 5179e5d51c..29fdb4f6be 100644
--- a/packages/backend/src/models/json-schema/meta.ts
+++ b/packages/backend/src/models/json-schema/meta.ts
@@ -139,6 +139,10 @@ export const packedMetaLiteSchema = {
 			type: 'boolean',
 			optional: false, nullable: true,
 		},
+		robotsTxt: {
+			type: 'string',
+			optional: false, nullable: true,
+		},
 		enableTestcaptcha: {
 			type: 'boolean',
 			optional: false, nullable: false,
diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts
index 6495e3b7da..436dcf27cb 100644
--- a/packages/backend/src/server/api/endpoints/admin/meta.ts
+++ b/packages/backend/src/server/api/endpoints/admin/meta.ts
@@ -391,6 +391,10 @@ export const meta = {
 				type: 'boolean',
 				optional: false, nullable: false,
 			},
+			robotsTxt: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 			enableIdenticonGeneration: {
 				type: 'boolean',
 				optional: false, nullable: false,
@@ -708,6 +712,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				enableStatsForFederatedInstances: instance.enableStatsForFederatedInstances,
 				enableServerMachineStats: instance.enableServerMachineStats,
 				enableAchievements: instance.enableAchievements,
+				robotsTxt: instance.robotsTxt,
 				enableIdenticonGeneration: instance.enableIdenticonGeneration,
 				bannedEmailDomains: instance.bannedEmailDomains,
 				policies: { ...DEFAULT_POLICIES, ...instance.policies },
diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts
index 72f428d85f..b3733d3d39 100644
--- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts
+++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts
@@ -149,6 +149,7 @@ export const paramDef = {
 		enableStatsForFederatedInstances: { type: 'boolean' },
 		enableServerMachineStats: { type: 'boolean' },
 		enableAchievements: { type: 'boolean' },
+		robotsTxt: { type: 'string', nullable: true },
 		enableIdenticonGeneration: { type: 'boolean' },
 		serverRules: { type: 'array', items: { type: 'string' } },
 		bannedEmailDomains: { type: 'array', items: { type: 'string' } },
@@ -636,6 +637,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.enableAchievements = ps.enableAchievements;
 			}
 
+			if (ps.robotsTxt !== undefined) {
+				set.robotsTxt = ps.robotsTxt;
+			}
+
 			if (ps.enableIdenticonGeneration !== undefined) {
 				set.enableIdenticonGeneration = ps.enableIdenticonGeneration;
 			}
diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts
index e59314bf55..e93900b358 100644
--- a/packages/backend/src/server/web/ClientServerService.ts
+++ b/packages/backend/src/server/web/ClientServerService.ts
@@ -488,7 +488,14 @@ export class ClientServerService {
 		});
 
 		fastify.get('/robots.txt', async (request, reply) => {
-			return await reply.sendFile('/robots.txt', staticAssets);
+			if (this.meta.robotsTxt) {
+				let content = '';
+				content += this.meta.robotsTxt;
+				reply.header('Content-Type', 'text/plain');
+				return await reply.send(content);
+			} else {
+				return await reply.sendFile('/robots.txt', staticAssets);
+			}
 		});
 
 		// OpenSearch XML
diff --git a/packages/frontend/src/pages/admin/settings.vue b/packages/frontend/src/pages/admin/settings.vue
index 68f211de5c..cd05b43be8 100644
--- a/packages/frontend/src/pages/admin/settings.vue
+++ b/packages/frontend/src/pages/admin/settings.vue
@@ -159,6 +159,11 @@ SPDX-License-Identifier: AGPL-3.0-only
 							<template #label>{{ i18n.ts.enableBotTrending }}<span v-if="otherForm.modifiedStates.enableBotTrending" class="_modified">{{ i18n.ts.modified }}</span></template>
 							<template #caption>{{ i18n.ts.turnOffBotTrending }}</template>
 						</MkSwitch>
+
+						<MkTextarea v-model="otherForm.state.robotsTxt">
+							<template #label>{{ i18n.ts.robotsTxt }}<span v-if="otherForm.modifiedStates.robotsTxt" class="_modified">{{ i18n.ts.modified }}</span></template>
+							<template #caption>{{ i18n.ts.robotsTxtDescription }}</template>
+						</MkTextarea>
 					</div>
 				</MkFolder>
 
@@ -369,10 +374,12 @@ const serviceWorkerForm = useForm({
 const otherForm = useForm({
 	enableAchievements: meta.enableAchievements,
 	enableBotTrending: meta.enableBotTrending,
+	robotsTxt: meta.robotsTxt,
 }, async (state) => {
 	await os.apiWithDialog('admin/update-meta', {
 		enableAchievements: state.enableAchievements,
 		enableBotTrending: state.enableBotTrending,
+		robotsTxt: state.robotsTxt,
 	});
 	fetchInstance(true);
 });