From 993532bc1fe0f67d84e16a99ee916f7fff9b0935 Mon Sep 17 00:00:00 2001 From: Kinetix Date: Tue, 28 Jan 2025 15:57:45 -0800 Subject: Adding robots.txt override via admin control panel This is a requested low priority feature in #418 - I created the changes to follow similarly to how the Instance Description is handled. --- packages/backend/src/core/entities/MetaEntityService.ts | 1 + packages/backend/src/models/Meta.ts | 5 +++++ packages/backend/src/models/json-schema/meta.ts | 4 ++++ packages/backend/src/server/api/endpoints/admin/meta.ts | 5 +++++ packages/backend/src/server/api/endpoints/admin/update-meta.ts | 5 +++++ packages/backend/src/server/web/ClientServerService.ts | 9 ++++++++- 6 files changed, 28 insertions(+), 1 deletion(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index 7d7b4cbd81..857e8f5a7b 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -95,6 +95,7 @@ export class MetaEntityService { mcaptchaInstanceUrl: instance.mcaptchaInstanceUrl, enableRecaptcha: instance.enableRecaptcha, enableAchievements: instance.enableAchievements, + robotsTxt: instance.robotsTxt, recaptchaSiteKey: instance.recaptchaSiteKey, enableTurnstile: instance.enableTurnstile, turnstileSiteKey: instance.turnstileSiteKey, diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 3fc3f273dd..a224117676 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -599,6 +599,11 @@ export class MiMeta { }) public enableAchievements: boolean; + @Column('varchar', { + length: 2048, nullable: true, + }) + public robotsTxt: string | null; + @Column('jsonb', { default: { }, }) diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts index 5179e5d51c..29fdb4f6be 100644 --- a/packages/backend/src/models/json-schema/meta.ts +++ b/packages/backend/src/models/json-schema/meta.ts @@ -139,6 +139,10 @@ export const packedMetaLiteSchema = { type: 'boolean', optional: false, nullable: true, }, + robotsTxt: { + type: 'string', + optional: false, nullable: true, + }, enableTestcaptcha: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 6495e3b7da..436dcf27cb 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -391,6 +391,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + robotsTxt: { + type: 'string', + optional: false, nullable: true, + }, enableIdenticonGeneration: { type: 'boolean', optional: false, nullable: false, @@ -708,6 +712,7 @@ export default class extends Endpoint { // eslint- enableStatsForFederatedInstances: instance.enableStatsForFederatedInstances, enableServerMachineStats: instance.enableServerMachineStats, enableAchievements: instance.enableAchievements, + robotsTxt: instance.robotsTxt, enableIdenticonGeneration: instance.enableIdenticonGeneration, bannedEmailDomains: instance.bannedEmailDomains, policies: { ...DEFAULT_POLICIES, ...instance.policies }, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 72f428d85f..b3733d3d39 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -149,6 +149,7 @@ export const paramDef = { enableStatsForFederatedInstances: { type: 'boolean' }, enableServerMachineStats: { type: 'boolean' }, enableAchievements: { type: 'boolean' }, + robotsTxt: { type: 'string', nullable: true }, enableIdenticonGeneration: { type: 'boolean' }, serverRules: { type: 'array', items: { type: 'string' } }, bannedEmailDomains: { type: 'array', items: { type: 'string' } }, @@ -636,6 +637,10 @@ export default class extends Endpoint { // eslint- set.enableAchievements = ps.enableAchievements; } + if (ps.robotsTxt !== undefined) { + set.robotsTxt = ps.robotsTxt; + } + if (ps.enableIdenticonGeneration !== undefined) { set.enableIdenticonGeneration = ps.enableIdenticonGeneration; } diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index e59314bf55..e93900b358 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -488,7 +488,14 @@ export class ClientServerService { }); fastify.get('/robots.txt', async (request, reply) => { - return await reply.sendFile('/robots.txt', staticAssets); + if (this.meta.robotsTxt) { + let content = ''; + content += this.meta.robotsTxt; + reply.header('Content-Type', 'text/plain'); + return await reply.send(content); + } else { + return await reply.sendFile('/robots.txt', staticAssets); + } }); // OpenSearch XML -- cgit v1.2.3-freya From 4d91baaa13a8c4e8b9df221ce710d7005102c5a3 Mon Sep 17 00:00:00 2001 From: Kinetix Date: Fri, 31 Jan 2025 10:22:13 -0800 Subject: Adjust as per suggestions in !877 --- packages/backend/migration/1738098171990-robotsTxt.js | 16 ---------------- packages/backend/migration/1738346484187-robotsTxt.js | 16 ++++++++++++++++ packages/backend/src/server/web/ClientServerService.ts | 4 +--- 3 files changed, 17 insertions(+), 19 deletions(-) delete mode 100644 packages/backend/migration/1738098171990-robotsTxt.js create mode 100644 packages/backend/migration/1738346484187-robotsTxt.js (limited to 'packages/backend/src') diff --git a/packages/backend/migration/1738098171990-robotsTxt.js b/packages/backend/migration/1738098171990-robotsTxt.js deleted file mode 100644 index 947f21cc46..0000000000 --- a/packages/backend/migration/1738098171990-robotsTxt.js +++ /dev/null @@ -1,16 +0,0 @@ -/* - * SPDX-FileCopyrightText: marie and other Sharkey contributors - * SPDX-License-Identifier: AGPL-3.0-only - */ - -export class RobotsTxt1738098171990 { - name = 'RobotsTxt1738098171990' - - async up(queryRunner) { - await queryRunner.query(`ALTER TABLE "meta" ADD "robotsTxt" character varying(2048)`); - } - - async down(queryRunner) { - await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "robotsTxt"`); - } -} diff --git a/packages/backend/migration/1738346484187-robotsTxt.js b/packages/backend/migration/1738346484187-robotsTxt.js new file mode 100644 index 0000000000..00ea1fb030 --- /dev/null +++ b/packages/backend/migration/1738346484187-robotsTxt.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: marie and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class RobotsTxt1738346484187 { + name = 'RobotsTxt1738346484187' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "robotsTxt" text`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "robotsTxt"`); + } +} diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index e93900b358..ae923ada1f 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -489,10 +489,8 @@ export class ClientServerService { fastify.get('/robots.txt', async (request, reply) => { if (this.meta.robotsTxt) { - let content = ''; - content += this.meta.robotsTxt; reply.header('Content-Type', 'text/plain'); - return await reply.send(content); + return await reply.send(this.meta.robotsTxt); } else { return await reply.sendFile('/robots.txt', staticAssets); } -- cgit v1.2.3-freya