From ce08bd1b425f14a1ca80287ca4cf43deeb32ea76 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 16 Mar 2025 12:37:46 -0400 Subject: reduce, clarify, and normalize more error messages --- packages/backend/src/server/FileServerService.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'packages/backend/src/server/FileServerService.ts') diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 4ef5539cff..2e3e8df7ea 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -32,6 +32,7 @@ import { getIpHash } from '@/misc/get-ip-hash.js'; import { AuthenticateService } from '@/server/api/AuthenticateService.js'; import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify'; const _filename = fileURLToPath(import.meta.url); @@ -120,7 +121,7 @@ export class FileServerService { @bindThis private async errorHandler(request: FastifyRequest<{ Params?: { [x: string]: any }; Querystring?: { [x: string]: any }; }>, reply: FastifyReply, err?: any) { - this.logger.error(`${err}`); + this.logger.error(`Unhandled error in file server: ${renderInlineError(err)}`); reply.header('Cache-Control', 'max-age=300'); -- cgit v1.2.3-freya From 75b85eb1651acb891a8a4c577ffcdc983e6469c5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 4 May 2025 10:51:37 -0400 Subject: record details of recursive proxy rejections --- packages/backend/src/server/FileServerService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/backend/src/server/FileServerService.ts') diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 2e3e8df7ea..e5d8f39e5a 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -354,7 +354,7 @@ export class FileServerService { if (!request.headers['user-agent']) { throw new StatusError('User-Agent is required', 400, 'User-Agent is required'); } else if (request.headers['user-agent'].toLowerCase().indexOf('misskey/') !== -1) { - throw new StatusError('Refusing to proxy a request from another proxy', 403, 'Proxy is recursive'); + throw new StatusError(`Refusing to proxy recursive request to ${url} (from user-agent ${request.headers['user-agent']})`, 403, 'Proxy is recursive'); } // Create temp file -- cgit v1.2.3-freya From 6627e8a9b848473740a8e9b9864520701226179c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 22 May 2025 10:53:38 -0400 Subject: add more details to StatusErrors --- packages/backend/src/core/DownloadService.ts | 2 +- packages/backend/src/core/HttpRequestService.ts | 2 +- packages/backend/src/misc/render-inline-error.ts | 2 ++ packages/backend/src/server/FileServerService.ts | 6 +++--- 4 files changed, 7 insertions(+), 5 deletions(-) (limited to 'packages/backend/src/server/FileServerService.ts') diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 5d0aaa9945..cb5bdb6cb7 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -101,7 +101,7 @@ export class DownloadService { await stream.pipeline(req, fs.createWriteStream(path)); } catch (e) { if (e instanceof Got.HTTPError) { - throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage, e); + throw new StatusError(`download error from ${url}`, e.response.statusCode, e.response.statusMessage, e); } else if (e instanceof Got.RequestError || e instanceof Got.AbortError) { throw new Error(String(e), { cause: e }); } else if (e instanceof Error) { diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index a0f2607ddc..151097095d 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -331,7 +331,7 @@ export class HttpRequestService { }); if (!res.ok && extra.throwErrorWhenResponseNotOk) { - throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); + throw new StatusError(`request error from ${url}`, res.status, res.statusText); } if (res.ok) { diff --git a/packages/backend/src/misc/render-inline-error.ts b/packages/backend/src/misc/render-inline-error.ts index 36de0262ef..9536d5ea0e 100644 --- a/packages/backend/src/misc/render-inline-error.ts +++ b/packages/backend/src/misc/render-inline-error.ts @@ -18,6 +18,8 @@ export function renderInlineError(err: unknown): string { if (err instanceof StatusError) { if (err.message) { return `${err.name} ${err.statusCode}: ${err.message}`; + } else if (err.statusMessage) { + return `${err.name} ${err.statusCode}: ${err.statusMessage}`; } else { return `${err.name} ${err.statusCode}`; } diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index e5d8f39e5a..1a372cb789 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -384,7 +384,7 @@ export class FileServerService { ) { if (!isConvertibleImage) { // 画像でないなら404でお茶を濁す - throw new StatusError('Unexpected mime', 404); + throw new StatusError(`Unexpected non-convertible mime: ${file.mime}`, 404, 'Unexpected mime'); } } @@ -448,7 +448,7 @@ export class FileServerService { } else if (file.mime === 'image/svg+xml') { image = this.imageProcessingService.convertToWebpStream(file.path, 2048, 2048); } else if (!file.mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(file.mime)) { - throw new StatusError('Rejected type', 403, 'Rejected type'); + throw new StatusError(`Blocked mime type: ${file.mime}`, 403, 'Blocked mime type'); } if (!image) { @@ -522,7 +522,7 @@ export class FileServerService { > { if (url.startsWith(`${this.config.url}/files/`)) { const key = url.replace(`${this.config.url}/files/`, '').split('/').shift(); - if (!key) throw new StatusError('Invalid File Key', 400, 'Invalid File Key'); + if (!key) throw new StatusError(`Invalid file URL ${url}`, 400, 'Invalid file url'); return await this.getFileFromKey(key); } -- cgit v1.2.3-freya From b44abe0eaaeb85111d94046b182e03e67993a101 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 13 Jun 2025 23:18:06 -0400 Subject: set X-Robots-Tag to disable indexing API endpoints --- packages/backend/src/server/ActivityPubServerService.ts | 4 ++++ packages/backend/src/server/FileServerService.ts | 4 ++++ packages/backend/src/server/api/ApiCallService.ts | 4 ++++ .../backend/src/server/api/mastodon/MastodonApiServerService.ts | 7 +++++++ packages/backend/src/server/web/UrlPreviewService.ts | 4 ++++ 5 files changed, 23 insertions(+) (limited to 'packages/backend/src/server/FileServerService.ts') diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 41beadb56d..a362308b17 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -791,6 +791,10 @@ export class ActivityPubServerService { reply.header('Access-Control-Allow-Origin', '*'); reply.header('Access-Control-Expose-Headers', 'Vary'); + // Tell crawlers not to index AP endpoints. + // https://developers.google.com/search/docs/crawling-indexing/block-indexing + reply.header('X-Robots-Tag', 'noindex'); + /* tell any caching proxy that they should not cache these responses: we wouldn't want the proxy to return a 403 to someone presenting a valid signature, or return a cached diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 1a372cb789..0910c0d36b 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -70,6 +70,10 @@ export class FileServerService { fastify.addHook('onRequest', (request, reply, done) => { reply.header('Content-Security-Policy', 'default-src \'none\'; img-src \'self\'; media-src \'self\'; style-src \'unsafe-inline\''); reply.header('Access-Control-Allow-Origin', '*'); + + // Tell crawlers not to index files endpoints. + // https://developers.google.com/search/docs/crawling-indexing/block-indexing + reply.header('X-Robots-Tag', 'noindex'); done(); }); diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 6d6c86bb82..66d968224a 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -148,6 +148,10 @@ export class ApiCallService implements OnApplicationShutdown { request: FastifyRequest<{ Body: Record | undefined, Querystring: Record }>, reply: FastifyReply, ): void { + // Tell crawlers not to index API endpoints. + // https://developers.google.com/search/docs/crawling-indexing/block-indexing + reply.header('X-Robots-Tag', 'noindex'); + const body = request.method === 'GET' ? request.query : request.body; diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 74fd9d7d59..072dacf708 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -71,6 +71,13 @@ export class MastodonApiServerService { done(); }); + // Tell crawlers not to index API endpoints. + // https://developers.google.com/search/docs/crawling-indexing/block-indexing + fastify.addHook('onRequest', (request, reply, done) => { + reply.header('X-Robots-Tag', 'noindex'); + done(); + }); + // External endpoints this.apiAccountMastodon.register(fastify); this.apiAppsMastodon.register(fastify); diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index ed5d87d15d..71a142fc6f 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -125,6 +125,10 @@ export class UrlPreviewService { reply: FastifyReply, ): Promise { if (!this.meta.urlPreviewEnabled) { + // Tell crawlers not to index URL previews. + // https://developers.google.com/search/docs/crawling-indexing/block-indexing + reply.header('X-Robots-Tag', 'noindex'); + return reply.code(403).send({ error: { message: 'URL preview is disabled', -- cgit v1.2.3-freya