From ef860a8f844977aea9043f83dbbf761c3da70727 Mon Sep 17 00:00:00 2001 From: RyotaK <49341894+Ry0taK@users.noreply.github.com> Date: Sun, 12 Feb 2023 08:21:13 +0900 Subject: CLIページにおいてAPIリクエスト時にContent-Typeヘッダを付与するように変更 (#9887) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/server/web/cli.js | 3 +++ 1 file changed, 3 insertions(+) (limited to 'packages/backend/src/server') diff --git a/packages/backend/src/server/web/cli.js b/packages/backend/src/server/web/cli.js index 3dff1d4860..3467f7ac2a 100644 --- a/packages/backend/src/server/web/cli.js +++ b/packages/backend/src/server/web/cli.js @@ -11,6 +11,9 @@ window.onload = async () => { // Send request fetch(endpoint.indexOf('://') > -1 ? endpoint : `/api/${endpoint}`, { + headers: { + 'Content-Type': 'application/json' + }, method: 'POST', body: JSON.stringify(data), credentials: 'omit', -- cgit v1.2.3-freya From f28aea9e303af96978da5e27739b5fd8e23b8f86 Mon Sep 17 00:00:00 2001 From: momoirodouhu Date: Sun, 12 Feb 2023 08:22:42 +0900 Subject: add cors header to ActivityPubServerService.ts (#9888) * add cors header to ActivityPubServerService.ts * Update CHANGELOG.md --- CHANGELOG.md | 1 + packages/backend/src/server/ActivityPubServerService.ts | 8 ++++++++ 2 files changed, 9 insertions(+) (limited to 'packages/backend/src/server') diff --git a/CHANGELOG.md b/CHANGELOG.md index c11d186926..7d0fbd0a23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ You should also include the user name that made the change. ### Improvements - アニメーションを少なくする設定の時、MkPageHeaderのタブアニメーションを無効化 +- Backend: activitypub情報がcorsでブロックされないようヘッダーを追加 ### Bugfixes - Client: ユーザーページでアクティビティを見ることができない問題を修正 diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 186d3822d8..5480395eeb 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -441,6 +441,14 @@ export class ActivityPubServerService { fastify.addContentTypeParser('application/activity+json', { parseAs: 'string' }, fastify.getDefaultJsonParser('ignore', 'ignore')); fastify.addContentTypeParser('application/ld+json', { parseAs: 'string' }, fastify.getDefaultJsonParser('ignore', 'ignore')); + fastify.addHook('onRequest', (request, reply, done) => { + reply.header('Access-Control-Allow-Headers', 'Accept'); + reply.header('Access-Control-Allow-Methods', 'GET, OPTIONS'); + reply.header('Access-Control-Allow-Origin', '*'); + reply.header('Access-Control-Expose-Headers', 'Vary'); + done(); + }); + //#region Routing // inbox (limit: 64kb) fastify.post('/inbox', { bodyLimit: 1024 * 64 }, async (request, reply) => await this.inbox(request, reply)); -- cgit v1.2.3-freya From ee03ab8d2c06254480e8b78d3c4eab4a78409ad5 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 12 Feb 2023 09:13:47 +0900 Subject: enhance(server): videoThumbnailGenerator config (#9845) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enhance(server): videoThumbnailGenerator config * :v: * fix * 相対url * サムネイルのproxyRemoteFilesは直接プロキシを指定する * メディアプロキシ --- .config/example.yml | 9 ++++ packages/backend/src/config.ts | 6 +++ packages/backend/src/core/DriveService.ts | 8 ++++ .../backend/src/core/VideoProcessingService.ts | 14 ++++++ .../src/core/entities/DriveFileEntityService.ts | 53 ++++++++++++++++------ packages/backend/src/server/FileServerService.ts | 6 +++ 6 files changed, 82 insertions(+), 14 deletions(-) (limited to 'packages/backend/src/server') diff --git a/.config/example.yml b/.config/example.yml index a19b5d04e8..92b8726623 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -131,11 +131,20 @@ proxyBypassHosts: # Media Proxy # Reference Implementation: https://github.com/misskey-dev/media-proxy +# * Deliver a common cache between instances +# * Perform image compression (on a different server resource than the main process) #mediaProxy: https://example.com/proxy # Proxy remote files (default: false) +# Proxy remote files by this instance or mediaProxy to prevent remote files from running in remote domains. #proxyRemoteFiles: true +# Movie Thumbnail Generation URL +# There is no reference implementation. +# For example, Misskey will point to the following URL: +# https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4 +#videoThumbnailGenerator: https://example.com + # Sign to ActivityPub GET request (default: true) signToActivityPubGet: true diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index aa98ef1d22..73f45e92e1 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -67,6 +67,7 @@ export type Source = { mediaProxy?: string; proxyRemoteFiles?: boolean; + videoThumbnailGenerator?: string; signToActivityPubGet?: boolean; }; @@ -89,6 +90,7 @@ export type Mixin = { clientManifestExists: boolean; mediaProxy: string; externalMediaProxyEnabled: boolean; + videoThumbnailGenerator: string | null; }; export type Config = Source & Mixin; @@ -144,6 +146,10 @@ export function loadConfig() { mixin.mediaProxy = externalMediaProxy ?? internalMediaProxy; mixin.externalMediaProxyEnabled = externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy; + mixin.videoThumbnailGenerator = config.videoThumbnailGenerator ? + config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator + : null; + if (!config.redis.prefix) config.redis.prefix = mixin.host; return Object.assign(config, mixin); diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 598a457e83..42a430ea75 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -250,6 +250,14 @@ export class DriveService { @bindThis public async generateAlts(path: string, type: string, generateWeb: boolean) { if (type.startsWith('video/')) { + if (this.config.videoThumbnailGenerator != null) { + // videoThumbnailGeneratorが指定されていたら動画サムネイル生成はスキップ + return { + webpublic: null, + thumbnail: null, + } + } + try { const thumbnail = await this.videoProcessingService.generateVideoThumbnail(path); return { diff --git a/packages/backend/src/core/VideoProcessingService.ts b/packages/backend/src/core/VideoProcessingService.ts index ea5701decc..dd6c51c217 100644 --- a/packages/backend/src/core/VideoProcessingService.ts +++ b/packages/backend/src/core/VideoProcessingService.ts @@ -6,6 +6,7 @@ import { ImageProcessingService } from '@/core/ImageProcessingService.js'; import type { IImage } from '@/core/ImageProcessingService.js'; import { createTempDir } from '@/misc/create-temp.js'; import { bindThis } from '@/decorators.js'; +import { appendQuery, query } from '@/misc/prelude/url.js'; @Injectable() export class VideoProcessingService { @@ -41,5 +42,18 @@ export class VideoProcessingService { cleanup(); } } + + @bindThis + public getExternalVideoThumbnailUrl(url: string): string | null { + if (this.config.videoThumbnailGenerator == null) return null; + + return appendQuery( + `${this.config.videoThumbnailGenerator}/thumbnail.webp`, + query({ + thumbnail: '1', + url, + }) + ) + } } diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts index 9dd115d45a..b8550cd73e 100644 --- a/packages/backend/src/core/entities/DriveFileEntityService.ts +++ b/packages/backend/src/core/entities/DriveFileEntityService.ts @@ -13,6 +13,7 @@ import { deepClone } from '@/misc/clone.js'; import { UtilityService } from '../UtilityService.js'; import { UserEntityService } from './UserEntityService.js'; import { DriveFolderEntityService } from './DriveFolderEntityService.js'; +import { VideoProcessingService } from '../VideoProcessingService.js'; type PackOptions = { detail?: boolean, @@ -43,6 +44,7 @@ export class DriveFileEntityService { private utilityService: UtilityService, private driveFolderEntityService: DriveFolderEntityService, + private videoProcessingService: VideoProcessingService, ) { } @@ -72,40 +74,63 @@ export class DriveFileEntityService { } @bindThis - public getPublicUrl(file: DriveFile, mode? : 'static' | 'avatar'): string | null { // static = thumbnail - const proxiedUrl = (url: string) => appendQuery( + private getProxiedUrl(url: string, mode?: 'static' | 'avatar'): string | null { + return appendQuery( `${this.config.mediaProxy}/${mode ?? 'image'}.webp`, query({ url, ...(mode ? { [mode]: '1' } : {}), }) - ); + ) + } + @bindThis + public getThumbnailUrl(file: DriveFile): string | null { + if (file.type.startsWith('video')) { + if (file.thumbnailUrl) return file.thumbnailUrl; + + if (this.config.videoThumbnailGenerator == null) { + return this.videoProcessingService.getExternalVideoThumbnailUrl(file.webpublicUrl ?? file.url ?? file.uri); + } + } else if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) { + // 動画ではなくリモートかつメディアプロキシ + return this.getProxiedUrl(file.uri, 'static'); + } + + if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) { + // リモートかつ期限切れはローカルプロキシを試みる + // 従来は/files/${thumbnailAccessKey}にアクセスしていたが、 + // /filesはメディアプロキシにリダイレクトするようにしたため直接メディアプロキシを指定する + return this.getProxiedUrl(file.uri, 'static'); + } + + const url = file.webpublicUrl ?? file.url; + + return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? this.getProxiedUrl(url, 'static') : null); + } + + @bindThis + public getPublicUrl(file: DriveFile, mode?: 'avatar'): string | null { // static = thumbnail // リモートかつメディアプロキシ if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) { - if (!(mode === 'static' && file.type.startsWith('video'))) { - return proxiedUrl(file.uri); - } + return this.getProxiedUrl(file.uri, mode); } // リモートかつ期限切れはローカルプロキシを試みる if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) { - const key = mode === 'static' ? file.thumbnailAccessKey : file.webpublicAccessKey; + const key = file.webpublicAccessKey; if (key && !key.match('/')) { // 古いものはここにオブジェクトストレージキーが入ってるので除外 const url = `${this.config.url}/files/${key}`; - if (mode === 'avatar') return proxiedUrl(file.uri); + if (mode === 'avatar') return this.getProxiedUrl(file.uri, 'avatar'); return url; } } const url = file.webpublicUrl ?? file.url; - if (mode === 'static') { - return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? proxiedUrl(url) : null); - } if (mode === 'avatar') { - return proxiedUrl(url); + return this.getProxiedUrl(url, 'avatar'); } return url; } @@ -183,7 +208,7 @@ export class DriveFileEntityService { blurhash: file.blurhash, properties: opts.self ? file.properties : this.getPublicProperties(file), url: opts.self ? file.url : this.getPublicUrl(file), - thumbnailUrl: this.getPublicUrl(file, 'static'), + thumbnailUrl: this.getThumbnailUrl(file), comment: file.comment, folderId: file.folderId, folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, { @@ -218,7 +243,7 @@ export class DriveFileEntityService { blurhash: file.blurhash, properties: opts.self ? file.properties : this.getPublicProperties(file), url: opts.self ? file.url : this.getPublicUrl(file), - thumbnailUrl: this.getPublicUrl(file, 'static'), + thumbnailUrl: this.getThumbnailUrl(file), comment: file.comment, folderId: file.folderId, folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, { diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 49ded6c28e..f4bc568fdc 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -150,6 +150,12 @@ export class FileServerService { file.cleanup(); return await reply.redirect(301, url.toString()); } else if (file.mime.startsWith('video/')) { + const externalThumbnail = this.videoProcessingService.getExternalVideoThumbnailUrl(file.url); + if (externalThumbnail) { + file.cleanup(); + return await reply.redirect(301, externalThumbnail); + } + image = await this.videoProcessingService.generateVideoThumbnail(file.path); } } -- cgit v1.2.3-freya From 9ddf62d8b7b955217108a17e9074d50b5722dd3a Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 12 Feb 2023 10:26:27 +0900 Subject: enhance: レートリミットを0%にできるように MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + packages/backend/src/server/api/ApiCallService.ts | 18 ++++++++++-------- packages/frontend/src/pages/admin/roles.editor.vue | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) (limited to 'packages/backend/src/server') diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d0fbd0a23..2ce5292040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ You should also include the user name that made the change. ### Improvements - アニメーションを少なくする設定の時、MkPageHeaderのタブアニメーションを無効化 - Backend: activitypub情報がcorsでブロックされないようヘッダーを追加 +- enhance: レートリミットを0%にできるように ### Bugfixes - Client: ユーザーページでアクティビティを見ることができない問題を修正 diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 395a1c468a..2f3e7a44a9 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -227,15 +227,17 @@ export class ApiCallService implements OnApplicationShutdown { // TODO: 毎リクエスト計算するのもあれだしキャッシュしたい const factor = user ? (await this.roleService.getUserPolicies(user.id)).rateLimitFactor : 1; - // Rate limit - await this.rateLimiterService.limit(limit as IEndpointMeta['limit'] & { key: NonNullable }, limitActor, factor).catch(err => { - throw new ApiError({ - message: 'Rate limit exceeded. Please try again later.', - code: 'RATE_LIMIT_EXCEEDED', - id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef', - httpStatusCode: 429, + if (factor > 0) { + // Rate limit + await this.rateLimiterService.limit(limit as IEndpointMeta['limit'] & { key: NonNullable }, limitActor, factor).catch(err => { + throw new ApiError({ + message: 'Rate limit exceeded. Please try again later.', + code: 'RATE_LIMIT_EXCEEDED', + id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef', + httpStatusCode: 429, + }); }); - }); + } } if (ep.meta.requireCredential || ep.meta.requireModerator || ep.meta.requireAdmin) { diff --git a/packages/frontend/src/pages/admin/roles.editor.vue b/packages/frontend/src/pages/admin/roles.editor.vue index 086537a94a..d89a68f982 100644 --- a/packages/frontend/src/pages/admin/roles.editor.vue +++ b/packages/frontend/src/pages/admin/roles.editor.vue @@ -72,7 +72,7 @@ - + -- cgit v1.2.3-freya From b55d26387b6a56c90c9dc96a90d9fffaef98bab4 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 12 Feb 2023 10:36:43 +0900 Subject: improve error handling --- packages/backend/src/core/UserListService.ts | 4 +++- .../src/server/api/endpoints/users/lists/push.ts | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) (limited to 'packages/backend/src/server') diff --git a/packages/backend/src/core/UserListService.ts b/packages/backend/src/core/UserListService.ts index c174394999..bc726a1feb 100644 --- a/packages/backend/src/core/UserListService.ts +++ b/packages/backend/src/core/UserListService.ts @@ -14,6 +14,8 @@ import { RoleService } from '@/core/RoleService.js'; @Injectable() export class UserListService { + public static TooManyUsersError = class extends Error {}; + constructor( @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -36,7 +38,7 @@ export class UserListService { userListId: list.id, }); if (currentCount > (await this.roleService.getUserPolicies(me.id)).userEachUserListsLimit) { - throw new Error('Too many users'); + throw new UserListService.TooManyUsersError(); } await this.userListJoiningsRepository.insert({ diff --git a/packages/backend/src/server/api/endpoints/users/lists/push.ts b/packages/backend/src/server/api/endpoints/users/lists/push.ts index 3a079ee1ab..1c1fdc23f1 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/push.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/push.ts @@ -45,6 +45,12 @@ export const meta = { code: 'YOU_HAVE_BEEN_BLOCKED', id: '990232c5-3f9d-4d83-9f3f-ef27b6332a4b', }, + + tooManyUsers: { + message: 'You can not push users any more.', + code: 'TOO_MANY_USERS', + id: '2dd9752e-a338-413d-8eec-41814430989b', + }, }, } as const; @@ -110,8 +116,15 @@ export default class extends Endpoint { throw new ApiError(meta.errors.alreadyAdded); } - // Push the user - await this.userListService.push(user, userList, me); + try { + await this.userListService.push(user, userList, me); + } catch (err) { + if (err instanceof UserListService.TooManyUsersError) { + throw new ApiError(meta.errors.tooManyUsers); + } + + throw err; + } }); } } -- cgit v1.2.3-freya