perf(server): improvement of external mediaProxy (#9787)

* perf(server): improvement of external mediaProxy

* add a comment

* :v:

* /filesでsharpの処理を行わずリダイレクトする

* fix

* thumbnail => static

* Fix #9788

* add avatar mode

* add url

* fix

* static.webp

* remove encodeURIComponent from media proxy path

* remove existance check

12 files changed, 107 insertions, 59 deletions

diff --git a/.config/example.yml b/.config/example.yml
index 8fe41da15a..a19b5d04e8 100644
--- a/.config/example.yml
+++ b/.config/example.yml
@@ -130,6 +130,7 @@ proxyBypassHosts:
 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5
 
 # Media Proxy
+# Reference Implementation: https://github.com/misskey-dev/media-proxy
 #mediaProxy: https://example.com/proxy
 
 # Proxy remote files (default: false)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1e10edf683..0ad1e36213 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,9 @@ You should also include the user name that made the change.
 - syslogのサポートが削除されました
 
 ### Improvements
+- 外部メディアプロキシへの対応を強化しました  
+  外部メディアプロキシのFastify実装を作りました  
+  https://github.com/misskey-dev/media-proxy
 - ロールで広告の非表示が有効になっている場合は最初から広告を非表示にするように
 
 ## 13.2.6 (2023/02/01)
diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts
index 1d4e700656..aa98ef1d22 100644
--- a/packages/backend/src/config.ts
+++ b/packages/backend/src/config.ts
@@ -87,6 +87,8 @@ export type Mixin = {
 	userAgent: string;
 	clientEntry: string;
 	clientManifestExists: boolean;
+	mediaProxy: string;
+	externalMediaProxyEnabled: boolean;
 };
 
 export type Config = Source & Mixin;
@@ -135,6 +137,13 @@ export function loadConfig() {
 	mixin.clientEntry = clientManifest['src/init.ts'];
 	mixin.clientManifestExists = clientManifestExists;
 
+	const externalMediaProxy = config.mediaProxy ?
+		config.mediaProxy.endsWith('/') ? config.mediaProxy.substring(0, config.mediaProxy.length - 1) : config.mediaProxy
+		: null;
+	const internalMediaProxy = `${mixin.scheme}://${mixin.host}/proxy`;
+	mixin.mediaProxy = externalMediaProxy ?? internalMediaProxy;
+	mixin.externalMediaProxyEnabled = externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy;
+
 	if (!config.redis.prefix) config.redis.prefix = mixin.host;
 
 	return Object.assign(config, mixin);
diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts
index 39814e1be6..63f0319442 100644
--- a/packages/backend/src/core/CustomEmojiService.ts
+++ b/packages/backend/src/core/CustomEmojiService.ts
@@ -120,7 +120,7 @@ export class CustomEmojiService {
 		const url = isLocal
 			? emojiUrl
 			: this.config.proxyRemoteFiles
-				? `${this.config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}`
+				? `${this.config.mediaProxy}/emoji.webp?${query({ url: emojiUrl })}`
 				: emojiUrl;
 
 		return url;
diff --git a/packages/backend/src/core/entities/ChannelEntityService.ts b/packages/backend/src/core/entities/ChannelEntityService.ts
index 5e2f019a12..6ce590aa96 100644
--- a/packages/backend/src/core/entities/ChannelEntityService.ts
+++ b/packages/backend/src/core/entities/ChannelEntityService.ts
@@ -54,7 +54,7 @@ export class ChannelEntityService {
 			name: channel.name,
 			description: channel.description,
 			userId: channel.userId,
-			bannerUrl: banner ? this.driveFileEntityService.getPublicUrl(banner, false) : null,
+			bannerUrl: banner ? this.driveFileEntityService.getPublicUrl(banner) : null,
 			usersCount: channel.usersCount,
 			notesCount: channel.notesCount,
 
diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts
index 7f54cfdeac..efc196f74a 100644
--- a/packages/backend/src/core/entities/DriveFileEntityService.ts
+++ b/packages/backend/src/core/entities/DriveFileEntityService.ts
@@ -71,27 +71,41 @@ export class DriveFileEntityService {
 	}
 
 	@bindThis
-	public getPublicUrl(file: DriveFile, thumbnail = false): string | null {
+	public getPublicUrl(file: DriveFile, mode? : 'static' | 'avatar'): string | null { // static = thumbnail
+		const proxiedUrl = (url: string) => appendQuery(
+			`${this.config.mediaProxy}/${mode ?? 'image'}.webp`,
+			query({
+				url,
+				...(mode ? { [mode]: '1' } : {}),
+			})
+		);
+
 		// リモートかつメディアプロキシ
-		if (file.uri != null && file.userHost != null && this.config.mediaProxy != null) {
-			return appendQuery(this.config.mediaProxy, query({
-				url: file.uri,
-				thumbnail: thumbnail ? '1' : undefined,
-			}));
+		if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) {
+			return proxiedUrl(file.uri);
 		}
 
 		// リモートかつ期限切れはローカルプロキシを試みる
 		if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) {
-			const key = thumbnail ? file.thumbnailAccessKey : file.webpublicAccessKey;
+			const key = mode === 'static' ? file.thumbnailAccessKey : file.webpublicAccessKey;
 
 			if (key && !key.match('/')) {	// 古いものはここにオブジェクトストレージキーが入ってるので除外
-				return `${this.config.url}/files/${key}`;
+				const url = `${this.config.url}/files/${key}`;
+				if (mode === 'avatar') return proxiedUrl(url);
+				return url;
 			}
 		}
 
 		const isImage = file.type && ['image/png', 'image/apng', 'image/gif', 'image/jpeg', 'image/webp', 'image/avif', 'image/svg+xml'].includes(file.type);
 
-		return thumbnail ? (file.thumbnailUrl ?? (isImage ? (file.webpublicUrl ?? file.url) : null)) : (file.webpublicUrl ?? file.url);
+		if (mode === 'static') {
+			return file.thumbnailUrl ?? (isImage ? (file.webpublicUrl ?? file.url) : null);
+		}
+
+		const url = file.webpublicUrl ?? file.url;
+
+		if (mode === 'avatar') return proxiedUrl(url);
+		return url;
 	}
 
 	@bindThis
@@ -166,8 +180,8 @@ export class DriveFileEntityService {
 			isSensitive: file.isSensitive,
 			blurhash: file.blurhash,
 			properties: opts.self ? file.properties : this.getPublicProperties(file),
-			url: opts.self ? file.url : this.getPublicUrl(file, false),
-			thumbnailUrl: this.getPublicUrl(file, true),
+			url: opts.self ? file.url : this.getPublicUrl(file),
+			thumbnailUrl: this.getPublicUrl(file, 'static'),
 			comment: file.comment,
 			folderId: file.folderId,
 			folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, {
@@ -201,8 +215,8 @@ export class DriveFileEntityService {
 			isSensitive: file.isSensitive,
 			blurhash: file.blurhash,
 			properties: opts.self ? file.properties : this.getPublicProperties(file),
-			url: opts.self ? file.url : this.getPublicUrl(file, false),
-			thumbnailUrl: this.getPublicUrl(file, true),
+			url: opts.self ? file.url : this.getPublicUrl(file),
+			thumbnailUrl: this.getPublicUrl(file, 'static'),
 			comment: file.comment,
 			folderId: file.folderId,
 			folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, {
diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts
index aaa80033b3..ff42c07359 100644
--- a/packages/backend/src/core/entities/UserEntityService.ts
+++ b/packages/backend/src/core/entities/UserEntityService.ts
@@ -314,10 +314,10 @@ export class UserEntityService implements OnModuleInit {
 	@bindThis
 	public async getAvatarUrl(user: User): Promise<string> {
 		if (user.avatar) {
-			return this.driveFileEntityService.getPublicUrl(user.avatar, true) ?? this.getIdenticonUrl(user.id);
+			return this.driveFileEntityService.getPublicUrl(user.avatar, 'avatar') ?? this.getIdenticonUrl(user.id);
 		} else if (user.avatarId) {
 			const avatar = await this.driveFilesRepository.findOneByOrFail({ id: user.avatarId });
-			return this.driveFileEntityService.getPublicUrl(avatar, true) ?? this.getIdenticonUrl(user.id);
+			return this.driveFileEntityService.getPublicUrl(avatar, 'avatar') ?? this.getIdenticonUrl(user.id);
 		} else {
 			return this.getIdenticonUrl(user.id);
 		}
@@ -326,7 +326,7 @@ export class UserEntityService implements OnModuleInit {
 	@bindThis
 	public getAvatarUrlSync(user: User): string {
 		if (user.avatar) {
-			return this.driveFileEntityService.getPublicUrl(user.avatar, true) ?? this.getIdenticonUrl(user.id);
+			return this.driveFileEntityService.getPublicUrl(user.avatar, 'avatar') ?? this.getIdenticonUrl(user.id);
 		} else {
 			return this.getIdenticonUrl(user.id);
 		}
@@ -422,7 +422,7 @@ export class UserEntityService implements OnModuleInit {
 				createdAt: user.createdAt.toISOString(),
 				updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null,
 				lastFetchedAt: user.lastFetchedAt ? user.lastFetchedAt.toISOString() : null,
-				bannerUrl: user.banner ? this.driveFileEntityService.getPublicUrl(user.banner, false) : null,
+				bannerUrl: user.banner ? this.driveFileEntityService.getPublicUrl(user.banner) : null,
 				bannerBlurhash: user.banner?.blurhash ?? null,
 				isLocked: user.isLocked,
 				isSilenced: this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote),
diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts
index 40024270ae..39bc4c1d96 100644
--- a/packages/backend/src/server/FileServerService.ts
+++ b/packages/backend/src/server/FileServerService.ts
@@ -137,38 +137,38 @@ export class FileServerService {
 
 		try {
 			if (file.state === 'remote') {
-				const convertFile = async () => {
-					if (file.fileRole === 'thumbnail') {
-						if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(file.mime)) {
-							return this.imageProcessingService.convertToWebpStream(
-								file.path,
-								498,
-								280
-							);
-						} else if (file.mime.startsWith('video/')) {
-							return await this.videoProcessingService.generateVideoThumbnail(file.path);
-						}
+				let image: IImageStreamable | null = null;
+
+				if (file.fileRole === 'thumbnail') {
+					if (isMimeImage(file.mime, 'sharp-convertible-image')) {
+						reply.header('Cache-Control', 'max-age=31536000, immutable');
+
+						const url = new URL(`${this.config.mediaProxy}/static.webp`);
+						url.searchParams.set('url', file.url);
+						url.searchParams.set('static', '1');
+						return await reply.redirect(301, url.toString());
+					} else if (file.mime.startsWith('video/')) {
+						image = await this.videoProcessingService.generateVideoThumbnail(file.path);
 					}
+				}
 
-					if (file.fileRole === 'webpublic') {
-						if (['image/svg+xml'].includes(file.mime)) {
-							return this.imageProcessingService.convertToWebpStream(
-								file.path,
-								2048,
-								2048,
-								{ ...webpDefault, lossless: true }
-							)
-						}
+				if (file.fileRole === 'webpublic') {
+					if (['image/svg+xml'].includes(file.mime)) {
+						reply.header('Cache-Control', 'max-age=31536000, immutable');
+
+						const url = new URL(`${this.config.mediaProxy}/svg.webp`);
+						url.searchParams.set('url', file.url);
+						return await reply.redirect(301, url.toString());
 					}
+				}
 
-					return {
+				if (!image) {
+					image = {
 						data: fs.createReadStream(file.path),
 						ext: file.ext,
 						type: file.mime,
 					};
-				};
-
-				const image = await convertFile();
+				}
 
 				if ('pipe' in image.data && typeof image.data.pipe === 'function') {
 					// image.dataがstreamなら、stream終了後にcleanup
@@ -180,7 +180,6 @@ export class FileServerService {
 				}
 
 				reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(image.type) ? image.type : 'application/octet-stream');
-				reply.header('Cache-Control', 'max-age=31536000, immutable');
 				return image.data;
 			}
 
@@ -217,6 +216,23 @@ export class FileServerService {
 			return;
 		}
 
+		if (this.config.externalMediaProxyEnabled) {
+			// 外部のメディアプロキシが有効なら、そちらにリダイレクト
+
+			reply.header('Cache-Control', 'public, max-age=259200'); // 3 days
+
+			const url = new URL(`${this.config.mediaProxy}/${request.params.url || ''}`);
+
+			for (const [key, value] of Object.entries(request.query)) {
+				url.searchParams.append(key, value);
+			}
+
+			return await reply.redirect(
+				301,
+				url.toString(),
+			);
+		}
+
 		// Create temp file
 		const file = await this.getStreamAndTypeFromUrl(url);
 		if (file === '404') {
@@ -236,7 +252,7 @@ export class FileServerService {
 			const isAnimationConvertibleImage = isMimeImage(file.mime, 'sharp-animation-convertible-image');
 
 			let image: IImageStreamable | null = null;
-			if ('emoji' in request.query && isConvertibleImage) {
+			if (('emoji' in request.query || 'avatar' in request.query) && isConvertibleImage) {
 				if (!isAnimationConvertibleImage && !('static' in request.query)) {
 					image = {
 						data: fs.createReadStream(file.path),
@@ -246,7 +262,7 @@ export class FileServerService {
 				} else {
 					const data = sharp(file.path, { animated: !('static' in request.query) })
 							.resize({
-								height: 128,
+								height: 'emoji' in request.query ? 128 : 320,
 								withoutEnlargement: true,
 							})
 							.webp(webpDefault);
@@ -370,7 +386,7 @@ export class FileServerService {
 
 	@bindThis
 	private async getFileFromKey(key: string): Promise<
-		{ state: 'remote'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; mime: string; ext: string | null; path: string; cleanup: () => void; }
+		{ state: 'remote'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; url: string; mime: string; ext: string | null; path: string; cleanup: () => void; }
 		| { state: 'stored_internal'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; mime: string; ext: string | null; path: string; }
 		| '404'
 		| '204'
@@ -392,6 +408,7 @@ export class FileServerService {
 			const result = await this.downloadAndDetectTypeFromUrl(file.uri);
 			return {
 				...result,
+				url: file.uri,
 				fileRole: isThumbnail ? 'thumbnail' : isWebpublic ? 'webpublic' : 'original',
 				file,
 			}
diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts
index beb3a34ecd..c7a2c99f94 100644
--- a/packages/backend/src/server/ServerService.ts
+++ b/packages/backend/src/server/ServerService.ts
@@ -106,7 +106,7 @@ export class ServerService {
 				}
 			}
 
-			const url = new URL('/proxy/emoji.webp', this.config.url);
+			const url = new URL(`${this.config.mediaProxy}/emoji.webp`);
 			// || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ）
 			url.searchParams.set('url', emoji.publicUrl || emoji.originalUrl);
 			url.searchParams.set('emoji', '1');
diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts
index 3baf945323..2fa7a09d49 100644
--- a/packages/backend/src/server/api/endpoints/meta.ts
+++ b/packages/backend/src/server/api/endpoints/meta.ts
@@ -181,6 +181,10 @@ export const meta = {
 				type: 'string',
 				optional: false, nullable: true,
 			},
+			mediaProxy: {
+				type: 'string',
+				optional: false, nullable: false,
+			},
 			features: {
 				type: 'object',
 				optional: true, nullable: false,
@@ -307,6 +311,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 
 				policies: { ...DEFAULT_POLICIES, ...instance.policies },
 
+				mediaProxy: this.config.mediaProxy,
+
 				...(ps.detail ? {
 					pinnedPages: instance.pinnedPages,
 					pinnedClipId: instance.pinnedClipId,
diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts
index 802b404ce6..1bf88fe434 100644
--- a/packages/backend/src/server/web/UrlPreviewService.ts
+++ b/packages/backend/src/server/web/UrlPreviewService.ts
@@ -33,7 +33,7 @@ export class UrlPreviewService {
 	private wrap(url?: string): string | null {
 		return url != null
 			? url.match(/^https?:\/\//)
-				? `${this.config.url}/proxy/preview.webp?${query({
+				? `${this.config.mediaProxy}/preview.webp?${query({
 					url,
 					preview: '1',
 				})}`
diff --git a/packages/frontend/src/scripts/media-proxy.ts b/packages/frontend/src/scripts/media-proxy.ts
index bea164e7c8..274e96e0a1 100644
--- a/packages/frontend/src/scripts/media-proxy.ts
+++ b/packages/frontend/src/scripts/media-proxy.ts
@@ -1,8 +1,9 @@
 import { query, appendQuery } from '@/scripts/url';
 import { url } from '@/config';
+import { instance } from '@/instance';
 
 export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string {
-	if (imageUrl.startsWith(`${url}/proxy/`) || imageUrl.startsWith('/proxy/')) {
+	if (imageUrl.startsWith(instance.mediaProxy + '/') || imageUrl.startsWith('/proxy/')) {
 		// もう既にproxyっぽそうだったらsearchParams付けるだけ
 		return appendQuery(imageUrl, query({
 			fallback: '1',
@@ -10,7 +11,7 @@ export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string {
 		}));
 	}
 
-	return `${url}/proxy/image.webp?${query({
+	return `${instance.mediaProxy}/image.webp?${query({
 		url: imageUrl,
 		fallback: '1',
 		...(type ? { [type]: '1' } : {}),
@@ -25,22 +26,19 @@ export function getProxiedImageUrlNullable(imageUrl: string | null | undefined,
 export function getStaticImageUrl(baseUrl: string): string {
 	const u = baseUrl.startsWith('http') ? new URL(baseUrl) : new URL(baseUrl, url);
 
-	if (u.href.startsWith(`${url}/proxy/`)) {
-		// もう既にproxyっぽそうだったらsearchParams付けるだけ
+	if (u.href.startsWith(`${url}/emoji/`)) {
+		// もう既にemojiっぽそうだったらsearchParams付けるだけ
 		u.searchParams.set('static', '1');
 		return u.href;
 	}
 
-	if (u.href.startsWith(`${url}/emoji/`)) {
-		// もう既にemojiっぽそうだったらsearchParams付けるだけ
+	if (u.href.startsWith(instance.mediaProxy + '/')) {
+		// もう既にproxyっぽそうだったらsearchParams付けるだけ
 		u.searchParams.set('static', '1');
 		return u.href;
 	}
 
-	// 拡張子がないとキャッシュしてくれないCDNがあるのでダミーの名前を指定する
-	const dummy = `${encodeURIComponent(`${u.host}${u.pathname}`)}.webp`;
-
-	return `${url}/proxy/${dummy}?${query({
+	return `${instance.mediaProxy}/static.webp?${query({
 		url: u.href,
 		static: '1',
 	})}`;