enhance(server): videoThumbnailGenerator config (#9845)

* enhance(server): videoThumbnailGenerator config

* :v:

* fix

* 相対url

* サムネイルのproxyRemoteFilesは直接プロキシを指定する

* メディアプロキシ

6 files changed, 82 insertions, 14 deletions

diff --git a/.config/example.yml b/.config/example.yml
index a19b5d04e8..92b8726623 100644
--- a/.config/example.yml
+++ b/.config/example.yml
@@ -131,11 +131,20 @@ proxyBypassHosts:
 
 # Media Proxy
 # Reference Implementation: https://github.com/misskey-dev/media-proxy
+# * Deliver a common cache between instances
+# * Perform image compression (on a different server resource than the main process)
 #mediaProxy: https://example.com/proxy
 
 # Proxy remote files (default: false)
+# Proxy remote files by this instance or mediaProxy to prevent remote files from running in remote domains.
 #proxyRemoteFiles: true
 
+# Movie Thumbnail Generation URL
+# There is no reference implementation.
+# For example, Misskey will point to the following URL:
+#   https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4
+#videoThumbnailGenerator: https://example.com
+
 # Sign to ActivityPub GET request (default: true)
 signToActivityPubGet: true
 
diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts
index aa98ef1d22..73f45e92e1 100644
--- a/packages/backend/src/config.ts
+++ b/packages/backend/src/config.ts
@@ -67,6 +67,7 @@ export type Source = {
 
 	mediaProxy?: string;
 	proxyRemoteFiles?: boolean;
+	videoThumbnailGenerator?: string;
 
 	signToActivityPubGet?: boolean;
 };
@@ -89,6 +90,7 @@ export type Mixin = {
 	clientManifestExists: boolean;
 	mediaProxy: string;
 	externalMediaProxyEnabled: boolean;
+	videoThumbnailGenerator: string | null;
 };
 
 export type Config = Source & Mixin;
@@ -144,6 +146,10 @@ export function loadConfig() {
 	mixin.mediaProxy = externalMediaProxy ?? internalMediaProxy;
 	mixin.externalMediaProxyEnabled = externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy;
 
+	mixin.videoThumbnailGenerator = config.videoThumbnailGenerator ?
+		config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator
+		: null;
+
 	if (!config.redis.prefix) config.redis.prefix = mixin.host;
 
 	return Object.assign(config, mixin);
diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts
index 598a457e83..42a430ea75 100644
--- a/packages/backend/src/core/DriveService.ts
+++ b/packages/backend/src/core/DriveService.ts
@@ -250,6 +250,14 @@ export class DriveService {
 	@bindThis
 	public async generateAlts(path: string, type: string, generateWeb: boolean) {
 		if (type.startsWith('video/')) {
+			if (this.config.videoThumbnailGenerator != null) {
+				// videoThumbnailGeneratorが指定されていたら動画サムネイル生成はスキップ
+				return {
+					webpublic: null,
+					thumbnail: null,
+				}
+			}
+
 			try {
 				const thumbnail = await this.videoProcessingService.generateVideoThumbnail(path);
 				return {
diff --git a/packages/backend/src/core/VideoProcessingService.ts b/packages/backend/src/core/VideoProcessingService.ts
index ea5701decc..dd6c51c217 100644
--- a/packages/backend/src/core/VideoProcessingService.ts
+++ b/packages/backend/src/core/VideoProcessingService.ts
@@ -6,6 +6,7 @@ import { ImageProcessingService } from '@/core/ImageProcessingService.js';
 import type { IImage } from '@/core/ImageProcessingService.js';
 import { createTempDir } from '@/misc/create-temp.js';
 import { bindThis } from '@/decorators.js';
+import { appendQuery, query } from '@/misc/prelude/url.js';
 
 @Injectable()
 export class VideoProcessingService {
@@ -41,5 +42,18 @@ export class VideoProcessingService {
 			cleanup();
 		}
 	}
+
+	@bindThis
+	public getExternalVideoThumbnailUrl(url: string): string | null {
+		if (this.config.videoThumbnailGenerator == null) return null;
+
+		return appendQuery(
+			`${this.config.videoThumbnailGenerator}/thumbnail.webp`,
+			query({
+				thumbnail: '1',
+				url,
+			})
+		)
+	}
 }
 
diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts
index 9dd115d45a..b8550cd73e 100644
--- a/packages/backend/src/core/entities/DriveFileEntityService.ts
+++ b/packages/backend/src/core/entities/DriveFileEntityService.ts
@@ -13,6 +13,7 @@ import { deepClone } from '@/misc/clone.js';
 import { UtilityService } from '../UtilityService.js';
 import { UserEntityService } from './UserEntityService.js';
 import { DriveFolderEntityService } from './DriveFolderEntityService.js';
+import { VideoProcessingService } from '../VideoProcessingService.js';
 
 type PackOptions = {
 	detail?: boolean,
@@ -43,6 +44,7 @@ export class DriveFileEntityService {
 
 		private utilityService: UtilityService,
 		private driveFolderEntityService: DriveFolderEntityService,
+		private videoProcessingService: VideoProcessingService,
 	) {
 	}
 	
@@ -72,40 +74,63 @@ export class DriveFileEntityService {
 	}
 
 	@bindThis
-	public getPublicUrl(file: DriveFile, mode? : 'static' | 'avatar'): string | null { // static = thumbnail
-		const proxiedUrl = (url: string) => appendQuery(
+	private getProxiedUrl(url: string, mode?: 'static' | 'avatar'): string | null {
+		return appendQuery(
 			`${this.config.mediaProxy}/${mode ?? 'image'}.webp`,
 			query({
 				url,
 				...(mode ? { [mode]: '1' } : {}),
 			})
-		);
+		)
+	}
 
+	@bindThis
+	public getThumbnailUrl(file: DriveFile): string | null {
+		if (file.type.startsWith('video')) {
+			if (file.thumbnailUrl) return file.thumbnailUrl;
+
+			if (this.config.videoThumbnailGenerator == null) {
+				return this.videoProcessingService.getExternalVideoThumbnailUrl(file.webpublicUrl ?? file.url ?? file.uri);
+			}
+		} else if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) {
+			// 動画ではなくリモートかつメディアプロキシ
+			return this.getProxiedUrl(file.uri, 'static');
+		}
+
+		if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) {
+			// リモートかつ期限切れはローカルプロキシを試みる
+			// 従来は/files/${thumbnailAccessKey}にアクセスしていたが、
+			// /filesはメディアプロキシにリダイレクトするようにしたため直接メディアプロキシを指定する
+			return this.getProxiedUrl(file.uri, 'static');
+		}
+
+		const url = file.webpublicUrl ?? file.url;
+
+		return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? this.getProxiedUrl(url, 'static') : null);
+	}
+
+	@bindThis
+	public getPublicUrl(file: DriveFile, mode?: 'avatar'): string | null { // static = thumbnail
 		// リモートかつメディアプロキシ
 		if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) {
-			if (!(mode === 'static' && file.type.startsWith('video'))) {
-				return proxiedUrl(file.uri);
-			}
+			return this.getProxiedUrl(file.uri, mode);
 		}
 
 		// リモートかつ期限切れはローカルプロキシを試みる
 		if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) {
-			const key = mode === 'static' ? file.thumbnailAccessKey : file.webpublicAccessKey;
+			const key = file.webpublicAccessKey;
 
 			if (key && !key.match('/')) {	// 古いものはここにオブジェクトストレージキーが入ってるので除外
 				const url = `${this.config.url}/files/${key}`;
-				if (mode === 'avatar') return proxiedUrl(file.uri);
+				if (mode === 'avatar') return this.getProxiedUrl(file.uri, 'avatar');
 				return url;
 			}
 		}
 
 		const url = file.webpublicUrl ?? file.url;
 
-		if (mode === 'static') {
-			return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? proxiedUrl(url) : null);
-		}
 		if (mode === 'avatar') {
-			return proxiedUrl(url);
+			return this.getProxiedUrl(url, 'avatar');
 		}
 		return url;
 	}
@@ -183,7 +208,7 @@ export class DriveFileEntityService {
 			blurhash: file.blurhash,
 			properties: opts.self ? file.properties : this.getPublicProperties(file),
 			url: opts.self ? file.url : this.getPublicUrl(file),
-			thumbnailUrl: this.getPublicUrl(file, 'static'),
+			thumbnailUrl: this.getThumbnailUrl(file),
 			comment: file.comment,
 			folderId: file.folderId,
 			folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, {
@@ -218,7 +243,7 @@ export class DriveFileEntityService {
 			blurhash: file.blurhash,
 			properties: opts.self ? file.properties : this.getPublicProperties(file),
 			url: opts.self ? file.url : this.getPublicUrl(file),
-			thumbnailUrl: this.getPublicUrl(file, 'static'),
+			thumbnailUrl: this.getThumbnailUrl(file),
 			comment: file.comment,
 			folderId: file.folderId,
 			folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, {
diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts
index 49ded6c28e..f4bc568fdc 100644
--- a/packages/backend/src/server/FileServerService.ts
+++ b/packages/backend/src/server/FileServerService.ts
@@ -150,6 +150,12 @@ export class FileServerService {
 						file.cleanup();
 						return await reply.redirect(301, url.toString());
 					} else if (file.mime.startsWith('video/')) {
+						const externalThumbnail = this.videoProcessingService.getExternalVideoThumbnailUrl(file.url);
+						if (externalThumbnail) {
+							file.cleanup();
+							return await reply.redirect(301, externalThumbnail);
+						}
+
 						image = await this.videoProcessingService.generateVideoThumbnail(file.path);
 					}
 				}