enhance: AVIF support (#9281)

* chore: Make image/avif browsersafe

* server side

* change FileInfoService

* :v:

* avifはMastodonでは絶望的 see https://github.com/misskey-dev/misskey/issues/9283

Co-authored-by: syuilo <Syuilotan@yahoo.co.jp>

8 files changed, 38 insertions, 10 deletions

diff --git a/packages/backend/src/const.ts b/packages/backend/src/const.ts
index 6d3b9559e8..e2203b6b45 100644
--- a/packages/backend/src/const.ts
+++ b/packages/backend/src/const.ts
@@ -12,6 +12,7 @@ export const FILE_TYPE_BROWSERSAFE = [
 	'image/gif',
 	'image/jpeg',
 	'image/webp',
+	'image/avif',
 	'image/apng',
 	'image/bmp',
 	'image/tiff',
diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts
index 895073c32c..bbdb5fae83 100644
--- a/packages/backend/src/core/DriveService.ts
+++ b/packages/backend/src/core/DriveService.ts
@@ -138,6 +138,7 @@ export class DriveService {
 				if (type === 'image/jpeg') ext = '.jpg';
 				if (type === 'image/png') ext = '.png';
 				if (type === 'image/webp') ext = '.webp';
+				if (type === 'image/avif') ext = '.avif';
 				if (type === 'image/apng') ext = '.apng';
 				if (type === 'image/vnd.mozilla.apng') ext = '.apng';
 			}
@@ -262,7 +263,7 @@ export class DriveService {
 			}
 		}
 
-		if (!['image/jpeg', 'image/png', 'image/webp', 'image/svg+xml'].includes(type)) {
+		if (!['image/jpeg', 'image/png', 'image/webp', 'image/avif', 'image/svg+xml'].includes(type)) {
 			this.registerLogger.debug('web image and thumbnail not created (not an required file)');
 			return {
 				webpublic: null,
@@ -287,7 +288,7 @@ export class DriveService {
 			}
 
 			satisfyWebpublic = !!(
-				type !== 'image/svg+xml' && type !== 'image/webp' &&
+				type !== 'image/svg+xml' && type !== 'image/webp' && type !== 'image/avif' &&
 			!(metadata.exif ?? metadata.iptc ?? metadata.xmp ?? metadata.tifftagPhotoshop) &&
 			metadata.width && metadata.width <= 2048 &&
 			metadata.height && metadata.height <= 2048
@@ -307,7 +308,7 @@ export class DriveService {
 			this.registerLogger.info('creating web image');
 
 			try {
-				if (['image/jpeg', 'image/webp'].includes(type)) {
+				if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) {
 					webpublic = await this.imageProcessingService.convertSharpToJpeg(img, 2048, 2048);
 				} else if (['image/png'].includes(type)) {
 					webpublic = await this.imageProcessingService.convertSharpToPng(img, 2048, 2048);
@@ -329,7 +330,7 @@ export class DriveService {
 		let thumbnail: IImage | null = null;
 
 		try {
-			if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(type)) {
+			if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(type)) {
 				thumbnail = await this.imageProcessingService.convertSharpToWebp(img, 498, 280);
 			} else {
 				this.registerLogger.debug('thumbnail not created (not an required file)');
diff --git a/packages/backend/src/core/FileInfoService.ts b/packages/backend/src/core/FileInfoService.ts
index bea1b3402e..8043993047 100644
--- a/packages/backend/src/core/FileInfoService.ts
+++ b/packages/backend/src/core/FileInfoService.ts
@@ -73,7 +73,18 @@ export class FileInfoService {
 		let height: number | undefined;
 		let orientation: number | undefined;
 
-		if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/bmp', 'image/tiff', 'image/svg+xml', 'image/vnd.adobe.photoshop'].includes(type.mime)) {
+		if ([
+			'image/png',
+			'image/gif',
+			'image/jpeg',
+			'image/webp',
+			'image/avif',
+			'image/apng',
+			'image/bmp',
+			'image/tiff',
+			'image/svg+xml',
+			'image/vnd.adobe.photoshop',
+		].includes(type.mime)) {
 			const imageSize = await this.detectImageSize(path).catch(e => {
 				warnings.push(`detectImageSize failed: ${e}`);
 				return undefined;
@@ -100,7 +111,15 @@ export class FileInfoService {
 
 		let blurhash: string | undefined;
 
-		if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/svg+xml'].includes(type.mime)) {
+		if ([
+			'image/jpeg',
+			'image/gif',
+			'image/png',
+			'image/apng',
+			'image/webp',
+			'image/avif',
+			'image/svg+xml',
+		].includes(type.mime)) {
 			blurhash = await this.getBlurhash(path).catch(e => {
 				warnings.push(`getBlurhash failed: ${e}`);
 				return undefined;
@@ -156,7 +175,11 @@ export class FileInfoService {
 			return [sensitive, porn];
 		}
 	
-		if (['image/jpeg', 'image/png', 'image/webp'].includes(mime)) {
+		if ([
+			'image/jpeg',
+			'image/png',
+			'image/webp',
+		].includes(mime)) {
 			const result = await this.aiService.detectSensitive(source);
 			if (result) {
 				[sensitive, porn] = judgePrediction(result);
diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts
index 706c8c1186..7f54cfdeac 100644
--- a/packages/backend/src/core/entities/DriveFileEntityService.ts
+++ b/packages/backend/src/core/entities/DriveFileEntityService.ts
@@ -89,7 +89,7 @@ export class DriveFileEntityService {
 			}
 		}
 
-		const isImage = file.type && ['image/png', 'image/apng', 'image/gif', 'image/jpeg', 'image/webp', 'image/svg+xml'].includes(file.type);
+		const isImage = file.type && ['image/png', 'image/apng', 'image/gif', 'image/jpeg', 'image/webp', 'image/avif', 'image/svg+xml'].includes(file.type);
 
 		return thumbnail ? (file.thumbnailUrl ?? (isImage ? (file.webpublicUrl ?? file.url) : null)) : (file.webpublicUrl ?? file.url);
 	}
diff --git a/packages/backend/src/misc/is-mime-image.ts b/packages/backend/src/misc/is-mime-image.ts
index 8993ede33a..168a9a7af6 100644
--- a/packages/backend/src/misc/is-mime-image.ts
+++ b/packages/backend/src/misc/is-mime-image.ts
@@ -2,7 +2,7 @@ import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
 
 const dictionary = {
 	'safe-file': FILE_TYPE_BROWSERSAFE,
-	'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/svg+xml'],
+	'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml'],
 };
 
 export const isMimeImage = (mime: string, type: keyof typeof dictionary): boolean => dictionary[type].includes(mime);
diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts
index b7ab549611..8b1a130657 100644
--- a/packages/backend/src/server/FileServerService.ts
+++ b/packages/backend/src/server/FileServerService.ts
@@ -114,7 +114,7 @@ export class FileServerService {
 
 					const convertFile = async () => {
 						if (isThumbnail) {
-							if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(mime)) {
+							if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(mime)) {
 								return await this.imageProcessingService.convertToWebp(path, 498, 280);
 							} else if (mime.startsWith('video/')) {
 								return await this.videoProcessingService.generateVideoThumbnail(path);
diff --git a/packages/client/src/const.ts b/packages/client/src/const.ts
index 505cf2748e..77366cf07b 100644
--- a/packages/client/src/const.ts
+++ b/packages/client/src/const.ts
@@ -7,6 +7,7 @@ export const FILE_TYPE_BROWSERSAFE = [
 	'image/gif',
 	'image/jpeg',
 	'image/webp',
+	'image/avif',
 	'image/apng',
 	'image/bmp',
 	'image/tiff',
diff --git a/packages/client/src/pages/user/index.photos.vue b/packages/client/src/pages/user/index.photos.vue
index 5c9a73dcb7..fae2c005d8 100644
--- a/packages/client/src/pages/user/index.photos.vue
+++ b/packages/client/src/pages/user/index.photos.vue
@@ -47,6 +47,8 @@ function thumbnail(image: misskey.entities.DriveFile): string {
 onMounted(() => {
 	const image = [
 		'image/jpeg',
+		'image/webp',
+		'image/avif',
 		'image/png',
 		'image/gif',
 		'image/apng',