fix(backend/DriveService): convert WebP/AVIF to WebP (#10239)

* fix(backend/DriveService): convert transparent WebP/AVIF to PNG

* webpにする

その希望が複数ありましたので

* Update packages/backend/src/core/DriveService.ts

Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>

* update test

* webpはwebpublicにできる

---------

Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>
Co-authored-by: tamaina <tamaina@hotmail.co.jp>

8 files changed, 86 insertions, 42 deletions

diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts
index f4a06faebb..c3835faa33 100644
--- a/packages/backend/src/core/DriveService.ts
+++ b/packages/backend/src/core/DriveService.ts
@@ -299,7 +299,7 @@ export class DriveService {
 			}
 
 			satisfyWebpublic = !!(
-				type !== 'image/svg+xml' && type !== 'image/webp' && type !== 'image/avif' &&
+				type !== 'image/svg+xml' && type !== 'image/avif' &&
 			!(metadata.exif ?? metadata.iptc ?? metadata.xmp ?? metadata.tifftagPhotoshop) &&
 			metadata.width && metadata.width <= 2048 &&
 			metadata.height && metadata.height <= 2048
@@ -319,11 +319,11 @@ export class DriveService {
 			this.registerLogger.info('creating web image');
 
 			try {
-				if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) {
+				if (type === 'image/jpeg') {
 					webpublic = await this.imageProcessingService.convertSharpToJpeg(img, 2048, 2048);
-				} else if (['image/png'].includes(type)) {
-					webpublic = await this.imageProcessingService.convertSharpToPng(img, 2048, 2048);
-				} else if (['image/svg+xml'].includes(type)) {
+				} else if (['image/webp', 'image/avif'].includes(type)) {
+					webpublic = await this.imageProcessingService.convertSharpToWebp(img, 2048, 2048);
+				} else if (['image/png', 'image/svg+xml'].includes(type)) {
 					webpublic = await this.imageProcessingService.convertSharpToPng(img, 2048, 2048);
 				} else {
 					this.registerLogger.debug('web image not created (not an required image)');
@@ -749,7 +749,7 @@ export class DriveService {
 	}: UploadFromUrlArgs): Promise<DriveFile> {
 		// Create temp file
 		const [path, cleanup] = await createTemp();
-	
+
 		try {
 			// write content at URL to temp file
 			const { filename: name } = await this.downloadService.downloadUrl(url, path);
diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts
index 6a1f233bd8..4601eca2f0 100644
--- a/packages/backend/src/core/activitypub/ApRendererService.ts
+++ b/packages/backend/src/core/activitypub/ApRendererService.ts
@@ -116,7 +116,7 @@ export class ApRendererService {
 		if (block.blockee?.uri == null) {
 			throw new Error('renderBlock: missing blockee uri');
 		}
-	
+
 		return {
 			type: 'Block',
 			id: `${this.config.url}/blocks/${block.id}`,
@@ -134,10 +134,10 @@ export class ApRendererService {
 			published: note.createdAt.toISOString(),
 			object,
 		} as ICreate;
-	
+
 		if (object.to) activity.to = object.to;
 		if (object.cc) activity.cc = object.cc;
-	
+
 		return activity;
 	}
 
@@ -155,7 +155,7 @@ export class ApRendererService {
 	public renderDocument(file: DriveFile): IApDocument {
 		return {
 			type: 'Document',
-			mediaType: file.type,
+			mediaType: file.webpublicType ?? file.type,
 			url: this.driveFileEntityService.getPublicUrl(file),
 			name: file.comment,
 		};
@@ -297,16 +297,16 @@ export class ApRendererService {
 			const items = await this.driveFilesRepository.findBy({ id: In(ids) });
 			return ids.map(id => items.find(item => item.id === id)).filter(item => item != null) as DriveFile[];
 		};
-	
+
 		let inReplyTo;
 		let inReplyToNote: Note | null;
-	
+
 		if (note.replyId) {
 			inReplyToNote = await this.notesRepository.findOneBy({ id: note.replyId });
-	
+
 			if (inReplyToNote != null) {
 				const inReplyToUser = await this.usersRepository.findOneBy({ id: inReplyToNote.userId });
-	
+
 				if (inReplyToUser != null) {
 					if (inReplyToNote.uri) {
 						inReplyTo = inReplyToNote.uri;
@@ -322,24 +322,24 @@ export class ApRendererService {
 		} else {
 			inReplyTo = null;
 		}
-	
+
 		let quote;
-	
+
 		if (note.renoteId) {
 			const renote = await this.notesRepository.findOneBy({ id: note.renoteId });
-	
+
 			if (renote) {
 				quote = renote.uri ? renote.uri : `${this.config.url}/notes/${renote.id}`;
 			}
 		}
-	
+
 		const attributedTo = `${this.config.url}/users/${note.userId}`;
-	
+
 		const mentions = (JSON.parse(note.mentionedRemoteUsers) as IMentionedRemoteUsers).map(x => x.uri);
-	
+
 		let to: string[] = [];
 		let cc: string[] = [];
-	
+
 		if (note.visibility === 'public') {
 			to = ['https://www.w3.org/ns/activitystreams#Public'];
 			cc = [`${attributedTo}/followers`].concat(mentions);
@@ -352,44 +352,44 @@ export class ApRendererService {
 		} else {
 			to = mentions;
 		}
-	
+
 		const mentionedUsers = note.mentions.length > 0 ? await this.usersRepository.findBy({
 			id: In(note.mentions),
 		}) : [];
-	
+
 		const hashtagTags = (note.tags ?? []).map(tag => this.renderHashtag(tag));
 		const mentionTags = mentionedUsers.map(u => this.renderMention(u));
-	
+
 		const files = await getPromisedFiles(note.fileIds);
-	
+
 		const text = note.text ?? '';
 		let poll: Poll | null = null;
-	
+
 		if (note.hasPoll) {
 			poll = await this.pollsRepository.findOneBy({ noteId: note.id });
 		}
-	
+
 		let apText = text;
-	
+
 		if (quote) {
 			apText += `\n\nRE: ${quote}`;
 		}
-	
+
 		const summary = note.cw === '' ? String.fromCharCode(0x200B) : note.cw;
-	
+
 		const content = this.apMfmService.getNoteHtml(Object.assign({}, note, {
 			text: apText,
 		}));
-	
+
 		const emojis = await this.getEmojis(note.emojis);
 		const apemojis = emojis.map(emoji => this.renderEmoji(emoji));
-	
+
 		const tag = [
 			...hashtagTags,
 			...mentionTags,
 			...apemojis,
 		];
-	
+
 		const asPoll = poll ? {
 			type: 'Question',
 			content: this.apMfmService.getNoteHtml(Object.assign({}, note, {
@@ -601,7 +601,7 @@ export class ApRendererService {
 		if (typeof x === 'object' && x.id == null) {
 			x.id = `${this.config.url}/${uuid()}`;
 		}
-	
+
 		return Object.assign({
 			'@context': [
 				'https://www.w3.org/ns/activitystreams',
@@ -634,18 +634,18 @@ export class ApRendererService {
 			],
 		}, x as T & { id: string; });
 	}
-	
+
 	@bindThis
 	public async attachLdSignature(activity: any, user: { id: User['id']; host: null; }): Promise<IActivity> {
 		const keypair = await this.userKeypairStoreService.getUserKeypair(user.id);
-	
+
 		const ldSignature = this.ldSignatureService.use();
 		ldSignature.debug = false;
 		activity = await ldSignature.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`);
-	
+
 		return activity;
 	}
-	
+
 	/**
 	 * Render OrderedCollectionPage
 	 * @param id URL of self
@@ -686,11 +686,11 @@ export class ApRendererService {
 			type: 'OrderedCollection',
 			totalItems,
 		};
-	
+
 		if (first) page.first = first;
 		if (last) page.last = last;
 		if (orderedItems) page.orderedItems = orderedItems;
-	
+
 		return page;
 	}
 
diff --git a/packages/backend/test/e2e/endpoints.ts b/packages/backend/test/e2e/endpoints.ts
index c130093732..653520cf5f 100644
--- a/packages/backend/test/e2e/endpoints.ts
+++ b/packages/backend/test/e2e/endpoints.ts
@@ -4,7 +4,7 @@ import * as assert from 'assert';
 // node-fetch only supports it's own Blob yet
 // https://github.com/node-fetch/node-fetch/pull/1664
 import { Blob } from 'node-fetch';
-import { startServer, signup, post, api, uploadFile } from '../utils.js';
+import { startServer, signup, post, api, uploadFile, simpleGet } from '../utils.js';
 import type { INestApplicationContext } from '@nestjs/common';
 
 describe('Endpoints', () => {
@@ -439,6 +439,45 @@ describe('Endpoints', () => {
 			assert.strictEqual(res.body.name, 'image.svg');
 			assert.strictEqual(res.body.type, 'image/svg+xml');
 		});
+
+		for (const type of ['webp', 'avif']) {
+			const mediaType = `image/${type}`;
+
+			const getWebpublicType = async (user: any, fileId: string): Promise<string> => {
+				// drive/files/create does not expose webpublicType directly, so get it by posting it
+				const res = await post(user, {
+					text: mediaType,
+					fileIds: [fileId],
+				});
+				const apRes = await simpleGet(`notes/${res.id}`, 'application/activity+json');
+				assert.strictEqual(apRes.status, 200);
+				assert.ok(Array.isArray(apRes.body.attachment));
+				return apRes.body.attachment[0].mediaType;
+			};
+
+			test(`透明な${type}ファイルを作成できる`, async () => {
+				const path = `with-alpha.${type}`;
+				const res = await uploadFile(alice, { path });
+
+				assert.strictEqual(res.status, 200);
+				assert.strictEqual(res.body.name, path);
+				assert.strictEqual(res.body.type, mediaType);
+
+				const webpublicType = await getWebpublicType(alice, res.body.id);
+				assert.strictEqual(webpublicType, 'image/webp');
+			});
+
+			test(`透明じゃない${type}ファイルを作成できる`, async () => {
+				const path = `without-alpha.${type}`;
+				const res = await uploadFile(alice, { path });
+				assert.strictEqual(res.status, 200);
+				assert.strictEqual(res.body.name, path);
+				assert.strictEqual(res.body.type, mediaType);
+
+				const webpublicType = await getWebpublicType(alice, res.body.id);
+				assert.strictEqual(webpublicType, 'image/webp');
+			});
+		}
 	});
 
 	describe('drive/files/update', () => {
diff --git a/packages/backend/test/resources/with-alpha.avif b/packages/backend/test/resources/with-alpha.avif
new file mode 100644
index 0000000000..05f494212e
--- /dev/null
+++ b/packages/backend/test/resources/with-alpha.avif
diff --git a/packages/backend/test/resources/with-alpha.webp b/packages/backend/test/resources/with-alpha.webp
new file mode 100644
index 0000000000..d7b0d70b7f
--- /dev/null
+++ b/packages/backend/test/resources/with-alpha.webp
diff --git a/packages/backend/test/resources/without-alpha.avif b/packages/backend/test/resources/without-alpha.avif
new file mode 100644
index 0000000000..9ea23608b8
--- /dev/null
+++ b/packages/backend/test/resources/without-alpha.avif
diff --git a/packages/backend/test/resources/without-alpha.webp b/packages/backend/test/resources/without-alpha.webp
new file mode 100644
index 0000000000..a51091efe2
--- /dev/null
+++ b/packages/backend/test/resources/without-alpha.webp
diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts
index 37e5ae10d6..d1a5d6d949 100644
--- a/packages/backend/test/utils.ts
+++ b/packages/backend/test/utils.ts
@@ -204,7 +204,12 @@ export const simpleGet = async (path: string, accept = '*/*'): Promise<{ status:
 		redirect: 'manual',
 	});
 
-	const body = res.headers.get('content-type') === 'application/json; charset=utf-8'
+	const jsonTypes = [
+		'application/json; charset=utf-8',
+		'application/activity+json; charset=utf-8',
+	];
+
+	const body = jsonTypes.includes(res.headers.get('content-type') ?? '')
 		? await res.json()
 		: null;