feat(frontend): Video compression (#16574)

* wip

* Update CHANGELOG.md

* wip

* wip

* wip

* wip

* Update use-uploader.ts

* Update use-uploader.ts

7 files changed, 166 insertions, 17 deletions

diff --git a/packages/frontend/package.json b/packages/frontend/package.json
index bacdc7b133..104ec42a18 100644
--- a/packages/frontend/package.json
+++ b/packages/frontend/package.json
@@ -57,6 +57,7 @@
 		"json5": "2.2.3",
 		"magic-string": "0.30.18",
 		"matter-js": "0.20.0",
+		"mediabunny": "1.15.1",
 		"mfm-js": "0.25.0",
 		"misskey-bubble-game": "workspace:*",
 		"misskey-js": "workspace:*",
diff --git a/packages/frontend/src/components/MkPostForm.vue b/packages/frontend/src/components/MkPostForm.vue
index 9fec7ea4da..17f93a4ec8 100644
--- a/packages/frontend/src/components/MkPostForm.vue
+++ b/packages/frontend/src/components/MkPostForm.vue
@@ -105,7 +105,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 </template>
 
 <script lang="ts" setup>
-import { inject, watch, nextTick, onMounted, defineAsyncComponent, provide, shallowRef, ref, computed, useTemplateRef } from 'vue';
+import { inject, watch, nextTick, onMounted, defineAsyncComponent, provide, shallowRef, ref, computed, useTemplateRef, onUnmounted } from 'vue';
 import * as mfm from 'mfm-js';
 import * as Misskey from 'misskey-js';
 import insertTextAtCursor from 'insert-text-at-cursor';
@@ -218,6 +218,10 @@ const uploader = useUploader({
 	multiple: true,
 });
 
+onUnmounted(() => {
+	uploader.dispose();
+});
+
 uploader.events.on('itemUploaded', ctx => {
 	files.value.push(ctx.item.uploaded!);
 	uploader.removeItem(ctx.item);
@@ -1300,6 +1304,7 @@ async function canClose() {
 
 defineExpose({
 	clear,
+	abortUploader: () => uploader.abortAll(),
 	canClose,
 });
 </script>
diff --git a/packages/frontend/src/components/MkPostFormDialog.vue b/packages/frontend/src/components/MkPostFormDialog.vue
index bf332e706e..ba8d3a7210 100644
--- a/packages/frontend/src/components/MkPostFormDialog.vue
+++ b/packages/frontend/src/components/MkPostFormDialog.vue
@@ -54,6 +54,7 @@ function onPosted() {
 async function _close() {
 	const canClose = await form.value?.canClose();
 	if (!canClose) return;
+	form.value?.abortUploader();
 	modal.value?.close();
 }
 
diff --git a/packages/frontend/src/components/MkUploaderItems.vue b/packages/frontend/src/components/MkUploaderItems.vue
index f1370965c4..f31c717ad5 100644
--- a/packages/frontend/src/components/MkUploaderItems.vue
+++ b/packages/frontend/src/components/MkUploaderItems.vue
@@ -10,7 +10,10 @@ SPDX-License-Identifier: AGPL-3.0-only
 		:key="item.id"
 		v-panel
 		:class="[$style.item, { [$style.itemWaiting]: item.preprocessing, [$style.itemCompleted]: item.uploaded, [$style.itemFailed]: item.uploadFailed }]"
-		:style="{ '--p': item.progress != null ? `${item.progress.value / item.progress.max * 100}%` : '0%' }"
+		:style="{
+			'--p': item.progress != null ? `${item.progress.value / item.progress.max * 100}%` : '0%',
+			'--pp': item.preprocessProgress != null ? `${item.preprocessProgress * 100}%` : '100%',
+		}"
 		@contextmenu.prevent.stop="onContextmenu(item, $event)"
 	>
 		<div :class="$style.itemInner">
@@ -19,11 +22,15 @@ SPDX-License-Identifier: AGPL-3.0-only
 			</div>
 			<div :class="$style.itemThumbnail" :style="{ backgroundImage: `url(${ item.thumbnail })` }" @click="onThumbnailClick(item, $event)"></div>
 			<div :class="$style.itemBody">
-				<div><i v-if="item.isSensitive" style="color: var(--MI_THEME-warn); margin-right: 0.5em;" class="ti ti-eye-exclamation"></i><MkCondensedLine :minScale="2 / 3">{{ item.name }}</MkCondensedLine></div>
+				<div>
+					<i v-if="item.isSensitive" style="color: var(--MI_THEME-warn); margin-right: 0.5em;" class="ti ti-eye-exclamation"></i>
+					<MkCondensedLine :minScale="2 / 3">{{ item.name }}</MkCondensedLine>
+				</div>
 				<div :class="$style.itemInfo">
 					<span>{{ item.file.type }}</span>
 					<span v-if="item.compressedSize">({{ i18n.tsx._uploader.compressedToX({ x: bytes(item.compressedSize) }) }} = {{ i18n.tsx._uploader.savedXPercent({ x: Math.round((1 - item.compressedSize / item.file.size) * 100) }) }})</span>
 					<span v-else>{{ bytes(item.file.size) }}</span>
+					<span v-if="item.preprocessing">{{ i18n.ts.preprocessing }}<MkLoading inline em style="margin-left: 0.5em;"/></span>
 				</div>
 				<div>
 				</div>
@@ -97,7 +104,7 @@ function onThumbnailClick(item: UploaderItem, ev: MouseEvent) {
 			position: absolute;
 			top: 0;
 			left: 0;
-			width: 100%;
+			width: var(--pp, 100%);
 			height: 100%;
 			background: linear-gradient(-45deg, transparent 25%, var(--c) 25%,var(--c) 50%, transparent 50%, transparent 75%, var(--c) 75%, var(--c));
 			background-size: 25px 25px;
diff --git a/packages/frontend/src/composables/use-uploader.ts b/packages/frontend/src/composables/use-uploader.ts
index 826d8c5203..12b6e85940 100644
--- a/packages/frontend/src/composables/use-uploader.ts
+++ b/packages/frontend/src/composables/use-uploader.ts
@@ -43,6 +43,12 @@ const IMAGE_EDITING_SUPPORTED_TYPES = [
 	'image/webp',
 ];
 
+const VIDEO_COMPRESSION_SUPPORTED_TYPES = [ // TODO
+	'video/mp4',
+	'video/quicktime',
+	'video/x-matroska',
+];
+
 const WATERMARK_SUPPORTED_TYPES = IMAGE_EDITING_SUPPORTED_TYPES;
 
 const IMAGE_PREPROCESS_NEEDED_TYPES = [
@@ -51,6 +57,10 @@ const IMAGE_PREPROCESS_NEEDED_TYPES = [
 	...IMAGE_EDITING_SUPPORTED_TYPES,
 ];
 
+const VIDEO_PREPROCESS_NEEDED_TYPES = [
+	...VIDEO_COMPRESSION_SUPPORTED_TYPES,
+];
+
 const mimeTypeMap = {
 	'image/webp': 'webp',
 	'image/jpeg': 'jpg',
@@ -64,6 +74,7 @@ export type UploaderItem = {
 	progress: { max: number; value: number } | null;
 	thumbnail: string | null;
 	preprocessing: boolean;
+	preprocessProgress: number | null;
 	uploading: boolean;
 	uploaded: Misskey.entities.DriveFile | null;
 	uploadFailed: boolean;
@@ -76,6 +87,7 @@ export type UploaderItem = {
 	isSensitive?: boolean;
 	caption?: string | null;
 	abort?: (() => void) | null;
+	abortPreprocess?: (() => void) | null;
 };
 
 function getCompressionSettings(level: 0 | 1 | 2 | 3) {
@@ -129,11 +141,12 @@ export function useUploader(options: {
 			progress: null,
 			thumbnail: THUMBNAIL_SUPPORTED_TYPES.includes(file.type) ? window.URL.createObjectURL(file) : null,
 			preprocessing: false,
+			preprocessProgress: null,
 			uploading: false,
 			aborted: false,
 			uploaded: null,
 			uploadFailed: false,
-			compressionLevel: prefer.s.defaultImageCompressionLevel,
+			compressionLevel: IMAGE_COMPRESSION_SUPPORTED_TYPES.includes(file.type) ? prefer.s.defaultImageCompressionLevel : VIDEO_COMPRESSION_SUPPORTED_TYPES.includes(file.type) ? prefer.s.defaultVideoCompressionLevel : 0,
 			watermarkPresetId: uploaderFeatures.value.watermark && $i.policies.watermarkAvailable ? prefer.s.defaultWatermarkPresetId : null,
 			file: markRaw(file),
 		});
@@ -318,7 +331,7 @@ export function useUploader(options: {
 		}
 
 		if (
-			IMAGE_COMPRESSION_SUPPORTED_TYPES.includes(item.file.type) &&
+			(IMAGE_COMPRESSION_SUPPORTED_TYPES.includes(item.file.type) || VIDEO_COMPRESSION_SUPPORTED_TYPES.includes(item.file.type)) &&
 			!item.preprocessing &&
 			!item.uploading &&
 			!item.uploaded
@@ -391,6 +404,19 @@ export function useUploader(options: {
 					removeItem(item);
 				},
 			});
+		} else if (item.preprocessing && item.abortPreprocess != null) {
+			menu.push({
+				type: 'divider',
+			}, {
+				icon: 'ti ti-player-stop',
+				text: i18n.ts.abort,
+				danger: true,
+				action: () => {
+					if (item.abortPreprocess != null) {
+						item.abortPreprocess();
+					}
+				},
+			});
 		} else if (item.uploading) {
 			menu.push({
 				type: 'divider',
@@ -474,6 +500,10 @@ export function useUploader(options: {
 				continue;
 			}
 
+			if (item.abortPreprocess != null) {
+				item.abortPreprocess();
+			}
+
 			if (item.abort != null) {
 				item.abort();
 			}
@@ -484,18 +514,30 @@ export function useUploader(options: {
 
 	async function preprocess(item: UploaderItem): Promise<void> {
 		item.preprocessing = true;
+		item.preprocessProgress = null;
 
-		try {
-			if (IMAGE_PREPROCESS_NEEDED_TYPES.includes(item.file.type)) {
+		if (IMAGE_PREPROCESS_NEEDED_TYPES.includes(item.file.type)) {
+			try {
 				await preprocessForImage(item);
-			}
-		} catch (err) {
-			console.error('Failed to preprocess image', err);
+			} catch (err) {
+				console.error('Failed to preprocess image', err);
 
 			// nop
+			}
+		}
+
+		if (VIDEO_PREPROCESS_NEEDED_TYPES.includes(item.file.type)) {
+			try {
+				await preprocessForVideo(item);
+			} catch (err) {
+				console.error('Failed to preprocess video', err);
+
+				// nop
+			}
 		}
 
 		item.preprocessing = false;
+		item.preprocessProgress = null;
 	}
 
 	async function preprocessForImage(item: UploaderItem): Promise<void> {
@@ -564,10 +606,74 @@ export function useUploader(options: {
 		item.preprocessedFile = markRaw(preprocessedFile);
 	}
 
-	onUnmounted(() => {
+	async function preprocessForVideo(item: UploaderItem): Promise<void> {
+		let preprocessedFile: Blob | File = item.file;
+
+		const needsCompress = item.compressionLevel !== 0 && VIDEO_COMPRESSION_SUPPORTED_TYPES.includes(preprocessedFile.type);
+
+		if (needsCompress) {
+			const mediabunny = await import('mediabunny');
+
+			const source = new mediabunny.BlobSource(preprocessedFile);
+
+			const input = new mediabunny.Input({
+				source,
+				formats: mediabunny.ALL_FORMATS,
+			});
+
+			const output = new mediabunny.Output({
+				target: new mediabunny.BufferTarget(),
+				format: new mediabunny.Mp4OutputFormat(),
+			});
+
+			const currentConversion = await mediabunny.Conversion.init({
+				input,
+				output,
+				video: {
+					//width: 320, // Height will be deduced automatically to retain aspect ratio
+					bitrate: item.compressionLevel === 1 ? mediabunny.QUALITY_VERY_HIGH : item.compressionLevel === 2 ? mediabunny.QUALITY_MEDIUM : mediabunny.QUALITY_VERY_LOW,
+				},
+				audio: {
+					bitrate: item.compressionLevel === 1 ? mediabunny.QUALITY_VERY_HIGH : item.compressionLevel === 2 ? mediabunny.QUALITY_MEDIUM : mediabunny.QUALITY_VERY_LOW,
+				},
+			});
+
+			currentConversion.onProgress = newProgress => item.preprocessProgress = newProgress;
+
+			item.abortPreprocess = () => {
+				item.abortPreprocess = null;
+				currentConversion.cancel();
+				item.preprocessing = false;
+				item.preprocessProgress = null;
+			};
+
+			await currentConversion.execute();
+
+			item.abortPreprocess = null;
+
+			preprocessedFile = new Blob([output.target.buffer!], { type: output.format.mimeType });
+			item.compressedSize = output.target.buffer!.byteLength;
+			item.uploadName = `${item.name}.mp4`;
+		} else {
+			item.compressedSize = null;
+			item.uploadName = item.name;
+		}
+
+		if (item.thumbnail != null) URL.revokeObjectURL(item.thumbnail);
+		item.thumbnail = THUMBNAIL_SUPPORTED_TYPES.includes(preprocessedFile.type) ? window.URL.createObjectURL(preprocessedFile) : null;
+		item.preprocessedFile = markRaw(preprocessedFile);
+	}
+
+	function dispose() {
 		for (const item of items.value) {
 			if (item.thumbnail != null) URL.revokeObjectURL(item.thumbnail);
 		}
+
+		abortAll();
+	}
+
+	onUnmounted(() => {
+		dispose();
 	});
 
 	return {
@@ -575,6 +681,7 @@ export function useUploader(options: {
 		addFiles,
 		removeItem,
 		abortAll,
+		dispose,
 		upload,
 		getMenu,
 		uploading: computed(() => items.value.some(item => item.uploading)),
diff --git a/packages/frontend/src/pages/settings/drive.vue b/packages/frontend/src/pages/settings/drive.vue
index 2d794f2e30..f58ff4c78c 100644
--- a/packages/frontend/src/pages/settings/drive.vue
+++ b/packages/frontend/src/pages/settings/drive.vue
@@ -129,13 +129,37 @@ SPDX-License-Identifier: AGPL-3.0-only
 							<MkSelect
 								v-model="defaultImageCompressionLevel" :items="[
 									{ label: i18n.ts.none, value: 0 },
-									{ label: i18n.ts.low, value: 1 },
-									{ label: i18n.ts.medium, value: 2 },
-									{ label: i18n.ts.high, value: 3 },
+									{ label: `${i18n.ts.low} (${i18n.ts._compression._quality.high}; ${i18n.ts._compression._size.large})`, value: 1 },
+									{ label: `${i18n.ts.medium} (${i18n.ts._compression._quality.medium}; ${i18n.ts._compression._size.medium})`, value: 2 },
+									{ label: `${i18n.ts.high} (${i18n.ts._compression._quality.low}; ${i18n.ts._compression._size.small})`, value: 3 },
 								]"
 							>
-								<template #label><SearchLabel>{{ i18n.ts.defaultImageCompressionLevel }}</SearchLabel></template>
-								<template #caption><div v-html="i18n.ts.defaultImageCompressionLevel_description"></div></template>
+								<template #label><SearchLabel>{{ i18n.ts.defaultCompressionLevel }}</SearchLabel></template>
+								<template #caption><div v-html="i18n.ts.defaultCompressionLevel_description"></div></template>
+							</MkSelect>
+						</MkPreferenceContainer>
+					</SearchMarker>
+				</div>
+			</FormSection>
+		</SearchMarker>
+
+		<SearchMarker :keywords="['video']">
+			<FormSection>
+				<template #label><SearchLabel>{{ i18n.ts.video }}</SearchLabel></template>
+
+				<div class="_gaps_m">
+					<SearchMarker :keywords="['default', 'video', 'compression']">
+						<MkPreferenceContainer k="defaultVideoCompressionLevel">
+							<MkSelect
+								v-model="defaultVideoCompressionLevel" :items="[
+									{ label: i18n.ts.none, value: 0 },
+									{ label: `${i18n.ts.low} (${i18n.ts._compression._quality.high}; ${i18n.ts._compression._size.large})`, value: 1 },
+									{ label: `${i18n.ts.medium} (${i18n.ts._compression._quality.medium}; ${i18n.ts._compression._size.medium})`, value: 2 },
+									{ label: `${i18n.ts.high} (${i18n.ts._compression._quality.low}; ${i18n.ts._compression._size.small})`, value: 3 },
+								]"
+							>
+								<template #label><SearchLabel>{{ i18n.ts.defaultCompressionLevel }}</SearchLabel></template>
+								<template #caption><div v-html="i18n.ts.defaultCompressionLevel_description"></div></template>
 							</MkSelect>
 						</MkPreferenceContainer>
 					</SearchMarker>
@@ -196,6 +220,7 @@ const meterStyle = computed(() => {
 const keepOriginalFilename = prefer.model('keepOriginalFilename');
 const defaultWatermarkPresetId = prefer.model('defaultWatermarkPresetId');
 const defaultImageCompressionLevel = prefer.model('defaultImageCompressionLevel');
+const defaultVideoCompressionLevel = prefer.model('defaultVideoCompressionLevel');
 
 const watermarkPresetsSyncEnabled = ref(prefer.isSyncEnabled('watermarkPresets'));
 
diff --git a/packages/frontend/src/preferences/def.ts b/packages/frontend/src/preferences/def.ts
index df9c366118..a1e5ab888d 100644
--- a/packages/frontend/src/preferences/def.ts
+++ b/packages/frontend/src/preferences/def.ts
@@ -439,6 +439,9 @@ export const PREF_DEF = definePreferences({
 	defaultImageCompressionLevel: {
 		default: 2 as 0 | 1 | 2 | 3,
 	},
+	defaultVideoCompressionLevel: {
+		default: 2 as 0 | 1 | 2 | 3,
+	},
 
 	'sound.masterVolume': {
 		default: 0.5,