feat(frontend): EXIFフレーム機能 (#16725)

* wip

* wip

* Update ImageEffector.ts

* Update image-label-renderer.ts

* Update image-label-renderer.ts

* wip

* Update image-label-renderer.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update use-uploader.ts

* Update watermark.ts

* wip

* wu

* wip

* Update image-frame-renderer.ts

* wip

* wip

* Update image-frame-renderer.ts

* Create ImageCompositor.ts

* Update ImageCompositor.ts

* wip

* wip

* Update ImageEffector.ts

* wip

* Update use-uploader.ts

* wip

* wip

* wip

* wip

* Update fxs.ts

* wip

* wip

* wip

* Update CHANGELOG.md

* wip

* wip

* Update MkImageEffectorDialog.vue

* Update MkImageEffectorDialog.vue

* Update MkImageFrameEditorDialog.vue

* Update use-uploader.ts

* improve error handling

* Update use-uploader.ts

* 🎨

* wip

* wip

* lazy load

* lazy load

* wip

* wip

* wip

2 files changed, 315 insertions, 0 deletions

diff --git a/packages/frontend/src/lib/ImageCompositor.ts b/packages/frontend/src/lib/ImageCompositor.ts
new file mode 100644
index 0000000000..a26302af77
--- /dev/null
+++ b/packages/frontend/src/lib/ImageCompositor.ts
@@ -0,0 +1,313 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { createTexture, initShaderProgram } from '../utility/webgl.js';
+
+export type ImageCompositorFunctionParams = Record<string, any>;
+
+export type ImageCompositorFunction<PS extends ImageCompositorFunctionParams = ImageCompositorFunctionParams> = {
+	shader: string;
+	main: (ctx: {
+		gl: WebGL2RenderingContext;
+		program: WebGLProgram;
+		params: PS;
+		u: Record<string, WebGLUniformLocation>;
+		width: number;
+		height: number;
+		textures: Map<string, { texture: WebGLTexture; width: number; height: number; }>;
+	}) => void;
+};
+
+export type ImageCompositorLayer<FNS extends Record<string, ImageCompositorFunction> = any> = {
+	[K in keyof FNS]: {
+		id: string;
+		functionId: K;
+		params: Parameters<FNS[K]['main']>[0]['params'];
+	};
+}[keyof FNS];
+
+export function defineImageCompositorFunction<PS extends ImageCompositorFunctionParams>(fn: ImageCompositorFunction<PS>) {
+	return fn;
+}
+
+// TODO: per layer cache
+
+export class ImageCompositor<FNS extends Record<string, ImageCompositorFunction<any>>> {
+	private gl: WebGL2RenderingContext;
+	private canvas: HTMLCanvasElement | null = null;
+	private renderWidth: number;
+	private renderHeight: number;
+	private baseTexture: WebGLTexture;
+	private shaderCache: Map<string, WebGLProgram> = new Map();
+	private perLayerResultTextures: Map<string, WebGLTexture> = new Map();
+	private perLayerResultFrameBuffers: Map<string, WebGLFramebuffer> = new Map();
+	private nopProgram: WebGLProgram;
+	private registeredTextures: Map<string, { texture: WebGLTexture; width: number; height: number; }> = new Map();
+	private registeredFunctions: Map<string, ImageCompositorFunction & { id: string; uniforms: string[] }> = new Map();
+
+	constructor(options: {
+		canvas: HTMLCanvasElement;
+		renderWidth: number;
+		renderHeight: number;
+		image: ImageData | ImageBitmap | HTMLImageElement | HTMLCanvasElement | null;
+		functions: FNS;
+	}) {
+		this.canvas = options.canvas;
+		this.renderWidth = options.renderWidth;
+		this.renderHeight = options.renderHeight;
+
+		this.canvas.width = this.renderWidth;
+		this.canvas.height = this.renderHeight;
+
+		const gl = this.canvas.getContext('webgl2', {
+			preserveDrawingBuffer: false,
+			alpha: true,
+			premultipliedAlpha: false,
+		});
+
+		if (gl == null) throw new Error('Failed to initialize WebGL2 context');
+
+		this.gl = gl;
+
+		gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
+
+		const VERTICES = new Float32Array([-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1]);
+		const vertexBuffer = gl.createBuffer();
+		gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+		gl.bufferData(gl.ARRAY_BUFFER, VERTICES, gl.STATIC_DRAW);
+
+		if (options.image != null) {
+			this.baseTexture = createTexture(gl);
+			gl.activeTexture(gl.TEXTURE0);
+			gl.bindTexture(gl.TEXTURE_2D, this.baseTexture);
+			gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, options.image.width, options.image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, options.image);
+			gl.bindTexture(gl.TEXTURE_2D, null);
+		} else {
+			this.baseTexture = createTexture(gl);
+			gl.activeTexture(gl.TEXTURE0);
+		}
+
+		this.nopProgram = initShaderProgram(this.gl, `#version 300 es
+			in vec2 position;
+			out vec2 in_uv;
+
+			void main() {
+				in_uv = (position + 1.0) / 2.0;
+				gl_Position = vec4(position * vec2(1.0, -1.0), 0.0, 1.0);
+			}
+		`, `#version 300 es
+			precision mediump float;
+
+			in vec2 in_uv;
+			uniform sampler2D u_texture;
+			out vec4 out_color;
+
+			void main() {
+				out_color = texture(u_texture, in_uv);
+			}
+		`);
+
+		// レジスタ番号はシェーダープログラムに属しているわけではなく、独立の存在なので、とりあえず nopProgram を使って設定する(その後は効果が持続する)
+		// ref. https://qiita.com/emadurandal/items/5966c8374f03d4de3266
+		const positionLocation = gl.getAttribLocation(this.nopProgram, 'position');
+		gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
+		gl.enableVertexAttribArray(positionLocation);
+
+		for (const [id, fn] of Object.entries(options.functions)) {
+			const uniforms = this.extractUniformNamesFromShader(fn.shader);
+			this.registeredFunctions.set(id, { ...fn, id, uniforms });
+		}
+	}
+
+	private extractUniformNamesFromShader(shader: string): string[] {
+		const uniformRegex = /uniform\s+\w+\s+(\w+)\s*;/g;
+		const uniforms: string[] = [];
+		let match;
+		while ((match = uniformRegex.exec(shader)) !== null) {
+			uniforms.push(match[1].replace(/^u_/, ''));
+		}
+		return uniforms;
+	}
+
+	private renderLayer(layer: ImageCompositorLayer, preTexture: WebGLTexture, invert = false) {
+		const gl = this.gl;
+
+		const fn = this.registeredFunctions.get(layer.functionId);
+		if (fn == null) return;
+
+		const cachedShader = this.shaderCache.get(fn.id);
+		const shaderProgram = cachedShader ?? initShaderProgram(this.gl, `#version 300 es
+			in vec2 position;
+			uniform bool u_invert;
+			out vec2 in_uv;
+
+			void main() {
+				in_uv = (position + 1.0) / 2.0;
+				gl_Position = u_invert ? vec4(position * vec2(1.0, -1.0), 0.0, 1.0) : vec4(position, 0.0, 1.0);
+			}
+		`, fn.shader);
+		if (cachedShader == null) {
+			this.shaderCache.set(fn.id, shaderProgram);
+		}
+
+		gl.useProgram(shaderProgram);
+
+		const in_resolution = gl.getUniformLocation(shaderProgram, 'in_resolution');
+		gl.uniform2fv(in_resolution, [this.renderWidth, this.renderHeight]);
+
+		const u_invert = gl.getUniformLocation(shaderProgram, 'u_invert');
+		gl.uniform1i(u_invert, invert ? 1 : 0);
+
+		gl.activeTexture(gl.TEXTURE0);
+		gl.bindTexture(gl.TEXTURE_2D, preTexture);
+		const in_texture = gl.getUniformLocation(shaderProgram, 'in_texture');
+		gl.uniform1i(in_texture, 0);
+
+		fn.main({
+			gl: gl,
+			program: shaderProgram,
+			params: layer.params,
+			u: Object.fromEntries(fn.uniforms.map(u => [u, gl.getUniformLocation(shaderProgram, 'u_' + u)!])),
+			width: this.renderWidth,
+			height: this.renderHeight,
+			textures: this.registeredTextures,
+		});
+
+		gl.drawArrays(gl.TRIANGLES, 0, 6);
+	}
+
+	public render(layers: (ImageCompositorLayer<FNS>)[]) {
+		const gl = this.gl;
+
+		// 入力をそのまま出力
+		if (layers.length === 0) {
+			gl.activeTexture(gl.TEXTURE0);
+			gl.bindTexture(gl.TEXTURE_2D, this.baseTexture);
+
+			gl.useProgram(this.nopProgram);
+			gl.uniform1i(gl.getUniformLocation(this.nopProgram, 'u_texture')!, 0);
+
+			gl.drawArrays(gl.TRIANGLES, 0, 6);
+			return;
+		}
+
+		let preTexture = this.baseTexture;
+
+		for (const layer of layers) {
+			const isLast = layer === layers.at(-1);
+
+			const cachedResultTexture = this.perLayerResultTextures.get(layer.id);
+			const resultTexture = cachedResultTexture ?? createTexture(gl);
+			if (cachedResultTexture == null) {
+				this.perLayerResultTextures.set(layer.id, resultTexture);
+			}
+			gl.bindTexture(gl.TEXTURE_2D, resultTexture);
+			gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this.renderWidth, this.renderHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+			gl.bindTexture(gl.TEXTURE_2D, null);
+
+			if (isLast) {
+				gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+			} else {
+				const cachedResultFrameBuffer = this.perLayerResultFrameBuffers.get(layer.id);
+				const resultFrameBuffer = cachedResultFrameBuffer ?? gl.createFramebuffer();
+				if (cachedResultFrameBuffer == null) {
+					this.perLayerResultFrameBuffers.set(layer.id, resultFrameBuffer);
+				}
+				gl.bindFramebuffer(gl.FRAMEBUFFER, resultFrameBuffer);
+				gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, resultTexture, 0);
+			}
+
+			this.renderLayer(layer, preTexture, isLast);
+
+			preTexture = resultTexture;
+		}
+	}
+
+	public registerTexture(key: string, image: ImageData | ImageBitmap | HTMLImageElement | HTMLCanvasElement) {
+		const gl = this.gl;
+
+		const existing = this.registeredTextures.get(key);
+		if (existing != null) {
+			gl.deleteTexture(existing.texture);
+			this.registeredTextures.delete(key);
+		}
+
+		const texture = createTexture(gl);
+		gl.activeTexture(gl.TEXTURE0);
+		gl.bindTexture(gl.TEXTURE_2D, texture);
+		gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, image);
+		gl.bindTexture(gl.TEXTURE_2D, null);
+
+		this.registeredTextures.set(key, {
+			texture: texture,
+			width: image.width,
+			height: image.height,
+		});
+	}
+
+	public unregisterTexture(key: string) {
+		const gl = this.gl;
+
+		const existing = this.registeredTextures.get(key);
+		if (existing != null) {
+			gl.deleteTexture(existing.texture);
+			this.registeredTextures.delete(key);
+		}
+	}
+
+	public hasTexture(key: string) {
+		return this.registeredTextures.has(key);
+	}
+
+	public getKeysOfRegisteredTextures() {
+		return this.registeredTextures.keys();
+	}
+
+	public changeResolution(width: number, height: number) {
+		if (this.renderWidth === width && this.renderHeight === height) return;
+
+		this.renderWidth = width;
+		this.renderHeight = height;
+		if (this.canvas) {
+			this.canvas.width = this.renderWidth;
+			this.canvas.height = this.renderHeight;
+		}
+		this.gl.viewport(0, 0, this.renderWidth, this.renderHeight);
+	}
+
+	/*
+	 * disposeCanvas = true だとloseContextを呼ぶため、コンストラクタで渡されたcanvasも再利用不可になるので注意
+	 */
+	public destroy(disposeCanvas = true) {
+		this.gl.deleteProgram(this.nopProgram);
+
+		for (const shader of this.shaderCache.values()) {
+			this.gl.deleteProgram(shader);
+		}
+		this.shaderCache.clear();
+
+		for (const texture of this.perLayerResultTextures.values()) {
+			this.gl.deleteTexture(texture);
+		}
+		this.perLayerResultTextures.clear();
+
+		for (const framebuffer of this.perLayerResultFrameBuffers.values()) {
+			this.gl.deleteFramebuffer(framebuffer);
+		}
+		this.perLayerResultFrameBuffers.clear();
+
+		for (const texture of this.registeredTextures.values()) {
+			this.gl.deleteTexture(texture.texture);
+		}
+		this.registeredTextures.clear();
+
+		this.gl.deleteTexture(this.baseTexture);
+
+		if (disposeCanvas) {
+			const loseContextExt = this.gl.getExtension('WEBGL_lose_context');
+			if (loseContextExt) loseContextExt.loseContext();
+		}
+	}
+}
diff --git a/packages/frontend/src/lib/pizzax.ts b/packages/frontend/src/lib/pizzax.ts
index 6dffcf9478..8faac6155c 100644
--- a/packages/frontend/src/lib/pizzax.ts
+++ b/packages/frontend/src/lib/pizzax.ts
@@ -5,6 +5,8 @@
 
 // PIZZAX --- A lightweight store
 
+// TODO: Misskeyのドメイン知識があるのでutilityなどに移動する
+
 import { onUnmounted, ref, watch } from 'vue';
 import { BroadcastChannel } from 'broadcast-channel';
 import type { Ref } from 'vue';