From 9703ba53405b2f355c6e0317f714d82ff3d4dee3 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Sun, 12 Jan 2020 16:40:58 +0900 Subject: ファイルと画像認識処理の改善 (#5690) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dimensions制限とリファクタ * comment * 不要な変更削除 * use fromFile など * Add probe-image-size.d.ts * えーCRLFで作るなよ… * Update src/@types/probe-image-size.d.ts Co-Authored-By: Acid Chicken (硫酸鶏) * fix d.ts * Update src/@types/probe-image-size.d.ts Co-Authored-By: Acid Chicken (硫酸鶏) * Update src/@types/probe-image-size.d.ts Co-Authored-By: Acid Chicken (硫酸鶏) * fix Co-authored-by: Acid Chicken (硫酸鶏) --- src/misc/check-svg.ts | 12 --- src/misc/detect-mine.ts | 31 ------- src/misc/detect-url-mime.ts | 15 ++++ src/misc/detect-url-mine.ts | 15 ---- src/misc/get-file-info.ts | 201 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 216 insertions(+), 58 deletions(-) delete mode 100644 src/misc/check-svg.ts delete mode 100644 src/misc/detect-mine.ts create mode 100644 src/misc/detect-url-mime.ts delete mode 100644 src/misc/detect-url-mine.ts create mode 100644 src/misc/get-file-info.ts (limited to 'src/misc') diff --git a/src/misc/check-svg.ts b/src/misc/check-svg.ts deleted file mode 100644 index 8ddeefede9..0000000000 --- a/src/misc/check-svg.ts +++ /dev/null @@ -1,12 +0,0 @@ -import * as fs from 'fs'; -import isSvg from 'is-svg'; - -export default function(path: string) { - try { - const size = fs.statSync(path).size; - if (size > 1 * 1024 * 1024) return false; - return isSvg(fs.readFileSync(path)); - } catch { - return false; - } -} diff --git a/src/misc/detect-mine.ts b/src/misc/detect-mine.ts deleted file mode 100644 index f47f127353..0000000000 --- a/src/misc/detect-mine.ts +++ /dev/null @@ -1,31 +0,0 @@ -import * as fs from 'fs'; -import checkSvg from '../misc/check-svg'; -const FileType = require('file-type'); - -export async function detectMine(path: string) { - return new Promise<[string, string | null]>((res, rej) => { - const readable = fs.createReadStream(path); - readable - .on('error', rej) - .once('data', async (buffer: Buffer) => { - readable.destroy(); - const type = await FileType.fromBuffer(buffer); - if (type) { - if (type.mime == 'application/xml' && checkSvg(path)) { - res(['image/svg+xml', 'svg']); - } else { - res([type.mime, type.ext]); - } - } else if (checkSvg(path)) { - res(['image/svg+xml', 'svg']); - } else { - // 種類が同定できなかったら application/octet-stream にする - res(['application/octet-stream', null]); - } - }) - .on('end', () => { - // maybe 0 bytes - res(['application/octet-stream', null]); - }); - }); -} diff --git a/src/misc/detect-url-mime.ts b/src/misc/detect-url-mime.ts new file mode 100644 index 0000000000..8d71cd0137 --- /dev/null +++ b/src/misc/detect-url-mime.ts @@ -0,0 +1,15 @@ +import { createTemp } from './create-temp'; +import { downloadUrl } from './donwload-url'; +import { detectType } from './get-file-info'; + +export async function detectUrlMime(url: string) { + const [path, cleanup] = await createTemp(); + + try { + await downloadUrl(url, path); + const { mime } = await detectType(path); + return mime; + } finally { + cleanup(); + } +} diff --git a/src/misc/detect-url-mine.ts b/src/misc/detect-url-mine.ts deleted file mode 100644 index eef64cfc56..0000000000 --- a/src/misc/detect-url-mine.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { createTemp } from './create-temp'; -import { downloadUrl } from './donwload-url'; -import { detectMine } from './detect-mine'; - -export async function detectUrlMine(url: string) { - const [path, cleanup] = await createTemp(); - - try { - await downloadUrl(url, path); - const [type] = await detectMine(path); - return type; - } finally { - cleanup(); - } -} diff --git a/src/misc/get-file-info.ts b/src/misc/get-file-info.ts new file mode 100644 index 0000000000..5ccb280260 --- /dev/null +++ b/src/misc/get-file-info.ts @@ -0,0 +1,201 @@ +import * as fs from 'fs'; +import * as crypto from 'crypto'; +import * as fileType from 'file-type'; +import isSvg from 'is-svg'; +import * as probeImageSize from 'probe-image-size'; +import * as sharp from 'sharp'; + +export type FileInfo = { + size: number; + md5: string; + type: { + mime: string; + ext: string | null; + }; + width?: number; + height?: number; + avgColor?: number[]; + warnings: string[]; +}; + +const TYPE_OCTET_STREAM = { + mime: 'application/octet-stream', + ext: null +}; + +const TYPE_SVG = { + mime: 'image/svg+xml', + ext: 'svg' +}; + +/** + * Get file information + */ +export async function getFileInfo(path: string): Promise { + const warnings = [] as string[]; + + const size = await getFileSize(path); + const md5 = await calcHash(path); + + let type = await detectType(path); + + // image dimensions + let width: number | undefined; + let height: number | undefined; + + if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/bmp', 'image/tiff', 'image/svg+xml', 'image/vnd.adobe.photoshop'].includes(type.mime)) { + const imageSize = await detectImageSize(path).catch(e => { + warnings.push(`detectImageSize failed: ${e}`); + return undefined; + }); + + // うまく判定できない画像は octet-stream にする + if (!imageSize) { + warnings.push(`cannot detect image dimensions`); + type = TYPE_OCTET_STREAM; + } else if (imageSize.wUnits === 'px') { + width = imageSize.width; + height = imageSize.height; + + // 制限を超えている画像は octet-stream にする + if (imageSize.width > 16383 || imageSize.height > 16383) { + warnings.push(`image dimensions exceeds limits`); + type = TYPE_OCTET_STREAM; + } + } else { + warnings.push(`unsupported unit type: ${imageSize.wUnits}`); + } + } + + // average color + let avgColor: number[] | undefined; + + if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/svg+xml'].includes(type.mime)) { + avgColor = await calcAvgColor(path).catch(e => { + warnings.push(`calcAvgColor failed: ${e}`); + return undefined; + }); + } + + return { + size, + md5, + type, + width, + height, + avgColor, + warnings, + }; +} + +/** + * Detect MIME Type and extension + */ +export async function detectType(path: string) { + // Check 0 byte + const fileSize = await getFileSize(path); + if (fileSize === 0) { + return TYPE_OCTET_STREAM; + } + + const type = await fileType.fromFile(path); + + if (type) { + // XMLはSVGかもしれない + if (type.mime === 'application/xml' && await checkSvg(path)) { + return TYPE_SVG; + } + + return { + mime: type.mime, + ext: type.ext + }; + } + + // 種類が不明でもSVGかもしれない + if (await checkSvg(path)) { + return TYPE_SVG; + } + + // それでも種類が不明なら application/octet-stream にする + return TYPE_OCTET_STREAM; +} + +/** + * Check the file is SVG or not + */ +export async function checkSvg(path: string) { + try { + const size = await getFileSize(path); + if (size > 1 * 1024 * 1024) return false; + return isSvg(fs.readFileSync(path)); + } catch { + return false; + } +} + +/** + * Get file size + */ +export async function getFileSize(path: string): Promise { + return new Promise((res, rej) => { + fs.stat(path, (err, stats) => { + if (err) return rej(err); + res(stats.size); + }); + }); +} + +/** + * Calculate MD5 hash + */ +async function calcHash(path: string): Promise { + return new Promise((res, rej) => { + const readable = fs.createReadStream(path); + const hash = crypto.createHash('md5'); + const chunks: Buffer[] = []; + readable + .on('error', rej) + .pipe(hash) + .on('error', rej) + .on('data', chunk => chunks.push(chunk)) + .on('end', () => { + const buffer = Buffer.concat(chunks); + res(buffer.toString('hex')); + }); + }); +} + +/** + * Detect dimensions of image + */ +async function detectImageSize(path: string): Promise<{ + width: number; + height: number; + wUnits: string; + hUnits: string; +}> { + const readable = fs.createReadStream(path); + const imageSize = await probeImageSize(readable); + readable.destroy(); + return imageSize; +} + +/** + * Calculate average color of image + */ +async function calcAvgColor(path: string): Promise { + const img = sharp(path); + + const info = await (img as any).stats(); + + if (info.isOpaque) { + const r = Math.round(info.channels[0].mean); + const g = Math.round(info.channels[1].mean); + const b = Math.round(info.channels[2].mean); + + return [r, g, b]; + } else { + return [255, 255, 255]; + } +} -- cgit v1.2.3-freya