From 44a01c4b5a46d2b8b27cb6315babac8484051f7b Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 16 Apr 2022 17:19:30 +0900 Subject: refactoe --- packages/backend/src/misc/fetch.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/misc/fetch.ts b/packages/backend/src/misc/fetch.ts index 4b1013c9f5..af6bf2fca7 100644 --- a/packages/backend/src/misc/fetch.ts +++ b/packages/backend/src/misc/fetch.ts @@ -1,10 +1,10 @@ -import * as http from 'http'; -import * as https from 'https'; +import * as http from 'node:http'; +import * as https from 'node:https'; +import { URL } from 'node:url'; import CacheableLookup from 'cacheable-lookup'; import fetch from 'node-fetch'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import config from '@/config/index.js'; -import { URL } from 'node:url'; export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record) { const res = await getResponse({ @@ -35,7 +35,7 @@ export async function getHtml(url: string, accept = 'text/html, */*', timeout = } export async function getResponse(args: { url: string, method: string, body?: string, headers: Record, timeout?: number, size?: number }) { - const timeout = args?.timeout || 10 * 1000; + const timeout = args.timeout || 10 * 1000; const controller = new AbortController(); setTimeout(() => { @@ -47,7 +47,7 @@ export async function getResponse(args: { url: string, method: string, body?: st headers: args.headers, body: args.body, timeout, - size: args?.size || 10 * 1024 * 1024, + size: args.size || 10 * 1024 * 1024, agent: getAgentByUrl, signal: controller.signal, }); -- cgit v1.2.3-freya From 31e5c5f69a565b6aaef3f79bcba4bf5e2beee170 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 17 Apr 2022 20:58:37 +0900 Subject: refactor Resolve #8467 --- packages/backend/package.json | 1 - packages/backend/src/misc/cafy-id.ts | 33 ---------------------- .../src/remote/activitypub/models/person.ts | 32 ++++++++++++++------- packages/backend/yarn.lock | 5 ---- 4 files changed, 22 insertions(+), 49 deletions(-) delete mode 100644 packages/backend/src/misc/cafy-id.ts (limited to 'packages/backend/src/misc') diff --git a/packages/backend/package.json b/packages/backend/package.json index f07a15bc54..2354c95646 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -35,7 +35,6 @@ "broadcast-channel": "4.10.0", "bull": "4.8.1", "cacheable-lookup": "6.0.4", - "cafy": "15.2.1", "cbor": "8.1.0", "chalk": "5.0.1", "chalk-template": "0.4.0", diff --git a/packages/backend/src/misc/cafy-id.ts b/packages/backend/src/misc/cafy-id.ts deleted file mode 100644 index dd81c5c4cf..0000000000 --- a/packages/backend/src/misc/cafy-id.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Context } from 'cafy'; - -// eslint-disable-next-line @typescript-eslint/ban-types -export class ID extends Context { - public readonly name = 'ID'; - - constructor(optional = false, nullable = false) { - super(optional, nullable); - - this.push((v: any) => { - if (typeof v !== 'string') { - return new Error('must-be-an-id'); - } - return true; - }); - } - - public getType() { - return super.getType('String'); - } - - public makeOptional(): ID { - return new ID(true, false); - } - - public makeNullable(): ID { - return new ID(false, true); - } - - public makeOptionalNullable(): ID { - return new ID(true, true); - } -} diff --git a/packages/backend/src/remote/activitypub/models/person.ts b/packages/backend/src/remote/activitypub/models/person.ts index f722e74702..6097e3b6ed 100644 --- a/packages/backend/src/remote/activitypub/models/person.ts +++ b/packages/backend/src/remote/activitypub/models/person.ts @@ -1,7 +1,6 @@ import { URL } from 'node:url'; import promiseLimit from 'promise-limit'; -import $, { Context } from 'cafy'; import config from '@/config/index.js'; import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc.js'; import { Note } from '@/models/entities/note.js'; @@ -54,20 +53,33 @@ function validateActor(x: IObject, uri: string): IActor { throw new Error(`invalid Actor type '${x.type}'`); } - const validate = (name: string, value: any, validater: Context) => { - const e = validater.test(value); - if (e) throw new Error(`invalid Actor: ${name} ${e.message}`); - }; + if (!(typeof x.id === 'string' && x.id.length > 0)) { + throw new Error('invalid Actor: wrong id'); + } - validate('id', x.id, $.default.str.min(1)); - validate('inbox', x.inbox, $.default.str.min(1)); - validate('preferredUsername', x.preferredUsername, $.default.str.min(1).max(128).match(/^\w([\w-.]*\w)?$/)); + if (!(typeof x.inbox === 'string' && x.inbox.length > 0)) { + throw new Error('invalid Actor: wrong inbox'); + } + + if (!(typeof x.preferredUsername === 'string' && x.preferredUsername.length > 0 && x.preferredUsername.length <= 128 && /^\w([\w-.]*\w)?$/.test(x.preferredUsername))) { + throw new Error('invalid Actor: wrong username'); + } // These fields are only informational, and some AP software allows these // fields to be very long. If they are too long, we cut them off. This way // we can at least see these users and their activities. - validate('name', truncate(x.name, nameLength), $.default.optional.nullable.str); - validate('summary', truncate(x.summary, summaryLength), $.default.optional.nullable.str); + if (x.name) { + if (!(typeof x.name === 'string' && x.name.length > 0)) { + throw new Error('invalid Actor: wrong name'); + } + x.name = truncate(x.name, nameLength); + } + if (x.summary) { + if (!(typeof x.summary === 'string' && x.summary.length > 0)) { + throw new Error('invalid Actor: wrong summary'); + } + x.summary = truncate(x.summary, summaryLength); + } const idHost = toPuny(new URL(x.id!).hostname); if (idHost !== expectHost) { diff --git a/packages/backend/yarn.lock b/packages/backend/yarn.lock index e4421db9d7..981c359dd7 100644 --- a/packages/backend/yarn.lock +++ b/packages/backend/yarn.lock @@ -1591,11 +1591,6 @@ cacheable-request@^7.0.2: normalize-url "^6.0.1" responselike "^2.0.0" -cafy@15.2.1: - version "15.2.1" - resolved "https://registry.yarnpkg.com/cafy/-/cafy-15.2.1.tgz#5a55eaeb721c604c7dca652f3d555c392e5f995a" - integrity sha512-g2zOmFb63p6XcZ/zeMWKYP8YKQYNWnhJmi6K71Ql4EAFTAay31xF0PBPtdBCCfQ0fiETgWTMxKtySAVI/Od6aQ== - call-bind@^1.0.0, call-bind@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" -- cgit v1.2.3-freya From 1b2ba09be05338b1680d664970ef28b5ea6464a6 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 24 Apr 2022 11:43:15 +0900 Subject: fix: Fix schema key type error #8517 (#8538) --- packages/backend/src/misc/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/misc/schema.ts b/packages/backend/src/misc/schema.ts index 5b69812090..9da13d599b 100644 --- a/packages/backend/src/misc/schema.ts +++ b/packages/backend/src/misc/schema.ts @@ -89,7 +89,7 @@ export interface Schema extends OfSchema { readonly optional?: boolean; readonly items?: Schema; readonly properties?: Obj; - readonly required?: ReadonlyArray>; + readonly required?: ReadonlyArray, string>>; readonly description?: string; readonly example?: any; readonly format?: string; -- cgit v1.2.3-freya From 065324d30bddd2cf1ec48cb539cbb4b43c7b4169 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 27 Apr 2022 10:49:00 +0900 Subject: Fix #8535 Excessive stack ... 'SchemaTypeDef' (#8547) * Fix #8535 Excessive stack ... 'SchemaTypeDef' Co-authored-by: acid-chicken * add comment * clean up Co-authored-by: acid-chicken --- packages/backend/src/misc/schema.ts | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/misc/schema.ts b/packages/backend/src/misc/schema.ts index 9da13d599b..fdecc278d4 100644 --- a/packages/backend/src/misc/schema.ts +++ b/packages/backend/src/misc/schema.ts @@ -98,6 +98,9 @@ export interface Schema extends OfSchema { readonly default?: (this['type'] extends TypeStringef ? StringDefToType : any) | null; readonly maxLength?: number; readonly minLength?: number; + readonly maximum?: number; + readonly minimum?: number; + readonly pattern?: string; } type RequiredPropertyNames = { @@ -105,24 +108,26 @@ type RequiredPropertyNames = { // K is not optional s[K]['optional'] extends false ? K : // K has default value - s[K]['default'] extends null | string | number | boolean | Record ? K : never + s[K]['default'] extends null | string | number | boolean | Record ? K : + never }[keyof s]; -export interface Obj { [key: string]: Schema; } +export type Obj = Record; +// https://github.com/misskey-dev/misskey/issues/8535 +// To avoid excessive stack depth error, +// deceive TypeScript with UnionToIntersection (or more precisely, `infer` expression within it). export type ObjType = - { -readonly [P in keyof s]?: SchemaType } & - { -readonly [P in RequiredProps]: SchemaType } & - { -readonly [P in RequiredPropertyNames]: SchemaType }; + UnionToIntersection< + { -readonly [R in RequiredPropertyNames]-?: SchemaType } & + { -readonly [R in RequiredProps]-?: SchemaType } & + { -readonly [P in keyof s]?: SchemaType } + >; type NullOrUndefined

= - p['nullable'] extends true - ? p['optional'] extends true - ? (T | null | undefined) - : (T | null) - : p['optional'] extends true - ? (T | undefined) - : T; + | (p['nullable'] extends true ? null : never) + | (p['optional'] extends true ? undefined : never) + | T; // https://stackoverflow.com/questions/54938141/typescript-convert-union-to-intersection // Get intersection from union @@ -139,9 +144,9 @@ export type SchemaTypeDef

= p['type'] extends 'number' ? number : p['type'] extends 'string' ? ( p['enum'] extends readonly string[] ? - p['enum'][number] : - p['format'] extends 'date-time' ? string : // Dateにする？？ - string + p['enum'][number] : + p['format'] extends 'date-time' ? string : // Dateにする？？ + string ) : p['type'] extends 'boolean' ? boolean : p['type'] extends 'object' ? ( -- cgit v1.2.3-freya From 12a3c6872f0a31c923bf0cd7c183cb8776d58dda Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 28 Apr 2022 11:14:03 +0900 Subject: enhance: ドライブに画像ファイルをアップロードするときオリジナル画像を破棄してwebpublicのみ保持するオプション (#8216) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * Update packages/client/src/os.ts Co-authored-by: tamaina * メニューをComposition API化、switchアイテム追加 クライアントサイド画像圧縮の準備 * メニュー型定義を分離 (TypeScriptの型支援が効かないので) * disabled * make keepOriginal to follow setting value * :v: * fix * fix * :v: * WEBP * aaa * :v: * webp * lazy load browser-image-resizer * rename * rename 2 * Fix * clean up * add comment * clean up * jpeg, pngにもどす * fix * fix name * webpでなくする ただしサムネやプレビューはwebpのまま (テスト) * 動画サムネイルはjpegに * エラーハンドリング * :v: * v2.2.1-misskey-beta.2 * browser-image-resizer#v2.2.1-misskey.1 * :v: * fix alert * update browser-image-resizer to v2.2.1-misskey.2 * lockfile Co-authored-by: mei23 Co-authored-by: MeiMei <30769358+mei23@users.noreply.github.com> --- packages/backend/src/misc/populate-emojis.ts | 2 +- .../backend/src/server/file/send-drive-file.ts | 8 +- packages/backend/src/server/proxy/proxy-media.ts | 8 +- packages/backend/src/server/web/url-preview.ts | 2 +- packages/backend/src/services/drive/add-file.ts | 29 ++++-- .../src/services/drive/generate-video-thumbnail.ts | 1 + .../backend/src/services/drive/image-processor.ts | 28 +---- packages/client/package.json | 1 + packages/client/src/components/drive.vue | 10 +- packages/client/src/components/post-form.vue | 3 +- packages/client/src/os.ts | 75 +------------- .../src/pages/messaging/messaging-room.form.vue | 3 +- packages/client/src/scripts/select-file.ts | 3 +- packages/client/src/scripts/upload.ts | 114 +++++++++++++++++++++ packages/client/src/ui/_common_/common.vue | 3 +- packages/client/src/ui/_common_/upload.vue | 2 +- packages/client/yarn.lock | 4 + 17 files changed, 169 insertions(+), 127 deletions(-) create mode 100644 packages/client/src/scripts/upload.ts (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/misc/populate-emojis.ts b/packages/backend/src/misc/populate-emojis.ts index 86f1356c31..6a185d09f6 100644 --- a/packages/backend/src/misc/populate-emojis.ts +++ b/packages/backend/src/misc/populate-emojis.ts @@ -63,7 +63,7 @@ export async function populateEmoji(emojiName: string, noteUserHost: string | nu const isLocal = emoji.host == null; const emojiUrl = emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため - const url = isLocal ? emojiUrl : `${config.url}/proxy/image.png?${query({ url: emojiUrl })}`; + const url = isLocal ? emojiUrl : `${config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}`; return { name: emojiName, diff --git a/packages/backend/src/server/file/send-drive-file.ts b/packages/backend/src/server/file/send-drive-file.ts index 6bc220b362..027d078ce1 100644 --- a/packages/backend/src/server/file/send-drive-file.ts +++ b/packages/backend/src/server/file/send-drive-file.ts @@ -11,7 +11,7 @@ import { DriveFiles } from '@/models/index.js'; import { InternalStorage } from '@/services/drive/internal-storage.js'; import { downloadUrl } from '@/misc/download-url.js'; import { detectType } from '@/misc/get-file-info.js'; -import { convertToJpeg, convertToPng, convertToPngOrJpeg } from '@/services/drive/image-processor.js'; +import { convertToWebp, convertToJpeg, convertToPng } from '@/services/drive/image-processor.js'; import { GenerateVideoThumbnail } from '@/services/drive/generate-video-thumbnail.js'; import { StatusError } from '@/misc/fetch.js'; import { FILE_TYPE_BROWSERSAFE } from '@/const.js'; @@ -64,10 +64,8 @@ export default async function(ctx: Koa.Context) { const convertFile = async () => { if (isThumbnail) { - if (['image/jpeg', 'image/webp'].includes(mime)) { - return await convertToJpeg(path, 498, 280); - } else if (['image/png', 'image/svg+xml'].includes(mime)) { - return await convertToPngOrJpeg(path, 498, 280); + if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(mime)) { + return await convertToWebp(path, 498, 280); } else if (mime.startsWith('video/')) { return await GenerateVideoThumbnail(path); } diff --git a/packages/backend/src/server/proxy/proxy-media.ts b/packages/backend/src/server/proxy/proxy-media.ts index 3cc5b827a6..48887bf12f 100644 --- a/packages/backend/src/server/proxy/proxy-media.ts +++ b/packages/backend/src/server/proxy/proxy-media.ts @@ -1,7 +1,7 @@ import * as fs from 'node:fs'; import Koa from 'koa'; import { serverLogger } from '../index.js'; -import { IImage, convertToPng, convertToJpeg } from '@/services/drive/image-processor.js'; +import { IImage, convertToWebp } from '@/services/drive/image-processor.js'; import { createTemp } from '@/misc/create-temp.js'; import { downloadUrl } from '@/misc/download-url.js'; import { detectType } from '@/misc/get-file-info.js'; @@ -27,11 +27,11 @@ export async function proxyMedia(ctx: Koa.Context) { let image: IImage; if ('static' in ctx.query && ['image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/svg+xml'].includes(mime)) { - image = await convertToPng(path, 498, 280); + image = await convertToWebp(path, 498, 280); } else if ('preview' in ctx.query && ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/svg+xml'].includes(mime)) { - image = await convertToJpeg(path, 200, 200); + image = await convertToWebp(path, 200, 200); } else if (['image/svg+xml'].includes(mime)) { - image = await convertToPng(path, 2048, 2048); + image = await convertToWebp(path, 2048, 2048, 1); } else if (!mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(mime)) { throw new StatusError('Rejected type', 403, 'Rejected type'); } else { diff --git a/packages/backend/src/server/web/url-preview.ts b/packages/backend/src/server/web/url-preview.ts index 6bd8ead5b5..1e259649f9 100644 --- a/packages/backend/src/server/web/url-preview.ts +++ b/packages/backend/src/server/web/url-preview.ts @@ -56,7 +56,7 @@ export const urlPreviewHandler = async (ctx: Koa.Context) => { function wrap(url?: string): string | null { return url != null ? url.match(/^https?:\/\//) - ? `${config.url}/proxy/preview.jpg?${query({ + ? `${config.url}/proxy/preview.webp?${query({ url, preview: '1', })}` diff --git a/packages/backend/src/services/drive/add-file.ts b/packages/backend/src/services/drive/add-file.ts index 549b11c9fe..cfbcb60ddf 100644 --- a/packages/backend/src/services/drive/add-file.ts +++ b/packages/backend/src/services/drive/add-file.ts @@ -7,7 +7,7 @@ import { deleteFile } from './delete-file.js'; import { fetchMeta } from '@/misc/fetch-meta.js'; import { GenerateVideoThumbnail } from './generate-video-thumbnail.js'; import { driveLogger } from './logger.js'; -import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng, convertSharpToPngOrJpeg } from './image-processor.js'; +import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js'; import { contentDisposition } from '@/misc/content-disposition.js'; import { getFileInfo } from '@/misc/get-file-info.js'; import { DriveFiles, DriveFolders, Users, Instances, UserProfiles } from '@/models/index.js'; @@ -179,6 +179,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool } let img: sharp.Sharp | null = null; + let satisfyWebpublic: boolean; try { img = sharp(path); @@ -192,6 +193,13 @@ export async function generateAlts(path: string, type: string, generateWeb: bool thumbnail: null, }; } + + satisfyWebpublic = !!( + type !== 'image/svg+xml' && type !== 'image/webp' && + !(metadata.exif || metadata.iptc || metadata.xmp || metadata.tifftagPhotoshop) && + metadata.width && metadata.width <= 2048 && + metadata.height && metadata.height <= 2048 + ); } catch (err) { logger.warn(`sharp failed: ${err}`); return { @@ -203,15 +211,15 @@ export async function generateAlts(path: string, type: string, generateWeb: bool // #region webpublic let webpublic: IImage | null = null; - if (generateWeb) { + if (generateWeb && !satisfyWebpublic) { logger.info(`creating web image`); try { - if (['image/jpeg'].includes(type)) { + if (['image/jpeg', 'image/webp'].includes(type)) { webpublic = await convertSharpToJpeg(img, 2048, 2048); - } else if (['image/webp'].includes(type)) { - webpublic = await convertSharpToWebp(img, 2048, 2048); - } else if (['image/png', 'image/svg+xml'].includes(type)) { + } else if (['image/png'].includes(type)) { + webpublic = await convertSharpToPng(img, 2048, 2048); + } else if (['image/svg+xml'].includes(type)) { webpublic = await convertSharpToPng(img, 2048, 2048); } else { logger.debug(`web image not created (not an required image)`); @@ -220,7 +228,8 @@ export async function generateAlts(path: string, type: string, generateWeb: bool logger.warn(`web image not created (an error occured)`, err as Error); } } else { - logger.info(`web image not created (from remote)`); + if (satisfyWebpublic) logger.info(`web image not created (original satisfies webpublic)`); + else logger.info(`web image not created (from remote)`); } // #endregion webpublic @@ -228,10 +237,8 @@ export async function generateAlts(path: string, type: string, generateWeb: bool let thumbnail: IImage | null = null; try { - if (['image/jpeg', 'image/webp'].includes(type)) { - thumbnail = await convertSharpToJpeg(img, 498, 280); - } else if (['image/png', 'image/svg+xml'].includes(type)) { - thumbnail = await convertSharpToPngOrJpeg(img, 498, 280); + if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(type)) { + thumbnail = await convertSharpToWebp(img, 498, 280); } else { logger.debug(`thumbnail not created (not an required file)`); } diff --git a/packages/backend/src/services/drive/generate-video-thumbnail.ts b/packages/backend/src/services/drive/generate-video-thumbnail.ts index 04a7a83346..da93bc97c7 100644 --- a/packages/backend/src/services/drive/generate-video-thumbnail.ts +++ b/packages/backend/src/services/drive/generate-video-thumbnail.ts @@ -27,6 +27,7 @@ export async function GenerateVideoThumbnail(path: string): Promise { const outPath = `${outDir}/output.png`; + // JPEGに変換 (Webpでもいいが、MastodonはWebpをサポートせず表示できなくなる) const thumbnail = await convertToJpeg(outPath, 498, 280); // cleanup diff --git a/packages/backend/src/services/drive/image-processor.ts b/packages/backend/src/services/drive/image-processor.ts index 146dcfb6ca..2c564ea595 100644 --- a/packages/backend/src/services/drive/image-processor.ts +++ b/packages/backend/src/services/drive/image-processor.ts @@ -38,11 +38,11 @@ export async function convertSharpToJpeg(sharp: sharp.Sharp, width: number, heig * Convert to WebP * with resize, remove metadata, resolve orientation, stop animation */ -export async function convertToWebp(path: string, width: number, height: number): Promise { - return convertSharpToWebp(await sharp(path), width, height); +export async function convertToWebp(path: string, width: number, height: number, quality: number = 85): Promise { + return convertSharpToWebp(await sharp(path), width, height, quality); } -export async function convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number): Promise { +export async function convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, quality: number = 85): Promise { const data = await sharp .resize(width, height, { fit: 'inside', @@ -50,7 +50,7 @@ export async function convertSharpToWebp(sharp: sharp.Sharp, width: number, heig }) .rotate() .webp({ - quality: 85, + quality, }) .toBuffer(); @@ -85,23 +85,3 @@ export async function convertSharpToPng(sharp: sharp.Sharp, width: number, heigh type: 'image/png', }; } - -/** - * Convert to PNG or JPEG - * with resize, remove metadata, resolve orientation, stop animation - */ -export async function convertToPngOrJpeg(path: string, width: number, height: number): Promise { - return convertSharpToPngOrJpeg(await sharp(path), width, height); -} - -export async function convertSharpToPngOrJpeg(sharp: sharp.Sharp, width: number, height: number): Promise { - const stats = await sharp.stats(); - const metadata = await sharp.metadata(); - - // 不透明で300x300pxの範囲を超えていればJPEG - if (stats.isOpaque && ((metadata.width && metadata.width >= 300) || (metadata.height && metadata!.height >= 300))) { - return await convertSharpToJpeg(sharp, width, height); - } else { - return await convertSharpToPng(sharp, width, height); - } -} diff --git a/packages/client/package.json b/packages/client/package.json index 21093cdb7c..e533e1fb87 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -21,6 +21,7 @@ "autwh": "0.1.0", "blurhash": "1.1.5", "broadcast-channel": "4.11.0", + "browser-image-resizer": "git+https://github.com/misskey-dev/browser-image-resizer#v2.2.1-misskey.2", "chart.js": "3.7.1", "chartjs-adapter-date-fns": "2.0.0", "chartjs-plugin-gradient": "0.2.2", diff --git a/packages/client/src/components/drive.vue b/packages/client/src/components/drive.vue index e044c67523..2ec885b00c 100644 --- a/packages/client/src/components/drive.vue +++ b/packages/client/src/components/drive.vue @@ -97,6 +97,7 @@ import * as os from '@/os'; import { stream } from '@/stream'; import { defaultStore } from '@/store'; import { i18n } from '@/i18n'; +import { uploadFile, uploads } from '@/scripts/upload'; const props = withDefaults(defineProps<{ initialFolder?: Misskey.entities.DriveFolder; @@ -127,8 +128,9 @@ const moreFolders = ref(false); const hierarchyFolders = ref([]); const selectedFiles = ref([]); const selectedFolders = ref([]); -const uploadings = os.uploads; +const uploadings = uploads; const connection = stream.useChannel('drive'); +const keepOriginal = ref(defaultStore.state.keepOriginalUploading); // 外部渡しが多いので$refは使わないほうがよい // ドロップされようとしているか const draghover = ref(false); @@ -355,7 +357,7 @@ function onChangeFileInput() { } function upload(file: File, folderToUpload?: Misskey.entities.DriveFolder | null) { - os.upload(file, (folderToUpload && typeof folderToUpload == 'object') ? folderToUpload.id : null).then(res => { + uploadFile(file, (folderToUpload && typeof folderToUpload == 'object') ? folderToUpload.id : null, undefined, keepOriginal.value).then(res => { addFile(res, true); }); } @@ -562,6 +564,10 @@ function fetchMoreFiles() { function getMenu() { return [{ + type: 'switch', + text: i18n.ts.keepOriginalUploading, + ref: keepOriginal, + }, null, { text: i18n.ts.addFile, type: 'label' }, { diff --git a/packages/client/src/components/post-form.vue b/packages/client/src/components/post-form.vue index 656689ddcb..241c726c11 100644 --- a/packages/client/src/components/post-form.vue +++ b/packages/client/src/components/post-form.vue @@ -87,6 +87,7 @@ import MkInfo from '@/components/ui/info.vue'; import { i18n } from '@/i18n'; import { instance } from '@/instance'; import { $i, getAccounts, openAccountMenu as openAccountMenu_ } from '@/account'; +import { uploadFile } from '@/scripts/upload'; const modal = inject('modal'); @@ -372,7 +373,7 @@ function updateFileName(file, name) { } function upload(file: File, name?: string) { - os.upload(file, defaultStore.state.uploadFolder, name).then(res => { + uploadFile(file, defaultStore.state.uploadFolder, name).then(res => { files.push(res); }); } diff --git a/packages/client/src/os.ts b/packages/client/src/os.ts index 43c110555f..b8a3f94cc8 100644 --- a/packages/client/src/os.ts +++ b/packages/client/src/os.ts @@ -1,6 +1,6 @@ // TODO: なんでもかんでもos.tsに突っ込むのやめたいのでよしなに分割する -import { Component, defineAsyncComponent, markRaw, reactive, Ref, ref } from 'vue'; +import { Component, markRaw, Ref, ref } from 'vue'; import { EventEmitter } from 'eventemitter3'; import insertTextAtCursor from 'insert-text-at-cursor'; import * as Misskey from 'misskey-js'; @@ -10,7 +10,6 @@ import MkWaitingDialog from '@/components/waiting-dialog.vue'; import { MenuItem } from '@/types/menu'; import { resolve } from '@/router'; import { $i } from '@/account'; -import { defaultStore } from '@/store'; export const pendingApiRequestsCount = ref(0); @@ -537,78 +536,6 @@ export function post(props: Record = {}) { export const deckGlobalEvents = new EventEmitter(); -export const uploads = ref<{ - id: string; - name: string; - progressMax: number | undefined; - progressValue: number | undefined; - img: string; -}[]>([]); - -export function upload(file: File, folder?: any, name?: string, keepOriginal: boolean = defaultStore.state.keepOriginalUploading): Promise { - if (folder && typeof folder === 'object') folder = folder.id; - - return new Promise((resolve, reject) => { - const id = Math.random().toString(); - - const reader = new FileReader(); - reader.onload = (e) => { - const ctx = reactive({ - id: id, - name: name || file.name || 'untitled', - progressMax: undefined, - progressValue: undefined, - img: window.URL.createObjectURL(file) - }); - - uploads.value.push(ctx); - - console.log(keepOriginal); - - const data = new FormData(); - data.append('i', $i.token); - data.append('force', 'true'); - data.append('file', file); - - if (folder) data.append('folderId', folder); - if (name) data.append('name', name); - - const xhr = new XMLHttpRequest(); - xhr.open('POST', apiUrl + '/drive/files/create', true); - xhr.onload = (ev) => { - if (xhr.status !== 200 || ev.target == null || ev.target.response == null) { - // TODO: 消すのではなくて再送できるようにしたい - uploads.value = uploads.value.filter(x => x.id != id); - - alert({ - type: 'error', - text: 'upload failed' - }); - - reject(); - return; - } - - const driveFile = JSON.parse(ev.target.response); - - resolve(driveFile); - - uploads.value = uploads.value.filter(x => x.id != id); - }; - - xhr.upload.onprogress = e => { - if (e.lengthComputable) { - ctx.progressMax = e.total; - ctx.progressValue = e.loaded; - } - }; - - xhr.send(data); - }; - reader.readAsArrayBuffer(file); - }); -} - /* export function checkExistence(fileData: ArrayBuffer): Promise { return new Promise((resolve, reject) => { diff --git a/packages/client/src/pages/messaging/messaging-room.form.vue b/packages/client/src/pages/messaging/messaging-room.form.vue index 3863c8f82b..35cb75743f 100644 --- a/packages/client/src/pages/messaging/messaging-room.form.vue +++ b/packages/client/src/pages/messaging/messaging-room.form.vue @@ -31,6 +31,7 @@ import * as os from '@/os'; import { stream } from '@/stream'; import { Autocomplete } from '@/scripts/autocomplete'; import { throttle } from 'throttle-debounce'; +import { uploadFile } from '@/scripts/upload'; export default defineComponent({ props: { @@ -164,7 +165,7 @@ export default defineComponent({ }, upload(file: File, name?: string) { - os.upload(file, this.$store.state.uploadFolder, name).then(res => { + uploadFile(file, this.$store.state.uploadFolder, name).then(res => { this.file = res; }); }, diff --git a/packages/client/src/scripts/select-file.ts b/packages/client/src/scripts/select-file.ts index 23df4edf54..49a46f0bb2 100644 --- a/packages/client/src/scripts/select-file.ts +++ b/packages/client/src/scripts/select-file.ts @@ -4,6 +4,7 @@ import { stream } from '@/stream'; import { i18n } from '@/i18n'; import { defaultStore } from '@/store'; import { DriveFile } from 'misskey-js/built/entities'; +import { uploadFile } from '@/scripts/upload'; function select(src: any, label: string | null, multiple: boolean): Promise { return new Promise((res, rej) => { @@ -14,7 +15,7 @@ function select(src: any, label: string | null, multiple: boolean): Promise { - const promises = Array.from(input.files).map(file => os.upload(file, defaultStore.state.uploadFolder, undefined, keepOriginal.value)); + const promises = Array.from(input.files).map(file => uploadFile(file, defaultStore.state.uploadFolder, undefined, keepOriginal.value)); Promise.all(promises).then(driveFiles => { res(multiple ? driveFiles : driveFiles[0]); diff --git a/packages/client/src/scripts/upload.ts b/packages/client/src/scripts/upload.ts new file mode 100644 index 0000000000..7e4f793b44 --- /dev/null +++ b/packages/client/src/scripts/upload.ts @@ -0,0 +1,114 @@ +import { reactive, ref } from 'vue'; +import { defaultStore } from '@/store'; +import { apiUrl } from '@/config'; +import * as Misskey from 'misskey-js'; +import { $i } from '@/account'; +import { readAndCompressImage } from 'browser-image-resizer'; +import { alert } from '@/os'; + +type Uploading = { + id: string; + name: string; + progressMax: number | undefined; + progressValue: number | undefined; + img: string; +}; +export const uploads = ref([]); + +const compressTypeMap = { + 'image/jpeg': { quality: 0.85, mimeType: 'image/jpeg' }, + 'image/webp': { quality: 0.85, mimeType: 'image/jpeg' }, + 'image/svg+xml': { quality: 1, mimeType: 'image/png' }, +} as const; + +const mimeTypeMap = { + 'image/webp': 'webp', + 'image/jpeg': 'jpg', + 'image/png': 'png', +} as const; + +export function uploadFile( + file: File, + folder?: any, + name?: string, + keepOriginal: boolean = defaultStore.state.keepOriginalUploading +): Promise { + if (folder && typeof folder == 'object') folder = folder.id; + + return new Promise((resolve, reject) => { + const id = Math.random().toString(); + + const reader = new FileReader(); + reader.onload = async (e) => { + const ctx = reactive({ + id: id, + name: name || file.name || 'untitled', + progressMax: undefined, + progressValue: undefined, + img: window.URL.createObjectURL(file) + }); + + uploads.value.push(ctx); + + let resizedImage: any; + if (!keepOriginal && file.type in compressTypeMap) { + const imgConfig = compressTypeMap[file.type]; + + const config = { + maxWidth: 2048, + maxHeight: 2048, + debug: true, + ...imgConfig, + }; + + try { + resizedImage = await readAndCompressImage(file, config); + ctx.name = file.type !== imgConfig.mimeType ? `${ctx.name}.${mimeTypeMap[compressTypeMap[file.type].mimeType]}` : ctx.name; + } catch (e) { + console.error('Failed to resize image', e); + } + } + + const data = new FormData(); + data.append('i', $i.token); + data.append('force', 'true'); + data.append('file', resizedImage || file); + data.append('name', ctx.name); + if (folder) data.append('folderId', folder); + + const xhr = new XMLHttpRequest(); + xhr.open('POST', apiUrl + '/drive/files/create', true); + xhr.onload = (ev) => { + if (xhr.status !== 200 || ev.target == null || ev.target.response == null) { + // TODO: 消すのではなくて再送できるようにしたい + uploads.value = uploads.value.filter(x => x.id != id); + + alert({ + type: 'error', + title: 'Failed to upload', + text: `${JSON.stringify(ev.target?.response)}, ${JSON.stringify(xhr.response)}` + }); + + reject(); + return; + } + + const driveFile = JSON.parse(ev.target.response); + + resolve(driveFile); + + uploads.value = uploads.value.filter(x => x.id != id); + }; + + xhr.upload.onprogress = e => { + if (e.lengthComputable) { + ctx.progressMax = e.total; + ctx.progressValue = e.loaded; + } + }; + + xhr.send(data); + }; + reader.readAsArrayBuffer(file); + }); +} diff --git a/packages/client/src/ui/_common_/common.vue b/packages/client/src/ui/_common_/common.vue index 05688d7c53..50d95539d1 100644 --- a/packages/client/src/ui/_common_/common.vue +++ b/packages/client/src/ui/_common_/common.vue @@ -17,7 +17,8 @@ diff --git a/packages/client/yarn.lock b/packages/client/yarn.lock index 05b586eb17..59abe67862 100644 --- a/packages/client/yarn.lock +++ b/packages/client/yarn.lock @@ -1299,6 +1299,10 @@ broadcast-channel@4.11.0: rimraf "3.0.2" unload "2.3.1" +"browser-image-resizer@git+https://github.com/misskey-dev/browser-image-resizer#v2.2.1-misskey.2": + version "2.2.1-misskey.2" + resolved "git+https://github.com/misskey-dev/browser-image-resizer#a58834f5fe2af9f9f31ff115121aef3de6f9d416" + browser-stdout@1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" -- cgit v1.2.3-freya From 89c5fd0931e981e7d6aa0af5dd19f9492e345f78 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Sat, 14 May 2022 06:28:27 +0200 Subject: perf: fix caching (#8660) The cache implementation did previously not store the results of the computation and was thus not a cache at all. This can cause a significant number of database queries each time someone with a large number of followers does something that causes an activity to be federated. --- packages/backend/src/misc/cache.ts | 1 + 1 file changed, 1 insertion(+) (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 01bbe98a85..e5b911ed32 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -48,6 +48,7 @@ export class Cache { // Cache MISS const value = await fetcher(); + this.set(key, value); return value; } -- cgit v1.2.3-freya From 22bb1a17930bdbcf9a2068790efd880fe8ff4158 Mon Sep 17 00:00:00 2001 From: iwata Date: Sat, 14 May 2022 15:16:45 +0900 Subject: test: e2eテストがCIで失敗していた問題をいくつか修正 (#8642) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: indexeddbをテスト毎に初期化するように * fix: metaが無いときにfetch-metaを同時に呼ぶと死ぬことがある問題を修正 * test: ログイン後のクライアント側処理を待たずにリロードされてログイン出来ないことがあったのを修正 --- cypress/integration/basic.js | 16 +++++++++++++--- packages/backend/src/misc/fetch-meta.ts | 13 ++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) (limited to 'packages/backend/src/misc') diff --git a/cypress/integration/basic.js b/cypress/integration/basic.js index 7d27b649f4..eb15cfe223 100644 --- a/cypress/integration/basic.js +++ b/cypress/integration/basic.js @@ -1,5 +1,8 @@ describe('Before setup instance', () => { beforeEach(() => { + cy.window(win => { + win.indexedDB.deleteDatabase('keyval-store'); + }); cy.request('POST', '/api/reset-db').as('reset'); cy.get('@reset').its('status').should('equal', 204); cy.reload(true); @@ -32,6 +35,9 @@ describe('Before setup instance', () => { describe('After setup instance', () => { beforeEach(() => { + cy.window(win => { + win.indexedDB.deleteDatabase('keyval-store'); + }); cy.request('POST', '/api/reset-db').as('reset'); cy.get('@reset').its('status').should('equal', 204); cy.reload(true); @@ -70,6 +76,9 @@ describe('After setup instance', () => { describe('After user signup', () => { beforeEach(() => { + cy.window(win => { + win.indexedDB.deleteDatabase('keyval-store'); + }); cy.request('POST', '/api/reset-db').as('reset'); cy.get('@reset').its('status').should('equal', 204); cy.reload(true); @@ -129,6 +138,9 @@ describe('After user signup', () => { describe('After user singed in', () => { beforeEach(() => { + cy.window(win => { + win.indexedDB.deleteDatabase('keyval-store'); + }); cy.request('POST', '/api/reset-db').as('reset'); cy.get('@reset').its('status').should('equal', 204); cy.reload(true); @@ -163,12 +175,10 @@ describe('After user singed in', () => { }); it('successfully loads', () => { - cy.visit('/'); + cy.get('[data-cy-open-post-form]').should('be.visible'); }); it('note', () => { - cy.visit('/'); - cy.get('[data-cy-open-post-form]').click(); cy.get('[data-cy-post-form-text]').type('Hello, Misskey!'); cy.get('[data-cy-open-post-form-submit]').click(); diff --git a/packages/backend/src/misc/fetch-meta.ts b/packages/backend/src/misc/fetch-meta.ts index 5417c10962..e855ac28ee 100644 --- a/packages/backend/src/misc/fetch-meta.ts +++ b/packages/backend/src/misc/fetch-meta.ts @@ -20,9 +20,16 @@ export async function fetchMeta(noCache = false): Promise { cache = meta; return meta; } else { - const saved = await transactionalEntityManager.save(Meta, { - id: 'x', - }) as Meta; + // metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う + const saved = await transactionalEntityManager + .upsert( + Meta, + { + id: 'x', + }, + ['id'], + ) + .then((x) => transactionalEntityManager.findOneByOrFail(Meta, x.identifiers[0])); cache = saved; return saved; -- cgit v1.2.3-freya From e27c6abaeaf0e0e0be9fba7ffc6fd165474d8592 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Wed, 25 May 2022 09:50:22 +0200 Subject: refactor: temporary files (#8713) * simplify temporary files for thumbnails Because only a single file will be written to the directory, creating a separate directory seems unnecessary. If only a temporary file is created, the code from `createTemp` can be reused here as well. * refactor: deduplicate code for temporary files/directories To follow the DRY principle, the same code should not be duplicated across different files. Instead an already existing function is used. Because temporary directories are also create in multiple locations, a function for this is also newly added to reduce duplication. * fix: clean up identicon temp files The temporary files for identicons are not reused and can be deleted after they are fully read. This condition is met when the stream is closed and so the file can be cleaned up using the events API of the stream. * fix: ensure cleanup is called when download fails * fix: ensure cleanup is called in error conditions This covers import/export queue jobs and is mostly just wrapping all code in a try...finally statement where the finally runs the cleanup. * fix: use correct type instead of `any` --- packages/backend/src/misc/create-temp.ts | 13 ++- .../src/queue/processors/db/export-blocking.ts | 111 +++++++++---------- .../queue/processors/db/export-custom-emojis.ts | 17 +-- .../src/queue/processors/db/export-following.ts | 117 ++++++++++---------- .../backend/src/queue/processors/db/export-mute.ts | 113 ++++++++++--------- .../src/queue/processors/db/export-notes.ts | 123 ++++++++++----------- .../src/queue/processors/db/export-user-lists.ts | 63 ++++++----- .../queue/processors/db/import-custom-emojis.ts | 10 +- .../backend/src/server/file/send-drive-file.ts | 9 +- packages/backend/src/server/index.ts | 4 +- .../src/services/drive/generate-video-thumbnail.ts | 53 ++++----- .../backend/src/services/drive/upload-from-url.ts | 25 ++--- 12 files changed, 312 insertions(+), 346 deletions(-) (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/misc/create-temp.ts b/packages/backend/src/misc/create-temp.ts index 04604cf7d0..f07be634fb 100644 --- a/packages/backend/src/misc/create-temp.ts +++ b/packages/backend/src/misc/create-temp.ts @@ -1,10 +1,19 @@ import * as tmp from 'tmp'; -export function createTemp(): Promise<[string, any]> { - return new Promise<[string, any]>((res, rej) => { +export function createTemp(): Promise<[string, () => void]> { + return new Promise<[string, () => void]>((res, rej) => { tmp.file((e, path, fd, cleanup) => { if (e) return rej(e); res([path, cleanup]); }); }); } + +export function createTempDir(): Promise<[string, () => void]> { + return new Promise<[string, () => void]>((res, rej) => { + tmp.dir((e, path, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); +} diff --git a/packages/backend/src/queue/processors/db/export-blocking.ts b/packages/backend/src/queue/processors/db/export-blocking.ts index 166c9e4cd3..f5e0424a79 100644 --- a/packages/backend/src/queue/processors/db/export-blocking.ts +++ b/packages/backend/src/queue/processors/db/export-blocking.ts @@ -1,11 +1,11 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { queueLogger } from '../../logger.js'; import { addFile } from '@/services/drive/add-file.js'; import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host.js'; +import { createTemp } from '@/misc/create-temp.js'; import { Users, Blockings } from '@/models/index.js'; import { MoreThan } from 'typeorm'; import { DbUserJobData } from '@/queue/types.js'; @@ -22,73 +22,72 @@ export async function exportBlocking(job: Bull.Job, done: any): P } // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - const stream = fs.createWriteStream(path, { flags: 'a' }); - - let exportedCount = 0; - let cursor: any = null; - - while (true) { - const blockings = await Blockings.find({ - where: { - blockerId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }); - - if (blockings.length === 0) { - job.progress(100); - break; - } - - cursor = blockings[blockings.length - 1].id; + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); + + let exportedCount = 0; + let cursor: any = null; + + while (true) { + const blockings = await Blockings.find({ + where: { + blockerId: user.id, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }); - for (const block of blockings) { - const u = await Users.findOneBy({ id: block.blockeeId }); - if (u == null) { - exportedCount++; continue; + if (blockings.length === 0) { + job.progress(100); + break; } - const content = getFullApAccount(u.username, u.host); - await new Promise((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } + cursor = blockings[blockings.length - 1].id; + + for (const block of blockings) { + const u = await Users.findOneBy({ id: block.blockeeId }); + if (u == null) { + exportedCount++; continue; + } + + const content = getFullApAccount(u.username, u.host); + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); }); + exportedCount++; + } + + const total = await Blockings.countBy({ + blockerId: user.id, }); - exportedCount++; - } - const total = await Blockings.countBy({ - blockerId: user.id, - }); + job.progress(exportedCount / total); + } - job.progress(exportedCount / total); - } + stream.end(); + logger.succ(`Exported to: ${path}`); - stream.end(); - logger.succ(`Exported to: ${path}`); + const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const driveFile = await addFile({ user, path, name: fileName, force: true }); - const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); + logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); done(); } diff --git a/packages/backend/src/queue/processors/db/export-custom-emojis.ts b/packages/backend/src/queue/processors/db/export-custom-emojis.ts index c2467fb5f0..97ba62dcf6 100644 --- a/packages/backend/src/queue/processors/db/export-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/export-custom-emojis.ts @@ -1,5 +1,4 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { ulid } from 'ulid'; @@ -10,6 +9,7 @@ import { addFile } from '@/services/drive/add-file.js'; import { format as dateFormat } from 'date-fns'; import { Users, Emojis } from '@/models/index.js'; import { } from '@/queue/types.js'; +import { createTempDir } from '@/misc/create-temp.js'; import { downloadUrl } from '@/misc/download-url.js'; import config from '@/config/index.js'; import { IsNull } from 'typeorm'; @@ -25,13 +25,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi return; } - // Create temp dir - const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => { - tmp.dir((e, path, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTempDir(); logger.info(`Temp dir is ${path}`); @@ -98,12 +92,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi metaStream.end(); // Create archive - const [archivePath, archiveCleanup] = await new Promise<[string, () => void]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [archivePath, archiveCleanup] = await createTemp(); const archiveStream = fs.createWriteStream(archivePath); const archive = archiver('zip', { zlib: { level: 0 }, diff --git a/packages/backend/src/queue/processors/db/export-following.ts b/packages/backend/src/queue/processors/db/export-following.ts index 965500ac27..4ac165567b 100644 --- a/packages/backend/src/queue/processors/db/export-following.ts +++ b/packages/backend/src/queue/processors/db/export-following.ts @@ -1,11 +1,11 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { queueLogger } from '../../logger.js'; import { addFile } from '@/services/drive/add-file.js'; import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host.js'; +import { createTemp } from '@/misc/create-temp.js'; import { Users, Followings, Mutings } from '@/models/index.js'; import { In, MoreThan, Not } from 'typeorm'; import { DbUserJobData } from '@/queue/types.js'; @@ -23,73 +23,72 @@ export async function exportFollowing(job: Bull.Job, done: () => } // Create temp file - const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - const stream = fs.createWriteStream(path, { flags: 'a' }); - - let cursor: Following['id'] | null = null; - - const mutings = job.data.excludeMuting ? await Mutings.findBy({ - muterId: user.id, - }) : []; - - while (true) { - const followings = await Followings.find({ - where: { - followerId: user.id, - ...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}), - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as Following[]; - - if (followings.length === 0) { - break; - } - - cursor = followings[followings.length - 1].id; - - for (const following of followings) { - const u = await Users.findOneBy({ id: following.followeeId }); - if (u == null) { - continue; - } - - if (job.data.excludeInactive && u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) { - continue; + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); + + let cursor: Following['id'] | null = null; + + const mutings = job.data.excludeMuting ? await Mutings.findBy({ + muterId: user.id, + }) : []; + + while (true) { + const followings = await Followings.find({ + where: { + followerId: user.id, + ...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}), + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }) as Following[]; + + if (followings.length === 0) { + break; } - const content = getFullApAccount(u.username, u.host); - await new Promise((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } + cursor = followings[followings.length - 1].id; + + for (const following of followings) { + const u = await Users.findOneBy({ id: following.followeeId }); + if (u == null) { + continue; + } + + if (job.data.excludeInactive && u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) { + continue; + } + + const content = getFullApAccount(u.username, u.host); + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); }); - }); + } } - } - stream.end(); - logger.succ(`Exported to: ${path}`); + stream.end(); + logger.succ(`Exported to: ${path}`); - const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); + const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const driveFile = await addFile({ user, path, name: fileName, force: true }); + + logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); done(); } diff --git a/packages/backend/src/queue/processors/db/export-mute.ts b/packages/backend/src/queue/processors/db/export-mute.ts index 0ef81971f1..6a36cfa072 100644 --- a/packages/backend/src/queue/processors/db/export-mute.ts +++ b/packages/backend/src/queue/processors/db/export-mute.ts @@ -1,11 +1,11 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { queueLogger } from '../../logger.js'; import { addFile } from '@/services/drive/add-file.js'; import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host.js'; +import { createTemp } from '@/misc/create-temp.js'; import { Users, Mutings } from '@/models/index.js'; import { IsNull, MoreThan } from 'typeorm'; import { DbUserJobData } from '@/queue/types.js'; @@ -22,74 +22,73 @@ export async function exportMute(job: Bull.Job, done: any): Promi } // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - const stream = fs.createWriteStream(path, { flags: 'a' }); - - let exportedCount = 0; - let cursor: any = null; - - while (true) { - const mutes = await Mutings.find({ - where: { - muterId: user.id, - expiresAt: IsNull(), - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }); - - if (mutes.length === 0) { - job.progress(100); - break; - } - - cursor = mutes[mutes.length - 1].id; + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); + + let exportedCount = 0; + let cursor: any = null; + + while (true) { + const mutes = await Mutings.find({ + where: { + muterId: user.id, + expiresAt: IsNull(), + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }); - for (const mute of mutes) { - const u = await Users.findOneBy({ id: mute.muteeId }); - if (u == null) { - exportedCount++; continue; + if (mutes.length === 0) { + job.progress(100); + break; } - const content = getFullApAccount(u.username, u.host); - await new Promise((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } + cursor = mutes[mutes.length - 1].id; + + for (const mute of mutes) { + const u = await Users.findOneBy({ id: mute.muteeId }); + if (u == null) { + exportedCount++; continue; + } + + const content = getFullApAccount(u.username, u.host); + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); }); + exportedCount++; + } + + const total = await Mutings.countBy({ + muterId: user.id, }); - exportedCount++; - } - const total = await Mutings.countBy({ - muterId: user.id, - }); + job.progress(exportedCount / total); + } - job.progress(exportedCount / total); - } + stream.end(); + logger.succ(`Exported to: ${path}`); - stream.end(); - logger.succ(`Exported to: ${path}`); + const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const driveFile = await addFile({ user, path, name: fileName, force: true }); - const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); + logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); done(); } diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts index 7e12a6fac2..051fcdf385 100644 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ b/packages/backend/src/queue/processors/db/export-notes.ts @@ -1,5 +1,4 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { queueLogger } from '../../logger.js'; @@ -10,6 +9,7 @@ import { MoreThan } from 'typeorm'; import { Note } from '@/models/entities/note.js'; import { Poll } from '@/models/entities/poll.js'; import { DbUserJobData } from '@/queue/types.js'; +import { createTemp } from '@/misc/create-temp.js'; const logger = queueLogger.createSubLogger('export-notes'); @@ -23,82 +23,81 @@ export async function exportNotes(job: Bull.Job, done: any): Prom } // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - const stream = fs.createWriteStream(path, { flags: 'a' }); - - const write = (text: string): Promise => { - return new Promise((res, rej) => { - stream.write(text, err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); + + const write = (text: string): Promise => { + return new Promise((res, rej) => { + stream.write(text, err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); }); - }); - }; + }; + + await write('['); + + let exportedNotesCount = 0; + let cursor: Note['id'] | null = null; + + while (true) { + const notes = await Notes.find({ + where: { + userId: user.id, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }) as Note[]; + + if (notes.length === 0) { + job.progress(100); + break; + } - await write('['); + cursor = notes[notes.length - 1].id; - let exportedNotesCount = 0; - let cursor: Note['id'] | null = null; + for (const note of notes) { + let poll: Poll | undefined; + if (note.hasPoll) { + poll = await Polls.findOneByOrFail({ noteId: note.id }); + } + const content = JSON.stringify(serialize(note, poll)); + const isFirst = exportedNotesCount === 0; + await write(isFirst ? content : ',\n' + content); + exportedNotesCount++; + } - while (true) { - const notes = await Notes.find({ - where: { + const total = await Notes.countBy({ userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as Note[]; - - if (notes.length === 0) { - job.progress(100); - break; - } - - cursor = notes[notes.length - 1].id; + }); - for (const note of notes) { - let poll: Poll | undefined; - if (note.hasPoll) { - poll = await Polls.findOneByOrFail({ noteId: note.id }); - } - const content = JSON.stringify(serialize(note, poll)); - const isFirst = exportedNotesCount === 0; - await write(isFirst ? content : ',\n' + content); - exportedNotesCount++; + job.progress(exportedNotesCount / total); } - const total = await Notes.countBy({ - userId: user.id, - }); + await write(']'); - job.progress(exportedNotesCount / total); - } - - await write(']'); + stream.end(); + logger.succ(`Exported to: ${path}`); - stream.end(); - logger.succ(`Exported to: ${path}`); + const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; + const driveFile = await addFile({ user, path, name: fileName, force: true }); - const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); + logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); done(); } diff --git a/packages/backend/src/queue/processors/db/export-user-lists.ts b/packages/backend/src/queue/processors/db/export-user-lists.ts index 45852a6038..71dd72df27 100644 --- a/packages/backend/src/queue/processors/db/export-user-lists.ts +++ b/packages/backend/src/queue/processors/db/export-user-lists.ts @@ -1,11 +1,11 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { queueLogger } from '../../logger.js'; import { addFile } from '@/services/drive/add-file.js'; import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host.js'; +import { createTemp } from '@/misc/create-temp.js'; import { Users, UserLists, UserListJoinings } from '@/models/index.js'; import { In } from 'typeorm'; import { DbUserJobData } from '@/queue/types.js'; @@ -26,46 +26,45 @@ export async function exportUserLists(job: Bull.Job, done: any): }); // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - const stream = fs.createWriteStream(path, { flags: 'a' }); + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); - for (const list of lists) { - const joinings = await UserListJoinings.findBy({ userListId: list.id }); - const users = await Users.findBy({ - id: In(joinings.map(j => j.userId)), - }); + for (const list of lists) { + const joinings = await UserListJoinings.findBy({ userListId: list.id }); + const users = await Users.findBy({ + id: In(joinings.map(j => j.userId)), + }); - for (const u of users) { - const acct = getFullApAccount(u.username, u.host); - const content = `${list.name},${acct}`; - await new Promise((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } + for (const u of users) { + const acct = getFullApAccount(u.username, u.host); + const content = `${list.name},${acct}`; + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); }); - }); + } } - } - stream.end(); - logger.succ(`Exported to: ${path}`); + stream.end(); + logger.succ(`Exported to: ${path}`); + + const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const driveFile = await addFile({ user, path, name: fileName, force: true }); - const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); + logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); done(); } diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index 28e0b867a4..64dfe85374 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -1,9 +1,9 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import unzipper from 'unzipper'; import { queueLogger } from '../../logger.js'; +import { createTempDir } from '@/misc/create-temp.js'; import { downloadUrl } from '@/misc/download-url.js'; import { DriveFiles, Emojis } from '@/models/index.js'; import { DbUserImportJobData } from '@/queue/types.js'; @@ -25,13 +25,7 @@ export async function importCustomEmojis(job: Bull.Job, don return; } - // Create temp dir - const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => { - tmp.dir((e, path, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTempDir(); logger.info(`Temp dir is ${path}`); diff --git a/packages/backend/src/server/file/send-drive-file.ts b/packages/backend/src/server/file/send-drive-file.ts index 027d078ce1..c34e043145 100644 --- a/packages/backend/src/server/file/send-drive-file.ts +++ b/packages/backend/src/server/file/send-drive-file.ts @@ -4,11 +4,11 @@ import { dirname } from 'node:path'; import Koa from 'koa'; import send from 'koa-send'; import rename from 'rename'; -import * as tmp from 'tmp'; import { serverLogger } from '../index.js'; import { contentDisposition } from '@/misc/content-disposition.js'; import { DriveFiles } from '@/models/index.js'; import { InternalStorage } from '@/services/drive/internal-storage.js'; +import { createTemp } from '@/misc/create-temp.js'; import { downloadUrl } from '@/misc/download-url.js'; import { detectType } from '@/misc/get-file-info.js'; import { convertToWebp, convertToJpeg, convertToPng } from '@/services/drive/image-processor.js'; @@ -50,12 +50,7 @@ export default async function(ctx: Koa.Context) { if (!file.storedInternal) { if (file.isLink && file.uri) { // 期限切れリモートファイル - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); try { await downloadUrl(file.uri, path); diff --git a/packages/backend/src/server/index.ts b/packages/backend/src/server/index.ts index cd061da725..f31de2b7f4 100644 --- a/packages/backend/src/server/index.ts +++ b/packages/backend/src/server/index.ts @@ -89,10 +89,10 @@ router.get('/avatar/@:acct', async ctx => { }); router.get('/identicon/:x', async ctx => { - const [temp] = await createTemp(); + const [temp, cleanup] = await createTemp(); await genIdenticon(ctx.params.x, fs.createWriteStream(temp)); ctx.set('Content-Type', 'image/png'); - ctx.body = fs.createReadStream(temp); + ctx.body = fs.createReadStream(temp).on('close', () => cleanup()); }); router.get('/verify-email/:code', async ctx => { diff --git a/packages/backend/src/services/drive/generate-video-thumbnail.ts b/packages/backend/src/services/drive/generate-video-thumbnail.ts index ef75a9f585..ca12ab8d3d 100644 --- a/packages/backend/src/services/drive/generate-video-thumbnail.ts +++ b/packages/backend/src/services/drive/generate-video-thumbnail.ts @@ -1,38 +1,31 @@ import * as fs from 'node:fs'; -import * as tmp from 'tmp'; +import * as path from 'node:path'; +import { createTemp } from '@/misc/create-temp.js'; import { IImage, convertToJpeg } from './image-processor.js'; import FFmpeg from 'fluent-ffmpeg'; -export async function GenerateVideoThumbnail(path: string): Promise { - const [outDir, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.dir((e, path, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); +export async function GenerateVideoThumbnail(source: string): Promise { + const [file, cleanup] = await createTemp(); + const parsed = path.parse(file); - await new Promise((res, rej) => { - FFmpeg({ - source: path, - }) - .on('end', res) - .on('error', rej) - .screenshot({ - folder: outDir, - filename: 'output.png', - count: 1, - timestamps: ['5%'], + try { + await new Promise((res, rej) => { + FFmpeg({ + source, + }) + .on('end', res) + .on('error', rej) + .screenshot({ + folder: parsed.dir, + filename: parsed.base, + count: 1, + timestamps: ['5%'], + }); }); - }); - - const outPath = `${outDir}/output.png`; - - // JPEGに変換 (Webpでもいいが、MastodonはWebpをサポートせず表示できなくなる) - const thumbnail = await convertToJpeg(outPath, 498, 280); - - // cleanup - await fs.promises.unlink(outPath); - cleanup(); - return thumbnail; + // JPEGに変換 (Webpでもいいが、MastodonはWebpをサポートせず表示できなくなる) + return await convertToJpeg(498, 280); + } finally { + cleanup(); + } } diff --git a/packages/backend/src/services/drive/upload-from-url.ts b/packages/backend/src/services/drive/upload-from-url.ts index 79b1b8c2e1..001fc49ee4 100644 --- a/packages/backend/src/services/drive/upload-from-url.ts +++ b/packages/backend/src/services/drive/upload-from-url.ts @@ -45,29 +45,20 @@ export async function uploadFromUrl({ // Create temp file const [path, cleanup] = await createTemp(); - // write content at URL to temp file - await downloadUrl(url, path); - - let driveFile: DriveFile; - let error; - try { - driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive }); + // write content at URL to temp file + await downloadUrl(url, path); + + const driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive }); logger.succ(`Got: ${driveFile.id}`); + return driveFile!; } catch (e) { - error = e; logger.error(`Failed to create drive file: ${e}`, { url: url, e: e, }); - } - - // clean-up - cleanup(); - - if (error) { - throw error; - } else { - return driveFile!; + throw e; + } finally { + cleanup(); } } -- cgit v1.2.3-freya From c05723ca6ad4f17b823662e83ed8b442fe10626a Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Tue, 31 May 2022 17:44:22 +0900 Subject: Fix IP address rate limit (#8758) * Fix IP address rate limit * CHANGELOG * Tune getIpHash --- CHANGELOG.md | 2 +- packages/backend/src/misc/get-ip-hash.ts | 9 +++++++++ packages/backend/src/server/api/call.ts | 11 +++-------- packages/backend/src/server/api/private/signin.ts | 3 ++- 4 files changed, 15 insertions(+), 10 deletions(-) create mode 100644 packages/backend/src/misc/get-ip-hash.ts (limited to 'packages/backend/src/misc') diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b086ddba9..cfcf52ce92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ You should also include the user name that made the change. Your own theme color may be unset if it was in an invalid format. Admins should check their instance settings if in doubt. - Perform port diagnosis at startup only when Listen fails @mei23 -- Rate limiting is now also usable for non-authenticated users. @Johann150 +- Rate limiting is now also usable for non-authenticated users. @Johann150 @mei23 Admins should make sure the reverse proxy sets the `X-Forwarded-For` header to the original address. ### Bugfixes diff --git a/packages/backend/src/misc/get-ip-hash.ts b/packages/backend/src/misc/get-ip-hash.ts new file mode 100644 index 0000000000..379325bb13 --- /dev/null +++ b/packages/backend/src/misc/get-ip-hash.ts @@ -0,0 +1,9 @@ +import IPCIDR from 'ip-cidr'; + +export function getIpHash(ip: string) { + // because a single person may control many IPv6 addresses, + // only a /64 subnet prefix of any IP will be taken into account. + // (this means for IPv4 the entire address is used) + const prefix = IPCIDR.createAddress(ip).mask(64); + return 'ip-' + BigInt('0b' + prefix).toString(36); +} diff --git a/packages/backend/src/server/api/call.ts b/packages/backend/src/server/api/call.ts index fbe25e1732..cd3e0abc06 100644 --- a/packages/backend/src/server/api/call.ts +++ b/packages/backend/src/server/api/call.ts @@ -6,7 +6,7 @@ import endpoints, { IEndpointMeta } from './endpoints.js'; import { ApiError } from './error.js'; import { apiLogger } from './logger.js'; import { AccessToken } from '@/models/entities/access-token.js'; -import IPCIDR from 'ip-cidr'; +import { getIpHash } from '@/misc/get-ip-hash.js'; const accessDenied = { message: 'Access denied.', @@ -33,18 +33,13 @@ export default async (endpoint: string, user: CacheableLocalUser | null | undefi throw new ApiError(accessDenied); } - if (ep.meta.requireCredential && ep.meta.limit && !isModerator) { + if (ep.meta.limit && !isModerator) { // koa will automatically load the `X-Forwarded-For` header if `proxy: true` is configured in the app. let limitActor: string; if (user) { limitActor = user.id; } else { - // because a single person may control many IPv6 addresses, - // only a /64 subnet prefix of any IP will be taken into account. - // (this means for IPv4 the entire address is used) - const ip = IPCIDR.createAddress(ctx.ip).mask(64); - - limitActor = 'ip-' + parseInt(ip, 2).toString(36); + limitActor = getIpHash(ctx!.ip); } const limit = Object.assign({}, ep.meta.limit); diff --git a/packages/backend/src/server/api/private/signin.ts b/packages/backend/src/server/api/private/signin.ts index b304550e29..79b31764fd 100644 --- a/packages/backend/src/server/api/private/signin.ts +++ b/packages/backend/src/server/api/private/signin.ts @@ -10,6 +10,7 @@ import { verifyLogin, hash } from '../2fa.js'; import { randomBytes } from 'node:crypto'; import { IsNull } from 'typeorm'; import { limiter } from '../limiter.js'; +import { getIpHash } from '@/misc/get-ip-hash.js'; export default async (ctx: Koa.Context) => { ctx.set('Access-Control-Allow-Origin', config.url); @@ -27,7 +28,7 @@ export default async (ctx: Koa.Context) => { try { // not more than 1 attempt per second and not more than 10 attempts per hour - await limiter({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, ctx.ip); + await limiter({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(ctx.ip)); } catch (err) { ctx.status = 429; ctx.body = { -- cgit v1.2.3-freya