From b75184ec8e3436200bacdcd832e3324702553d20 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 18 Sep 2022 03:27:08 +0900 Subject: なんかもうめっちゃ変えた MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/misc/acct.ts | 2 +- packages/backend/src/misc/antenna-cache.ts | 36 -- packages/backend/src/misc/app-lock.ts | 31 -- packages/backend/src/misc/before-shutdown.ts | 2 +- packages/backend/src/misc/captcha.ts | 57 ---- packages/backend/src/misc/check-hit-antenna.ts | 99 ------ packages/backend/src/misc/check-word-mute.ts | 4 +- packages/backend/src/misc/convert-host.ts | 26 -- packages/backend/src/misc/count-same-renotes.ts | 15 - packages/backend/src/misc/detect-url-mime.ts | 15 - packages/backend/src/misc/download-text-file.ts | 25 -- packages/backend/src/misc/download-url.ts | 89 ----- .../src/misc/extract-custom-emojis-from-mfm.ts | 2 +- packages/backend/src/misc/extract-hashtags.ts | 2 +- packages/backend/src/misc/extract-mentions.ts | 2 +- packages/backend/src/misc/fetch-meta.ts | 44 --- packages/backend/src/misc/fetch-proxy-account.ts | 9 - packages/backend/src/misc/fetch.ts | 141 -------- packages/backend/src/misc/gen-id.ts | 21 -- .../backend/src/misc/generate-native-user-token.ts | 3 + packages/backend/src/misc/get-file-info.ts | 374 --------------------- packages/backend/src/misc/get-note-summary.ts | 10 +- packages/backend/src/misc/identifiable-error.ts | 2 +- packages/backend/src/misc/is-native-token.ts | 1 + packages/backend/src/misc/is-quote.ts | 2 +- packages/backend/src/misc/keypair-store.ts | 10 - packages/backend/src/misc/populate-emojis.ts | 125 ------- packages/backend/src/misc/prelude/README.md | 3 + packages/backend/src/misc/prelude/array.ts | 138 ++++++++ packages/backend/src/misc/prelude/await-all.ts | 21 ++ packages/backend/src/misc/prelude/math.ts | 3 + packages/backend/src/misc/prelude/maybe.ts | 20 ++ packages/backend/src/misc/prelude/relation.ts | 5 + packages/backend/src/misc/prelude/string.ts | 15 + packages/backend/src/misc/prelude/symbol.ts | 1 + packages/backend/src/misc/prelude/time.ts | 39 +++ packages/backend/src/misc/prelude/url.ts | 13 + packages/backend/src/misc/prelude/xml.ts | 41 +++ packages/backend/src/misc/reaction-lib.ts | 131 -------- packages/backend/src/misc/reset-db.ts | 28 ++ packages/backend/src/misc/show-machine-info.ts | 2 +- packages/backend/src/misc/status-error.ts | 13 + packages/backend/src/misc/webhook-cache.ts | 49 --- 43 files changed, 359 insertions(+), 1312 deletions(-) delete mode 100644 packages/backend/src/misc/antenna-cache.ts delete mode 100644 packages/backend/src/misc/app-lock.ts delete mode 100644 packages/backend/src/misc/captcha.ts delete mode 100644 packages/backend/src/misc/check-hit-antenna.ts delete mode 100644 packages/backend/src/misc/convert-host.ts delete mode 100644 packages/backend/src/misc/count-same-renotes.ts delete mode 100644 packages/backend/src/misc/detect-url-mime.ts delete mode 100644 packages/backend/src/misc/download-text-file.ts delete mode 100644 packages/backend/src/misc/download-url.ts delete mode 100644 packages/backend/src/misc/fetch-meta.ts delete mode 100644 packages/backend/src/misc/fetch-proxy-account.ts delete mode 100644 packages/backend/src/misc/fetch.ts delete mode 100644 packages/backend/src/misc/gen-id.ts create mode 100644 packages/backend/src/misc/generate-native-user-token.ts delete mode 100644 packages/backend/src/misc/get-file-info.ts create mode 100644 packages/backend/src/misc/is-native-token.ts delete mode 100644 packages/backend/src/misc/keypair-store.ts delete mode 100644 packages/backend/src/misc/populate-emojis.ts create mode 100644 packages/backend/src/misc/prelude/README.md create mode 100644 packages/backend/src/misc/prelude/array.ts create mode 100644 packages/backend/src/misc/prelude/await-all.ts create mode 100644 packages/backend/src/misc/prelude/math.ts create mode 100644 packages/backend/src/misc/prelude/maybe.ts create mode 100644 packages/backend/src/misc/prelude/relation.ts create mode 100644 packages/backend/src/misc/prelude/string.ts create mode 100644 packages/backend/src/misc/prelude/symbol.ts create mode 100644 packages/backend/src/misc/prelude/time.ts create mode 100644 packages/backend/src/misc/prelude/url.ts create mode 100644 packages/backend/src/misc/prelude/xml.ts delete mode 100644 packages/backend/src/misc/reaction-lib.ts create mode 100644 packages/backend/src/misc/reset-db.ts create mode 100644 packages/backend/src/misc/status-error.ts delete mode 100644 packages/backend/src/misc/webhook-cache.ts (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/misc/acct.ts b/packages/backend/src/misc/acct.ts index c32cee86c9..d1a6852a95 100644 --- a/packages/backend/src/misc/acct.ts +++ b/packages/backend/src/misc/acct.ts @@ -6,7 +6,7 @@ export type Acct = { export function parse(acct: string): Acct { if (acct.startsWith('@')) acct = acct.substr(1); const split = acct.split('@', 2); - return { username: split[0], host: split[1] || null }; + return { username: split[0], host: split[1] ?? null }; } export function toString(acct: Acct): string { diff --git a/packages/backend/src/misc/antenna-cache.ts b/packages/backend/src/misc/antenna-cache.ts deleted file mode 100644 index dcf96c1610..0000000000 --- a/packages/backend/src/misc/antenna-cache.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Antennas } from '@/models/index.js'; -import { Antenna } from '@/models/entities/antenna.js'; -import { subsdcriber } from '../db/redis.js'; - -let antennasFetched = false; -let antennas: Antenna[] = []; - -export async function getAntennas() { - if (!antennasFetched) { - antennas = await Antennas.find(); - antennasFetched = true; - } - - return antennas; -} - -subsdcriber.on('message', async (_, data) => { - const obj = JSON.parse(data); - - if (obj.channel === 'internal') { - const { type, body } = obj.message; - switch (type) { - case 'antennaCreated': - antennas.push(body); - break; - case 'antennaUpdated': - antennas[antennas.findIndex(a => a.id === body.id)] = body; - break; - case 'antennaDeleted': - antennas = antennas.filter(a => a.id !== body.id); - break; - default: - break; - } - } -}); diff --git a/packages/backend/src/misc/app-lock.ts b/packages/backend/src/misc/app-lock.ts deleted file mode 100644 index b5089cc6a6..0000000000 --- a/packages/backend/src/misc/app-lock.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { redisClient } from '../db/redis.js'; -import { promisify } from 'node:util'; -import redisLock from 'redis-lock'; - -/** - * Retry delay (ms) for lock acquisition - */ -const retryDelay = 100; - -const lock: (key: string, timeout?: number) => Promise<() => void> - = redisClient - ? promisify(redisLock(redisClient, retryDelay)) - : async () => () => { }; - -/** - * Get AP Object lock - * @param uri AP object ID - * @param timeout Lock timeout (ms), The timeout releases previous lock. - * @returns Unlock function - */ -export function getApLock(uri: string, timeout = 30 * 1000) { - return lock(`ap-object:${uri}`, timeout); -} - -export function getFetchInstanceMetadataLock(host: string, timeout = 30 * 1000) { - return lock(`instance:${host}`, timeout); -} - -export function getChartInsertLock(lockKey: string, timeout = 30 * 1000) { - return lock(`chart-insert:${lockKey}`, timeout); -} diff --git a/packages/backend/src/misc/before-shutdown.ts b/packages/backend/src/misc/before-shutdown.ts index 93ac7a1f39..74489293d7 100644 --- a/packages/backend/src/misc/before-shutdown.ts +++ b/packages/backend/src/misc/before-shutdown.ts @@ -65,7 +65,7 @@ async function shutdownHandler(signalOrEvent: string) { await listener(signalOrEvent); } catch (err) { if (err instanceof Error) { - console.warn(`A shutdown handler failed before completing with: ${err.message || err}`); + console.warn(`A shutdown handler failed before completing with: ${err.message ?? err}`); } } } diff --git a/packages/backend/src/misc/captcha.ts b/packages/backend/src/misc/captcha.ts deleted file mode 100644 index 9a87a4a3c8..0000000000 --- a/packages/backend/src/misc/captcha.ts +++ /dev/null @@ -1,57 +0,0 @@ -import fetch from 'node-fetch'; -import { URLSearchParams } from 'node:url'; -import { getAgentByUrl } from './fetch.js'; -import config from '@/config/index.js'; - -export async function verifyRecaptcha(secret: string, response: string) { - const result = await getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(e => { - throw `recaptcha-request-failed: ${e}`; - }); - - if (result.success !== true) { - const errorCodes = result['error-codes'] ? result['error-codes']?.join(', ') : ''; - throw `recaptcha-failed: ${errorCodes}`; - } -} - -export async function verifyHcaptcha(secret: string, response: string) { - const result = await getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(e => { - throw `hcaptcha-request-failed: ${e}`; - }); - - if (result.success !== true) { - const errorCodes = result['error-codes'] ? result['error-codes']?.join(', ') : ''; - throw `hcaptcha-failed: ${errorCodes}`; - } -} - -type CaptchaResponse = { - success: boolean; - 'error-codes'?: string[]; -}; - -async function getCaptchaResponse(url: string, secret: string, response: string): Promise { - const params = new URLSearchParams({ - secret, - response, - }); - - const res = await fetch(url, { - method: 'POST', - body: params, - headers: { - 'User-Agent': config.userAgent, - }, - // TODO - //timeout: 10 * 1000, - agent: getAgentByUrl, - }).catch(e => { - throw `${e.message || e}`; - }); - - if (!res.ok) { - throw `${res.status}`; - } - - return await res.json() as CaptchaResponse; -} diff --git a/packages/backend/src/misc/check-hit-antenna.ts b/packages/backend/src/misc/check-hit-antenna.ts deleted file mode 100644 index d9cedee7df..0000000000 --- a/packages/backend/src/misc/check-hit-antenna.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { Antenna } from '@/models/entities/antenna.js'; -import { Note } from '@/models/entities/note.js'; -import { User } from '@/models/entities/user.js'; -import { UserListJoinings, UserGroupJoinings, Blockings } from '@/models/index.js'; -import { getFullApAccount } from './convert-host.js'; -import * as Acct from '@/misc/acct.js'; -import { Packed } from './schema.js'; -import { Cache } from './cache.js'; - -const blockingCache = new Cache(1000 * 60 * 5); - -// NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている - -/** - * noteUserFollowers / antennaUserFollowing はどちらか一方が指定されていればよい - */ -export async function checkHitAntenna(antenna: Antenna, note: (Note | Packed<'Note'>), noteUser: { id: User['id']; username: string; host: string | null; }, noteUserFollowers?: User['id'][], antennaUserFollowing?: User['id'][]): Promise { - if (note.visibility === 'specified') return false; - - // アンテナ作成者がノート作成者にブロックされていたらスキップ - const blockings = await blockingCache.fetch(noteUser.id, () => Blockings.findBy({ blockerId: noteUser.id }).then(res => res.map(x => x.blockeeId))); - if (blockings.some(blocking => blocking === antenna.userId)) return false; - - if (note.visibility === 'followers') { - if (noteUserFollowers && !noteUserFollowers.includes(antenna.userId)) return false; - if (antennaUserFollowing && !antennaUserFollowing.includes(note.userId)) return false; - } - - if (!antenna.withReplies && note.replyId != null) return false; - - if (antenna.src === 'home') { - if (noteUserFollowers && !noteUserFollowers.includes(antenna.userId)) return false; - if (antennaUserFollowing && !antennaUserFollowing.includes(note.userId)) return false; - } else if (antenna.src === 'list') { - const listUsers = (await UserListJoinings.findBy({ - userListId: antenna.userListId!, - })).map(x => x.userId); - - if (!listUsers.includes(note.userId)) return false; - } else if (antenna.src === 'group') { - const joining = await UserGroupJoinings.findOneByOrFail({ id: antenna.userGroupJoiningId! }); - - const groupUsers = (await UserGroupJoinings.findBy({ - userGroupId: joining.userGroupId, - })).map(x => x.userId); - - if (!groupUsers.includes(note.userId)) return false; - } else if (antenna.src === 'users') { - const accts = antenna.users.map(x => { - const { username, host } = Acct.parse(x); - return getFullApAccount(username, host).toLowerCase(); - }); - if (!accts.includes(getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false; - } - - const keywords = antenna.keywords - // Clean up - .map(xs => xs.filter(x => x !== '')) - .filter(xs => xs.length > 0); - - if (keywords.length > 0) { - if (note.text == null) return false; - - const matched = keywords.some(and => - and.every(keyword => - antenna.caseSensitive - ? note.text!.includes(keyword) - : note.text!.toLowerCase().includes(keyword.toLowerCase()) - )); - - if (!matched) return false; - } - - const excludeKeywords = antenna.excludeKeywords - // Clean up - .map(xs => xs.filter(x => x !== '')) - .filter(xs => xs.length > 0); - - if (excludeKeywords.length > 0) { - if (note.text == null) return false; - - const matched = excludeKeywords.some(and => - and.every(keyword => - antenna.caseSensitive - ? note.text!.includes(keyword) - : note.text!.toLowerCase().includes(keyword.toLowerCase()) - )); - - if (matched) return false; - } - - if (antenna.withFile) { - if (note.fileIds && note.fileIds.length === 0) return false; - } - - // TODO: eval expression - - return true; -} diff --git a/packages/backend/src/misc/check-word-mute.ts b/packages/backend/src/misc/check-word-mute.ts index d7662820af..d10aca9e88 100644 --- a/packages/backend/src/misc/check-word-mute.ts +++ b/packages/backend/src/misc/check-word-mute.ts @@ -1,6 +1,6 @@ import RE2 from 're2'; -import { Note } from '@/models/entities/note.js'; -import { User } from '@/models/entities/user.js'; +import type { Note } from '@/models/entities/Note.js'; +import type { User } from '@/models/entities/User.js'; type NoteLike = { userId: Note['userId']; diff --git a/packages/backend/src/misc/convert-host.ts b/packages/backend/src/misc/convert-host.ts deleted file mode 100644 index 7eb940a7e0..0000000000 --- a/packages/backend/src/misc/convert-host.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { URL } from 'node:url'; -import config from '@/config/index.js'; -import { toASCII } from 'punycode'; - -export function getFullApAccount(username: string, host: string | null) { - return host ? `${username}@${toPuny(host)}` : `${username}@${toPuny(config.host)}`; -} - -export function isSelfHost(host: string) { - if (host == null) return true; - return toPuny(config.host) === toPuny(host); -} - -export function extractDbHost(uri: string) { - const url = new URL(uri); - return toPuny(url.hostname); -} - -export function toPuny(host: string) { - return toASCII(host.toLowerCase()); -} - -export function toPunyNullable(host: string | null | undefined): string | null { - if (host == null) return null; - return toASCII(host.toLowerCase()); -} diff --git a/packages/backend/src/misc/count-same-renotes.ts b/packages/backend/src/misc/count-same-renotes.ts deleted file mode 100644 index b7f8ce90c8..0000000000 --- a/packages/backend/src/misc/count-same-renotes.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Notes } from '@/models/index.js'; - -export async function countSameRenotes(userId: string, renoteId: string, excludeNoteId: string | undefined): Promise { - // 指定したユーザーの指定したノートのリノートがいくつあるか数える - const query = Notes.createQueryBuilder('note') - .where('note.userId = :userId', { userId }) - .andWhere('note.renoteId = :renoteId', { renoteId }); - - // 指定した投稿を除く - if (excludeNoteId) { - query.andWhere('note.id != :excludeNoteId', { excludeNoteId }); - } - - return await query.getCount(); -} diff --git a/packages/backend/src/misc/detect-url-mime.ts b/packages/backend/src/misc/detect-url-mime.ts deleted file mode 100644 index cd143cf2fb..0000000000 --- a/packages/backend/src/misc/detect-url-mime.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { createTemp } from './create-temp.js'; -import { downloadUrl } from './download-url.js'; -import { detectType } from './get-file-info.js'; - -export async function detectUrlMime(url: string) { - const [path, cleanup] = await createTemp(); - - try { - await downloadUrl(url, path); - const { mime } = await detectType(path); - return mime; - } finally { - cleanup(); - } -} diff --git a/packages/backend/src/misc/download-text-file.ts b/packages/backend/src/misc/download-text-file.ts deleted file mode 100644 index c62c70ee33..0000000000 --- a/packages/backend/src/misc/download-text-file.ts +++ /dev/null @@ -1,25 +0,0 @@ -import * as fs from 'node:fs'; -import * as util from 'node:util'; -import Logger from '@/services/logger.js'; -import { createTemp } from './create-temp.js'; -import { downloadUrl } from './download-url.js'; - -const logger = new Logger('download-text-file'); - -export async function downloadTextFile(url: string): Promise { - // Create temp file - const [path, cleanup] = await createTemp(); - - logger.info(`Temp file is ${path}`); - - try { - // write content at URL to temp file - await downloadUrl(url, path); - - const text = await util.promisify(fs.readFile)(path, 'utf8'); - - return text; - } finally { - cleanup(); - } -} diff --git a/packages/backend/src/misc/download-url.ts b/packages/backend/src/misc/download-url.ts deleted file mode 100644 index 7c57b140ef..0000000000 --- a/packages/backend/src/misc/download-url.ts +++ /dev/null @@ -1,89 +0,0 @@ -import * as fs from 'node:fs'; -import * as stream from 'node:stream'; -import * as util from 'node:util'; -import got, * as Got from 'got'; -import { httpAgent, httpsAgent, StatusError } from './fetch.js'; -import config from '@/config/index.js'; -import chalk from 'chalk'; -import Logger from '@/services/logger.js'; -import IPCIDR from 'ip-cidr'; -import PrivateIp from 'private-ip'; - -const pipeline = util.promisify(stream.pipeline); - -export async function downloadUrl(url: string, path: string): Promise { - const logger = new Logger('download'); - - logger.info(`Downloading ${chalk.cyan(url)} ...`); - - const timeout = 30 * 1000; - const operationTimeout = 60 * 1000; - const maxSize = config.maxFileSize || 262144000; - - const req = got.stream(url, { - headers: { - 'User-Agent': config.userAgent, - }, - timeout: { - lookup: timeout, - connect: timeout, - secureConnect: timeout, - socket: timeout, // read timeout - response: timeout, - send: timeout, - request: operationTimeout, // whole operation timeout - }, - agent: { - http: httpAgent, - https: httpsAgent, - }, - http2: false, // default - retry: { - limit: 0, - }, - }).on('response', (res: Got.Response) => { - if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) { - if (isPrivateIp(res.ip)) { - logger.warn(`Blocked address: ${res.ip}`); - req.destroy(); - } - } - - const contentLength = res.headers['content-length']; - if (contentLength != null) { - const size = Number(contentLength); - if (size > maxSize) { - logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`); - req.destroy(); - } - } - }).on('downloadProgress', (progress: Got.Progress) => { - if (progress.transferred > maxSize) { - logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); - req.destroy(); - } - }); - - try { - await pipeline(req, fs.createWriteStream(path)); - } catch (e) { - if (e instanceof Got.HTTPError) { - throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); - } else { - throw e; - } - } - - logger.succ(`Download finished: ${chalk.cyan(url)}`); -} - -function isPrivateIp(ip: string): boolean { - for (const net of config.allowedPrivateNetworks || []) { - const cidr = new IPCIDR(net); - if (cidr.contains(ip)) { - return false; - } - } - - return PrivateIp(ip); -} diff --git a/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts b/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts index a0319d8dd5..8fb3f4b19e 100644 --- a/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts +++ b/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts @@ -1,5 +1,5 @@ import * as mfm from 'mfm-js'; -import { unique } from '@/prelude/array.js'; +import { unique } from '@/misc/prelude/array.js'; export function extractCustomEmojisFromMfm(nodes: mfm.MfmNode[]): string[] { const emojiNodes = mfm.extract(nodes, (node) => { diff --git a/packages/backend/src/misc/extract-hashtags.ts b/packages/backend/src/misc/extract-hashtags.ts index 0b0418eefd..f8cabda3d6 100644 --- a/packages/backend/src/misc/extract-hashtags.ts +++ b/packages/backend/src/misc/extract-hashtags.ts @@ -1,5 +1,5 @@ import * as mfm from 'mfm-js'; -import { unique } from '@/prelude/array.js'; +import { unique } from '@/misc/prelude/array.js'; export function extractHashtags(nodes: mfm.MfmNode[]): string[] { const hashtagNodes = mfm.extract(nodes, (node) => node.type === 'hashtag'); diff --git a/packages/backend/src/misc/extract-mentions.ts b/packages/backend/src/misc/extract-mentions.ts index cc19b161a8..c8762e797b 100644 --- a/packages/backend/src/misc/extract-mentions.ts +++ b/packages/backend/src/misc/extract-mentions.ts @@ -4,7 +4,7 @@ import * as mfm from 'mfm-js'; export function extractMentions(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] { // TODO: 重複を削除 - const mentionNodes = mfm.extract(nodes, (node) => node.type === 'mention'); + const mentionNodes = mfm.extract(nodes, (node) => node.type === 'mention') as mfm.MfmMention[]; const mentions = mentionNodes.map(x => x.props); return mentions; diff --git a/packages/backend/src/misc/fetch-meta.ts b/packages/backend/src/misc/fetch-meta.ts deleted file mode 100644 index e855ac28ee..0000000000 --- a/packages/backend/src/misc/fetch-meta.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { db } from '@/db/postgre.js'; -import { Meta } from '@/models/entities/meta.js'; - -let cache: Meta; - -export async function fetchMeta(noCache = false): Promise { - if (!noCache && cache) return cache; - - return await db.transaction(async transactionalEntityManager => { - // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する - const metas = await transactionalEntityManager.find(Meta, { - order: { - id: 'DESC', - }, - }); - - const meta = metas[0]; - - if (meta) { - cache = meta; - return meta; - } else { - // metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う - const saved = await transactionalEntityManager - .upsert( - Meta, - { - id: 'x', - }, - ['id'], - ) - .then((x) => transactionalEntityManager.findOneByOrFail(Meta, x.identifiers[0])); - - cache = saved; - return saved; - } - }); -} - -setInterval(() => { - fetchMeta(true).then(meta => { - cache = meta; - }); -}, 1000 * 10); diff --git a/packages/backend/src/misc/fetch-proxy-account.ts b/packages/backend/src/misc/fetch-proxy-account.ts deleted file mode 100644 index b61bba264b..0000000000 --- a/packages/backend/src/misc/fetch-proxy-account.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { fetchMeta } from './fetch-meta.js'; -import { ILocalUser } from '@/models/entities/user.js'; -import { Users } from '@/models/index.js'; - -export async function fetchProxyAccount(): Promise { - const meta = await fetchMeta(); - if (meta.proxyAccountId == null) return null; - return await Users.findOneByOrFail({ id: meta.proxyAccountId }) as ILocalUser; -} diff --git a/packages/backend/src/misc/fetch.ts b/packages/backend/src/misc/fetch.ts deleted file mode 100644 index af6bf2fca7..0000000000 --- a/packages/backend/src/misc/fetch.ts +++ /dev/null @@ -1,141 +0,0 @@ -import * as http from 'node:http'; -import * as https from 'node:https'; -import { URL } from 'node:url'; -import CacheableLookup from 'cacheable-lookup'; -import fetch from 'node-fetch'; -import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; -import config from '@/config/index.js'; - -export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record) { - const res = await getResponse({ - url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': config.userAgent, - Accept: accept, - }, headers || {}), - timeout, - }); - - return await res.json(); -} - -export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record) { - const res = await getResponse({ - url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': config.userAgent, - Accept: accept, - }, headers || {}), - timeout, - }); - - return await res.text(); -} - -export async function getResponse(args: { url: string, method: string, body?: string, headers: Record, timeout?: number, size?: number }) { - const timeout = args.timeout || 10 * 1000; - - const controller = new AbortController(); - setTimeout(() => { - controller.abort(); - }, timeout * 6); - - const res = await fetch(args.url, { - method: args.method, - headers: args.headers, - body: args.body, - timeout, - size: args.size || 10 * 1024 * 1024, - agent: getAgentByUrl, - signal: controller.signal, - }); - - if (!res.ok) { - throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); - } - - return res; -} - -const cache = new CacheableLookup({ - maxTtl: 3600, // 1hours - errorTtl: 30, // 30secs - lookup: false, // nativeのdns.lookupにfallbackしない -}); - -/** - * Get http non-proxy agent - */ -const _http = new http.Agent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, -} as http.AgentOptions); - -/** - * Get https non-proxy agent - */ -const _https = new https.Agent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, -} as https.AgentOptions); - -const maxSockets = Math.max(256, config.deliverJobConcurrency || 128); - -/** - * Get http proxy or non-proxy agent - */ -export const httpAgent = config.proxy - ? new HttpProxyAgent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - maxSockets, - maxFreeSockets: 256, - scheduling: 'lifo', - proxy: config.proxy, - }) - : _http; - -/** - * Get https proxy or non-proxy agent - */ -export const httpsAgent = config.proxy - ? new HttpsProxyAgent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - maxSockets, - maxFreeSockets: 256, - scheduling: 'lifo', - proxy: config.proxy, - }) - : _https; - -/** - * Get agent by URL - * @param url URL - * @param bypassProxy Allways bypass proxy - */ -export function getAgentByUrl(url: URL, bypassProxy = false) { - if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) { - return url.protocol === 'http:' ? _http : _https; - } else { - return url.protocol === 'http:' ? httpAgent : httpsAgent; - } -} - -export class StatusError extends Error { - public statusCode: number; - public statusMessage?: string; - public isClientError: boolean; - - constructor(message: string, statusCode: number, statusMessage?: string) { - super(message); - this.name = 'StatusError'; - this.statusCode = statusCode; - this.statusMessage = statusMessage; - this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500; - } -} diff --git a/packages/backend/src/misc/gen-id.ts b/packages/backend/src/misc/gen-id.ts deleted file mode 100644 index fcf476857f..0000000000 --- a/packages/backend/src/misc/gen-id.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { ulid } from 'ulid'; -import { genAid } from './id/aid.js'; -import { genMeid } from './id/meid.js'; -import { genMeidg } from './id/meidg.js'; -import { genObjectId } from './id/object-id.js'; -import config from '@/config/index.js'; - -const metohd = config.id.toLowerCase(); - -export function genId(date?: Date): string { - if (!date || (date > new Date())) date = new Date(); - - switch (metohd) { - case 'aid': return genAid(date); - case 'meid': return genMeid(date); - case 'meidg': return genMeidg(date); - case 'ulid': return ulid(date.getTime()); - case 'objectid': return genObjectId(date); - default: throw new Error('unrecognized id generation method'); - } -} diff --git a/packages/backend/src/misc/generate-native-user-token.ts b/packages/backend/src/misc/generate-native-user-token.ts new file mode 100644 index 0000000000..5d8a4c5378 --- /dev/null +++ b/packages/backend/src/misc/generate-native-user-token.ts @@ -0,0 +1,3 @@ +import { secureRndstr } from '@/misc/secure-rndstr.js'; + +export default () => secureRndstr(16, true); diff --git a/packages/backend/src/misc/get-file-info.ts b/packages/backend/src/misc/get-file-info.ts deleted file mode 100644 index 1c988b2487..0000000000 --- a/packages/backend/src/misc/get-file-info.ts +++ /dev/null @@ -1,374 +0,0 @@ -import * as fs from 'node:fs'; -import * as crypto from 'node:crypto'; -import { join } from 'node:path'; -import * as stream from 'node:stream'; -import * as util from 'node:util'; -import { FSWatcher } from 'chokidar'; -import { fileTypeFromFile } from 'file-type'; -import FFmpeg from 'fluent-ffmpeg'; -import isSvg from 'is-svg'; -import probeImageSize from 'probe-image-size'; -import { type predictionType } from 'nsfwjs'; -import sharp from 'sharp'; -import { encode } from 'blurhash'; -import { detectSensitive } from '@/services/detect-sensitive.js'; -import { createTempDir } from './create-temp.js'; - -const pipeline = util.promisify(stream.pipeline); - -export type FileInfo = { - size: number; - md5: string; - type: { - mime: string; - ext: string | null; - }; - width?: number; - height?: number; - orientation?: number; - blurhash?: string; - sensitive: boolean; - porn: boolean; - warnings: string[]; -}; - -const TYPE_OCTET_STREAM = { - mime: 'application/octet-stream', - ext: null, -}; - -const TYPE_SVG = { - mime: 'image/svg+xml', - ext: 'svg', -}; - -/** - * Get file information - */ -export async function getFileInfo(path: string, opts: { - skipSensitiveDetection: boolean; - sensitiveThreshold?: number; - sensitiveThresholdForPorn?: number; - enableSensitiveMediaDetectionForVideos?: boolean; -}): Promise { - const warnings = [] as string[]; - - const size = await getFileSize(path); - const md5 = await calcHash(path); - - let type = await detectType(path); - - // image dimensions - let width: number | undefined; - let height: number | undefined; - let orientation: number | undefined; - - if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/bmp', 'image/tiff', 'image/svg+xml', 'image/vnd.adobe.photoshop'].includes(type.mime)) { - const imageSize = await detectImageSize(path).catch(e => { - warnings.push(`detectImageSize failed: ${e}`); - return undefined; - }); - - // うまく判定できない画像は octet-stream にする - if (!imageSize) { - warnings.push('cannot detect image dimensions'); - type = TYPE_OCTET_STREAM; - } else if (imageSize.wUnits === 'px') { - width = imageSize.width; - height = imageSize.height; - orientation = imageSize.orientation; - - // 制限を超えている画像は octet-stream にする - if (imageSize.width > 16383 || imageSize.height > 16383) { - warnings.push('image dimensions exceeds limits'); - type = TYPE_OCTET_STREAM; - } - } else { - warnings.push(`unsupported unit type: ${imageSize.wUnits}`); - } - } - - let blurhash: string | undefined; - - if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/svg+xml'].includes(type.mime)) { - blurhash = await getBlurhash(path).catch(e => { - warnings.push(`getBlurhash failed: ${e}`); - return undefined; - }); - } - - let sensitive = false; - let porn = false; - - if (!opts.skipSensitiveDetection) { - await detectSensitivity( - path, - type.mime, - opts.sensitiveThreshold ?? 0.5, - opts.sensitiveThresholdForPorn ?? 0.75, - opts.enableSensitiveMediaDetectionForVideos ?? false, - ).then(value => { - [sensitive, porn] = value; - }, error => { - warnings.push(`detectSensitivity failed: ${error}`); - }); - } - - return { - size, - md5, - type, - width, - height, - orientation, - blurhash, - sensitive, - porn, - warnings, - }; -} - -async function detectSensitivity(source: string, mime: string, sensitiveThreshold: number, sensitiveThresholdForPorn: number, analyzeVideo: boolean): Promise<[sensitive: boolean, porn: boolean]> { - let sensitive = false; - let porn = false; - - function judgePrediction(result: readonly predictionType[]): [sensitive: boolean, porn: boolean] { - let sensitive = false; - let porn = false; - - if ((result.find(x => x.className === 'Sexy')?.probability ?? 0) > sensitiveThreshold) sensitive = true; - if ((result.find(x => x.className === 'Hentai')?.probability ?? 0) > sensitiveThreshold) sensitive = true; - if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThreshold) sensitive = true; - - if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThresholdForPorn) porn = true; - - return [sensitive, porn]; - } - - if (['image/jpeg', 'image/png', 'image/webp'].includes(mime)) { - const result = await detectSensitive(source); - if (result) { - [sensitive, porn] = judgePrediction(result); - } - } else if (analyzeVideo && (mime === 'image/apng' || mime.startsWith('video/'))) { - const [outDir, disposeOutDir] = await createTempDir(); - try { - const command = FFmpeg() - .input(source) - .inputOptions([ - '-skip_frame', 'nokey', // 可能ならキーフレームのみを取得してほしいとする（そうなるとは限らない） - '-lowres', '3', // 元の画質でデコードする必要はないので 1/8 画質でデコードしてもよいとする（そうなるとは限らない） - ]) - .noAudio() - .videoFilters([ - { - filter: 'select', // フレームのフィルタリング - options: { - e: 'eq(pict_type,PICT_TYPE_I)', // I-Frame のみをフィルタする（VP9 とかはデコードしてみないとわからないっぽい） - }, - }, - { - filter: 'blackframe', // 暗いフレームの検出 - options: { - amount: '0', // 暗さに関わらず全てのフレームで測定値を取る - }, - }, - { - filter: 'metadata', - options: { - mode: 'select', // フレーム選択モード - key: 'lavfi.blackframe.pblack', // フレームにおける暗部の百分率（前のフィルタからのメタデータを参照する） - value: '50', - function: 'less', // 50% 未満のフレームを選択する（50% 以上暗部があるフレームだと誤検知を招くかもしれないので） - }, - }, - { - filter: 'scale', - options: { - w: 299, - h: 299, - }, - }, - ]) - .format('image2') - .output(join(outDir, '%d.png')) - .outputOptions(['-vsync', '0']); // 可変フレームレートにすることで穴埋めをさせない - const results: ReturnType[] = []; - let frameIndex = 0; - let targetIndex = 0; - let nextIndex = 1; - for await (const path of asyncIterateFrames(outDir, command)) { - try { - const index = frameIndex++; - if (index !== targetIndex) { - continue; - } - targetIndex = nextIndex; - nextIndex += index; // fibonacci sequence によってフレーム数制限を掛ける - const result = await detectSensitive(path); - if (result) { - results.push(judgePrediction(result)); - } - } finally { - fs.promises.unlink(path); - } - } - sensitive = results.filter(x => x[0]).length >= Math.ceil(results.length * sensitiveThreshold); - porn = results.filter(x => x[1]).length >= Math.ceil(results.length * sensitiveThresholdForPorn); - } finally { - disposeOutDir(); - } - } - - return [sensitive, porn]; -} - -async function* asyncIterateFrames(cwd: string, command: FFmpeg.FfmpegCommand): AsyncGenerator { - const watcher = new FSWatcher({ - cwd, - disableGlobbing: true, - }); - let finished = false; - command.once('end', () => { - finished = true; - watcher.close(); - }); - command.run(); - for (let i = 1; true; i++) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition - const current = `${i}.png`; - const next = `${i + 1}.png`; - const framePath = join(cwd, current); - if (await exists(join(cwd, next))) { - yield framePath; - } else if (!finished) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition - watcher.add(next); - await new Promise((resolve, reject) => { - watcher.on('add', function onAdd(path) { - if (path === next) { // 次フレームの書き出しが始まっているなら、現在フレームの書き出しは終わっている - watcher.unwatch(current); - watcher.off('add', onAdd); - resolve(); - } - }); - command.once('end', resolve); // 全てのフレームを処理し終わったなら、最終フレームである現在フレームの書き出しは終わっている - command.once('error', reject); - }); - yield framePath; - } else if (await exists(framePath)) { - yield framePath; - } else { - return; - } - } -} - -function exists(path: string): Promise { - return fs.promises.access(path).then(() => true, () => false); -} - -/** - * Detect MIME Type and extension - */ -export async function detectType(path: string): Promise<{ - mime: string; - ext: string | null; -}> { - // Check 0 byte - const fileSize = await getFileSize(path); - if (fileSize === 0) { - return TYPE_OCTET_STREAM; - } - - const type = await fileTypeFromFile(path); - - if (type) { - // XMLはSVGかもしれない - if (type.mime === 'application/xml' && await checkSvg(path)) { - return TYPE_SVG; - } - - return { - mime: type.mime, - ext: type.ext, - }; - } - - // 種類が不明でもSVGかもしれない - if (await checkSvg(path)) { - return TYPE_SVG; - } - - // それでも種類が不明なら application/octet-stream にする - return TYPE_OCTET_STREAM; -} - -/** - * Check the file is SVG or not - */ -export async function checkSvg(path: string) { - try { - const size = await getFileSize(path); - if (size > 1 * 1024 * 1024) return false; - return isSvg(fs.readFileSync(path)); - } catch { - return false; - } -} - -/** - * Get file size - */ -export async function getFileSize(path: string): Promise { - const getStat = util.promisify(fs.stat); - return (await getStat(path)).size; -} - -/** - * Calculate MD5 hash - */ -async function calcHash(path: string): Promise { - const hash = crypto.createHash('md5').setEncoding('hex'); - await pipeline(fs.createReadStream(path), hash); - return hash.read(); -} - -/** - * Detect dimensions of image - */ -async function detectImageSize(path: string): Promise<{ - width: number; - height: number; - wUnits: string; - hUnits: string; - orientation?: number; -}> { - const readable = fs.createReadStream(path); - const imageSize = await probeImageSize(readable); - readable.destroy(); - return imageSize; -} - -/** - * Calculate average color of image - */ -function getBlurhash(path: string): Promise { - return new Promise((resolve, reject) => { - sharp(path) - .raw() - .ensureAlpha() - .resize(64, 64, { fit: 'inside' }) - .toBuffer((err, buffer, { width, height }) => { - if (err) return reject(err); - - let hash; - - try { - hash = encode(new Uint8ClampedArray(buffer), width, height, 7, 7); - } catch (e) { - return reject(e); - } - - resolve(hash); - }); - }); -} diff --git a/packages/backend/src/misc/get-note-summary.ts b/packages/backend/src/misc/get-note-summary.ts index 3f35ccee82..85bc2ec94d 100644 --- a/packages/backend/src/misc/get-note-summary.ts +++ b/packages/backend/src/misc/get-note-summary.ts @@ -1,4 +1,4 @@ -import { Packed } from './schema.js'; +import type { Packed } from './schema.js'; /** * 投稿を表す文字列を取得します。 @@ -6,11 +6,11 @@ import { Packed } from './schema.js'; */ export const getNoteSummary = (note: Packed<'Note'>): string => { if (note.deletedAt) { - return `(❌⛔)`; + return '(❌⛔)'; } if (note.isHidden) { - return `(⛔)`; + return '(⛔)'; } let summary = ''; @@ -23,13 +23,13 @@ export const getNoteSummary = (note: Packed<'Note'>): string => { } // ファイルが添付されているとき - if ((note.files || []).length !== 0) { + if ((note.files ?? []).length !== 0) { summary += ` (📎${note.files!.length})`; } // 投票が添付されているとき if (note.poll) { - summary += ` (📊)`; + summary += ' (📊)'; } // 返信のとき diff --git a/packages/backend/src/misc/identifiable-error.ts b/packages/backend/src/misc/identifiable-error.ts index 2d7c6bd0c6..e394123f1b 100644 --- a/packages/backend/src/misc/identifiable-error.ts +++ b/packages/backend/src/misc/identifiable-error.ts @@ -7,7 +7,7 @@ export class IdentifiableError extends Error { constructor(id: string, message?: string) { super(message); - this.message = message || ''; + this.message = message ?? ''; this.id = id; } } diff --git a/packages/backend/src/misc/is-native-token.ts b/packages/backend/src/misc/is-native-token.ts new file mode 100644 index 0000000000..2833c570c8 --- /dev/null +++ b/packages/backend/src/misc/is-native-token.ts @@ -0,0 +1 @@ +export default (token: string) => token.length === 16; diff --git a/packages/backend/src/misc/is-quote.ts b/packages/backend/src/misc/is-quote.ts index 779f548b03..6ea71cd878 100644 --- a/packages/backend/src/misc/is-quote.ts +++ b/packages/backend/src/misc/is-quote.ts @@ -1,4 +1,4 @@ -import { Note } from '@/models/entities/note.js'; +import { Note } from '@/models/entities/Note.js'; export default function(note: Note): boolean { return note.renoteId != null && (note.text != null || note.hasPoll || (note.fileIds != null && note.fileIds.length > 0)); diff --git a/packages/backend/src/misc/keypair-store.ts b/packages/backend/src/misc/keypair-store.ts deleted file mode 100644 index 1183b9a781..0000000000 --- a/packages/backend/src/misc/keypair-store.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { UserKeypairs } from '@/models/index.js'; -import { User } from '@/models/entities/user.js'; -import { UserKeypair } from '@/models/entities/user-keypair.js'; -import { Cache } from './cache.js'; - -const cache = new Cache(Infinity); - -export async function getUserKeypair(userId: User['id']): Promise { - return await cache.fetch(userId, () => UserKeypairs.findOneByOrFail({ userId: userId })); -} diff --git a/packages/backend/src/misc/populate-emojis.ts b/packages/backend/src/misc/populate-emojis.ts deleted file mode 100644 index 6a185d09f6..0000000000 --- a/packages/backend/src/misc/populate-emojis.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { In, IsNull } from 'typeorm'; -import { Emojis } from '@/models/index.js'; -import { Emoji } from '@/models/entities/emoji.js'; -import { Note } from '@/models/entities/note.js'; -import { Cache } from './cache.js'; -import { isSelfHost, toPunyNullable } from './convert-host.js'; -import { decodeReaction } from './reaction-lib.js'; -import config from '@/config/index.js'; -import { query } from '@/prelude/url.js'; - -const cache = new Cache(1000 * 60 * 60 * 12); - -/** - * 添付用絵文字情報 - */ -type PopulatedEmoji = { - name: string; - url: string; -}; - -function normalizeHost(src: string | undefined, noteUserHost: string | null): string | null { - // クエリに使うホスト - let host = src === '.' ? null // .はローカルホスト (ここがマッチするのはリアクションのみ) - : src === undefined ? noteUserHost // ノートなどでホスト省略表記の場合はローカルホスト (ここがリアクションにマッチすることはない) - : isSelfHost(src) ? null // 自ホスト指定 - : (src || noteUserHost); // 指定されたホスト || ノートなどの所有者のホスト (こっちがリアクションにマッチすることはない) - - host = toPunyNullable(host); - - return host; -} - -function parseEmojiStr(emojiName: string, noteUserHost: string | null) { - const match = emojiName.match(/^(\w+)(?:@([\w.-]+))?$/); - if (!match) return { name: null, host: null }; - - const name = match[1]; - - // ホスト正規化 - const host = toPunyNullable(normalizeHost(match[2], noteUserHost)); - - return { name, host }; -} - -/** - * 添付用絵文字情報を解決する - * @param emojiName ノートやユーザープロフィールに添付された、またはリアクションのカスタム絵文字名 (:は含めない, リアクションでローカルホストの場合は@.を付ける (これはdecodeReactionで可能)) - * @param noteUserHost ノートやユーザープロフィールの所有者のホスト - * @returns 絵文字情報, nullは未マッチを意味する - */ -export async function populateEmoji(emojiName: string, noteUserHost: string | null): Promise { - const { name, host } = parseEmojiStr(emojiName, noteUserHost); - if (name == null) return null; - - const queryOrNull = async () => (await Emojis.findOneBy({ - name, - host: host ?? IsNull(), - })) || null; - - const emoji = await cache.fetch(`${name} ${host}`, queryOrNull); - - if (emoji == null) return null; - - const isLocal = emoji.host == null; - const emojiUrl = emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため - const url = isLocal ? emojiUrl : `${config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}`; - - return { - name: emojiName, - url, - }; -} - -/** - * 複数の添付用絵文字情報を解決する (キャシュ付き, 存在しないものは結果から除外される) - */ -export async function populateEmojis(emojiNames: string[], noteUserHost: string | null): Promise { - const emojis = await Promise.all(emojiNames.map(x => populateEmoji(x, noteUserHost))); - return emojis.filter((x): x is PopulatedEmoji => x != null); -} - -export function aggregateNoteEmojis(notes: Note[]) { - let emojis: { name: string | null; host: string | null; }[] = []; - for (const note of notes) { - emojis = emojis.concat(note.emojis - .map(e => parseEmojiStr(e, note.userHost))); - if (note.renote) { - emojis = emojis.concat(note.renote.emojis - .map(e => parseEmojiStr(e, note.renote!.userHost))); - if (note.renote.user) { - emojis = emojis.concat(note.renote.user.emojis - .map(e => parseEmojiStr(e, note.renote!.userHost))); - } - } - const customReactions = Object.keys(note.reactions).map(x => decodeReaction(x)).filter(x => x.name != null) as typeof emojis; - emojis = emojis.concat(customReactions); - if (note.user) { - emojis = emojis.concat(note.user.emojis - .map(e => parseEmojiStr(e, note.userHost))); - } - } - return emojis.filter(x => x.name != null) as { name: string; host: string | null; }[]; -} - -/** - * 与えられた絵文字のリストをデータベースから取得し、キャッシュに追加します - */ -export async function prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise { - const notCachedEmojis = emojis.filter(emoji => cache.get(`${emoji.name} ${emoji.host}`) == null); - const emojisQuery: any[] = []; - const hosts = new Set(notCachedEmojis.map(e => e.host)); - for (const host of hosts) { - emojisQuery.push({ - name: In(notCachedEmojis.filter(e => e.host === host).map(e => e.name)), - host: host ?? IsNull(), - }); - } - const _emojis = emojisQuery.length > 0 ? await Emojis.find({ - where: emojisQuery, - select: ['name', 'host', 'originalUrl', 'publicUrl'], - }) : []; - for (const emoji of _emojis) { - cache.set(`${emoji.name} ${emoji.host}`, emoji); - } -} diff --git a/packages/backend/src/misc/prelude/README.md b/packages/backend/src/misc/prelude/README.md new file mode 100644 index 0000000000..bb728cfb1b --- /dev/null +++ b/packages/backend/src/misc/prelude/README.md @@ -0,0 +1,3 @@ +# Prelude +このディレクトリのコードはJavaScriptの表現能力を補うためのコードです。 +Misskey固有の処理とは独立したコードの集まりですが、Misskeyのコードを読みやすくすることを目的としています。 diff --git a/packages/backend/src/misc/prelude/array.ts b/packages/backend/src/misc/prelude/array.ts new file mode 100644 index 0000000000..0b2830cb7b --- /dev/null +++ b/packages/backend/src/misc/prelude/array.ts @@ -0,0 +1,138 @@ +import { EndoRelation, Predicate } from './relation.js'; + +/** + * Count the number of elements that satisfy the predicate + */ + +export function countIf(f: Predicate, xs: T[]): number { + return xs.filter(f).length; +} + +/** + * Count the number of elements that is equal to the element + */ +export function count(a: T, xs: T[]): number { + return countIf(x => x === a, xs); +} + +/** + * Concatenate an array of arrays + */ +export function concat(xss: T[][]): T[] { + return ([] as T[]).concat(...xss); +} + +/** + * Intersperse the element between the elements of the array + * @param sep The element to be interspersed + */ +export function intersperse(sep: T, xs: T[]): T[] { + return concat(xs.map(x => [sep, x])).slice(1); +} + +/** + * Returns the array of elements that is not equal to the element + */ +export function erase(a: T, xs: T[]): T[] { + return xs.filter(x => x !== a); +} + +/** + * Finds the array of all elements in the first array not contained in the second array. + * The order of result values are determined by the first array. + */ +export function difference(xs: T[], ys: T[]): T[] { + return xs.filter(x => !ys.includes(x)); +} + +/** + * Remove all but the first element from every group of equivalent elements + */ +export function unique(xs: T[]): T[] { + return [...new Set(xs)]; +} + +export function sum(xs: number[]): number { + return xs.reduce((a, b) => a + b, 0); +} + +export function maximum(xs: number[]): number { + return Math.max(...xs); +} + +/** + * Splits an array based on the equivalence relation. + * The concatenation of the result is equal to the argument. + */ +export function groupBy(f: EndoRelation, xs: T[]): T[][] { + const groups = [] as T[][]; + for (const x of xs) { + if (groups.length !== 0 && f(groups[groups.length - 1][0], x)) { + groups[groups.length - 1].push(x); + } else { + groups.push([x]); + } + } + return groups; +} + +/** + * Splits an array based on the equivalence relation induced by the function. + * The concatenation of the result is equal to the argument. + */ +export function groupOn(f: (x: T) => S, xs: T[]): T[][] { + return groupBy((a, b) => f(a) === f(b), xs); +} + +export function groupByX(collections: T[], keySelector: (x: T) => string) { + return collections.reduce((obj: Record, item: T) => { + const key = keySelector(item); + if (!Object.prototype.hasOwnProperty.call(obj, key)) { + obj[key] = []; + } + + obj[key].push(item); + + return obj; + }, {}); +} + +/** + * Compare two arrays by lexicographical order + */ +export function lessThan(xs: number[], ys: number[]): boolean { + for (let i = 0; i < Math.min(xs.length, ys.length); i++) { + if (xs[i] < ys[i]) return true; + if (xs[i] > ys[i]) return false; + } + return xs.length < ys.length; +} + +/** + * Returns the longest prefix of elements that satisfy the predicate + */ +export function takeWhile(f: Predicate, xs: T[]): T[] { + const ys = []; + for (const x of xs) { + if (f(x)) { + ys.push(x); + } else { + break; + } + } + return ys; +} + +export function cumulativeSum(xs: number[]): number[] { + const ys = Array.from(xs); // deep copy + for (let i = 1; i < ys.length; i++) ys[i] += ys[i - 1]; + return ys; +} + +export function toArray(x: T | T[] | undefined): T[] { + return Array.isArray(x) ? x : x != null ? [x] : []; +} + +export function toSingle(x: T | T[] | undefined): T | undefined { + return Array.isArray(x) ? x[0] : x; +} diff --git a/packages/backend/src/misc/prelude/await-all.ts b/packages/backend/src/misc/prelude/await-all.ts new file mode 100644 index 0000000000..b955c3a5d8 --- /dev/null +++ b/packages/backend/src/misc/prelude/await-all.ts @@ -0,0 +1,21 @@ +export type Promiseable = { + [K in keyof T]: Promise | T[K]; +}; + +export async function awaitAll(obj: Promiseable): Promise { + const target = {} as T; + const keys = Object.keys(obj) as unknown as (keyof T)[]; + const values = Object.values(obj) as any[]; + + const resolvedValues = await Promise.all(values.map(value => + (!value || !value.constructor || value.constructor.name !== 'Object') + ? value + : awaitAll(value) + )); + + for (let i = 0; i < keys.length; i++) { + target[keys[i]] = resolvedValues[i]; + } + + return target; +} diff --git a/packages/backend/src/misc/prelude/math.ts b/packages/backend/src/misc/prelude/math.ts new file mode 100644 index 0000000000..07b94bec30 --- /dev/null +++ b/packages/backend/src/misc/prelude/math.ts @@ -0,0 +1,3 @@ +export function gcd(a: number, b: number): number { + return b === 0 ? a : gcd(b, a % b); +} diff --git a/packages/backend/src/misc/prelude/maybe.ts b/packages/backend/src/misc/prelude/maybe.ts new file mode 100644 index 0000000000..df7c4ed52a --- /dev/null +++ b/packages/backend/src/misc/prelude/maybe.ts @@ -0,0 +1,20 @@ +export interface IMaybe { + isJust(): this is IJust; +} + +export interface IJust extends IMaybe { + get(): T; +} + +export function just(value: T): IJust { + return { + isJust: () => true, + get: () => value, + }; +} + +export function nothing(): IMaybe { + return { + isJust: () => false, + }; +} diff --git a/packages/backend/src/misc/prelude/relation.ts b/packages/backend/src/misc/prelude/relation.ts new file mode 100644 index 0000000000..1f4703f52f --- /dev/null +++ b/packages/backend/src/misc/prelude/relation.ts @@ -0,0 +1,5 @@ +export type Predicate = (a: T) => boolean; + +export type Relation = (a: T, b: U) => boolean; + +export type EndoRelation = Relation; diff --git a/packages/backend/src/misc/prelude/string.ts b/packages/backend/src/misc/prelude/string.ts new file mode 100644 index 0000000000..b907e0a2e1 --- /dev/null +++ b/packages/backend/src/misc/prelude/string.ts @@ -0,0 +1,15 @@ +export function concat(xs: string[]): string { + return xs.join(''); +} + +export function capitalize(s: string): string { + return toUpperCase(s.charAt(0)) + toLowerCase(s.slice(1)); +} + +export function toUpperCase(s: string): string { + return s.toUpperCase(); +} + +export function toLowerCase(s: string): string { + return s.toLowerCase(); +} diff --git a/packages/backend/src/misc/prelude/symbol.ts b/packages/backend/src/misc/prelude/symbol.ts new file mode 100644 index 0000000000..51e12f7450 --- /dev/null +++ b/packages/backend/src/misc/prelude/symbol.ts @@ -0,0 +1 @@ +export const fallback = Symbol('fallback'); diff --git a/packages/backend/src/misc/prelude/time.ts b/packages/backend/src/misc/prelude/time.ts new file mode 100644 index 0000000000..34e8b6b17c --- /dev/null +++ b/packages/backend/src/misc/prelude/time.ts @@ -0,0 +1,39 @@ +const dateTimeIntervals = { + 'day': 86400000, + 'hour': 3600000, + 'ms': 1, +}; + +export function dateUTC(time: number[]): Date { + const d = time.length === 2 ? Date.UTC(time[0], time[1]) + : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) + : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) + : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) + : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) + : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) + : null; + + if (!d) throw 'wrong number of arguments'; + + return new Date(d); +} + +export function isTimeSame(a: Date, b: Date): boolean { + return a.getTime() === b.getTime(); +} + +export function isTimeBefore(a: Date, b: Date): boolean { + return (a.getTime() - b.getTime()) < 0; +} + +export function isTimeAfter(a: Date, b: Date): boolean { + return (a.getTime() - b.getTime()) > 0; +} + +export function addTime(x: Date, value: number, span: keyof typeof dateTimeIntervals = 'ms'): Date { + return new Date(x.getTime() + (value * dateTimeIntervals[span])); +} + +export function subtractTime(x: Date, value: number, span: keyof typeof dateTimeIntervals = 'ms'): Date { + return new Date(x.getTime() - (value * dateTimeIntervals[span])); +} diff --git a/packages/backend/src/misc/prelude/url.ts b/packages/backend/src/misc/prelude/url.ts new file mode 100644 index 0000000000..a4f2f7f5a8 --- /dev/null +++ b/packages/backend/src/misc/prelude/url.ts @@ -0,0 +1,13 @@ +export function query(obj: Record): string { + const params = Object.entries(obj) + .filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined) + .reduce((a, [k, v]) => (a[k] = v, a), {} as Record); + + return Object.entries(params) + .map((e) => `${e[0]}=${encodeURIComponent(e[1])}`) + .join('&'); +} + +export function appendQuery(url: string, query: string): string { + return `${url}${/\?/.test(url) ? url.endsWith('?') ? '' : '&' : '?'}${query}`; +} diff --git a/packages/backend/src/misc/prelude/xml.ts b/packages/backend/src/misc/prelude/xml.ts new file mode 100644 index 0000000000..b4469a1d8d --- /dev/null +++ b/packages/backend/src/misc/prelude/xml.ts @@ -0,0 +1,41 @@ +const map: Record = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + '\'': ''', +}; + +const beginingOfCDATA = ''; + +export function escapeValue(x: string): string { + let insideOfCDATA = false; + let builder = ''; + for ( + let i = 0; + i < x.length; + ) { + if (insideOfCDATA) { + if (x.slice(i, i + beginingOfCDATA.length) === beginingOfCDATA) { + insideOfCDATA = true; + i += beginingOfCDATA.length; + } else { + builder += x[i++]; + } + } else { + if (x.slice(i, i + endOfCDATA.length) === endOfCDATA) { + insideOfCDATA = false; + i += endOfCDATA.length; + } else { + const b = x[i++]; + builder += map[b] || b; + } + } + } + return builder; +} + +export function escapeAttribute(x: string): string { + return Object.entries(map).reduce((a, [k, v]) => a.replace(k, v), x); +} diff --git a/packages/backend/src/misc/reaction-lib.ts b/packages/backend/src/misc/reaction-lib.ts deleted file mode 100644 index fefc2781f3..0000000000 --- a/packages/backend/src/misc/reaction-lib.ts +++ /dev/null @@ -1,131 +0,0 @@ -/* eslint-disable key-spacing */ -import { emojiRegex } from './emoji-regex.js'; -import { fetchMeta } from './fetch-meta.js'; -import { Emojis } from '@/models/index.js'; -import { toPunyNullable } from './convert-host.js'; -import { IsNull } from 'typeorm'; - -const legacies: Record = { - 'like': '👍', - 'love': '❤', // ここに記述する場合は異体字セレクタを入れない - 'laugh': '😆', - 'hmm': '🤔', - 'surprise': '😮', - 'congrats': '🎉', - 'angry': '💢', - 'confused': '😥', - 'rip': '😇', - 'pudding': '🍮', - 'star': '⭐', -}; - -export async function getFallbackReaction(): Promise { - const meta = await fetchMeta(); - return meta.useStarForReactionFallback ? '⭐' : '👍'; -} - -export function convertLegacyReactions(reactions: Record) { - const _reactions = {} as Record; - - for (const reaction of Object.keys(reactions)) { - if (reactions[reaction] <= 0) continue; - - if (Object.keys(legacies).includes(reaction)) { - if (_reactions[legacies[reaction]]) { - _reactions[legacies[reaction]] += reactions[reaction]; - } else { - _reactions[legacies[reaction]] = reactions[reaction]; - } - } else { - if (_reactions[reaction]) { - _reactions[reaction] += reactions[reaction]; - } else { - _reactions[reaction] = reactions[reaction]; - } - } - } - - const _reactions2 = {} as Record; - - for (const reaction of Object.keys(_reactions)) { - _reactions2[decodeReaction(reaction).reaction] = _reactions[reaction]; - } - - return _reactions2; -} - -export async function toDbReaction(reaction?: string | null, reacterHost?: string | null): Promise { - if (reaction == null) return await getFallbackReaction(); - - reacterHost = toPunyNullable(reacterHost); - - // 文字列タイプのリアクションを絵文字に変換 - if (Object.keys(legacies).includes(reaction)) return legacies[reaction]; - - // Unicode絵文字 - const match = emojiRegex.exec(reaction); - if (match) { - // 合字を含む1つの絵文字 - const unicode = match[0]; - - // 異体字セレクタ除去 - return unicode.match('\u200d') ? unicode : unicode.replace(/\ufe0f/g, ''); - } - - const custom = reaction.match(/^:([\w+-]+)(?:@\.)?:$/); - if (custom) { - const name = custom[1]; - const emoji = await Emojis.findOneBy({ - host: reacterHost ?? IsNull(), - name, - }); - - if (emoji) return reacterHost ? `:${name}@${reacterHost}:` : `:${name}:`; - } - - return await getFallbackReaction(); -} - -type DecodedReaction = { - /** - * リアクション名 (Unicode Emoji or ':name@hostname' or ':name@.') - */ - reaction: string; - - /** - * name (カスタム絵文字の場合name, Emojiクエリに使う) - */ - name?: string; - - /** - * host (カスタム絵文字の場合host, Emojiクエリに使う) - */ - host?: string | null; -}; - -export function decodeReaction(str: string): DecodedReaction { - const custom = str.match(/^:([\w+-]+)(?:@([\w.-]+))?:$/); - - if (custom) { - const name = custom[1]; - const host = custom[2] || null; - - return { - reaction: `:${name}@${host || '.'}:`, // ローカル分は@以降を省略するのではなく.にする - name, - host, - }; - } - - return { - reaction: str, - name: undefined, - host: undefined, - }; -} - -export function convertLegacyReaction(reaction: string): string { - reaction = decodeReaction(reaction).reaction; - if (Object.keys(legacies).includes(reaction)) return legacies[reaction]; - return reaction; -} diff --git a/packages/backend/src/misc/reset-db.ts b/packages/backend/src/misc/reset-db.ts new file mode 100644 index 0000000000..835cd2ba28 --- /dev/null +++ b/packages/backend/src/misc/reset-db.ts @@ -0,0 +1,28 @@ +import type { DataSource } from 'typeorm'; + +export async function resetDb(db: DataSource) { + const reset = async () => { + const tables = await db.query(`SELECT relname AS "table" + FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) + WHERE nspname NOT IN ('pg_catalog', 'information_schema') + AND C.relkind = 'r' + AND nspname !~ '^pg_toast';`); + for (const table of tables) { + await db.query(`DELETE FROM "${table.table}" CASCADE`); + } + }; + + for (let i = 1; i <= 3; i++) { + try { + await reset(); + } catch (e) { + if (i === 3) { + throw e; + } else { + await new Promise(resolve => setTimeout(resolve, 1000)); + continue; + } + } + break; + } +} diff --git a/packages/backend/src/misc/show-machine-info.ts b/packages/backend/src/misc/show-machine-info.ts index bc71cfbe96..bfb1b85f33 100644 --- a/packages/backend/src/misc/show-machine-info.ts +++ b/packages/backend/src/misc/show-machine-info.ts @@ -1,6 +1,6 @@ import * as os from 'node:os'; import sysUtils from 'systeminformation'; -import Logger from '@/services/logger.js'; +import Logger from '@/core/logger.js'; export async function showMachineInfo(parentLogger: Logger) { const logger = parentLogger.createSubLogger('machine'); diff --git a/packages/backend/src/misc/status-error.ts b/packages/backend/src/misc/status-error.ts new file mode 100644 index 0000000000..0a33f8acaf --- /dev/null +++ b/packages/backend/src/misc/status-error.ts @@ -0,0 +1,13 @@ +export class StatusError extends Error { + public statusCode: number; + public statusMessage?: string; + public isClientError: boolean; + + constructor(message: string, statusCode: number, statusMessage?: string) { + super(message); + this.name = 'StatusError'; + this.statusCode = statusCode; + this.statusMessage = statusMessage; + this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500; + } +} diff --git a/packages/backend/src/misc/webhook-cache.ts b/packages/backend/src/misc/webhook-cache.ts deleted file mode 100644 index 4bd2333661..0000000000 --- a/packages/backend/src/misc/webhook-cache.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { Webhooks } from '@/models/index.js'; -import { Webhook } from '@/models/entities/webhook.js'; -import { subsdcriber } from '../db/redis.js'; - -let webhooksFetched = false; -let webhooks: Webhook[] = []; - -export async function getActiveWebhooks() { - if (!webhooksFetched) { - webhooks = await Webhooks.findBy({ - active: true, - }); - webhooksFetched = true; - } - - return webhooks; -} - -subsdcriber.on('message', async (_, data) => { - const obj = JSON.parse(data); - - if (obj.channel === 'internal') { - const { type, body } = obj.message; - switch (type) { - case 'webhookCreated': - if (body.active) { - webhooks.push(body); - } - break; - case 'webhookUpdated': - if (body.active) { - const i = webhooks.findIndex(a => a.id === body.id); - if (i > -1) { - webhooks[i] = body; - } else { - webhooks.push(body); - } - } else { - webhooks = webhooks.filter(a => a.id !== body.id); - } - break; - case 'webhookDeleted': - webhooks = webhooks.filter(a => a.id !== body.id); - break; - default: - break; - } - } -}); -- cgit v1.2.3-freya