From 6ed010b19238be79435238df604a78b2e10e9c4b Mon Sep 17 00:00:00 2001 From: Johann150 Date: Sun, 1 May 2022 12:23:34 +0200 Subject: fix _misskey_content of quote renotes (#8533) --- packages/backend/src/services/note/create.ts | 2 ++ 1 file changed, 2 insertions(+) (limited to 'packages/backend/src/services') diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index f14bc2059b..ceb5e8cc71 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -187,6 +187,8 @@ export default async (user: { id: User['id']; username: User['username']; host: if (data.text) { data.text = data.text.trim(); + } else { + data.text = null; } let tags = data.apHashtags; -- cgit v1.2.3-freya From 55a578a8df67a9028001104d3a4c2584f3b51a7d Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Thu, 19 May 2022 16:19:23 +0900 Subject: fix: Unable to generate video thumbnails (#8696) * fix: Unable to generate video thumbnails * CHANGELOG --- CHANGELOG.md | 1 + packages/backend/src/services/drive/generate-video-thumbnail.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'packages/backend/src/services') diff --git a/CHANGELOG.md b/CHANGELOG.md index f692bdc46e..fb8b8fdee6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ You should also include the user name that made the change. - Server: fix internal in-memory caching @Johann150 - Server: use correct order of attachments on notes @Johann150 - Server: prevent crash when processing certain PNGs @syuilo +- Server: Fix unable to generate video thumbnails @mei23 ## 12.110.1 (2022/04/23) diff --git a/packages/backend/src/services/drive/generate-video-thumbnail.ts b/packages/backend/src/services/drive/generate-video-thumbnail.ts index da93bc97c7..ef75a9f585 100644 --- a/packages/backend/src/services/drive/generate-video-thumbnail.ts +++ b/packages/backend/src/services/drive/generate-video-thumbnail.ts @@ -1,7 +1,7 @@ import * as fs from 'node:fs'; import * as tmp from 'tmp'; import { IImage, convertToJpeg } from './image-processor.js'; -import * as FFmpeg from 'fluent-ffmpeg'; +import FFmpeg from 'fluent-ffmpeg'; export async function GenerateVideoThumbnail(path: string): Promise { const [outDir, cleanup] = await new Promise<[string, any]>((res, rej) => { -- cgit v1.2.3-freya From aaf5bb62abd6c1daefc675a7aa7eebfac561fb3a Mon Sep 17 00:00:00 2001 From: Johann150 Date: Thu, 19 May 2022 09:54:45 +0200 Subject: enhance: uniform theme color (#8702) * enhance: make theme color format uniform All newly fetched instance theme colors will be uniformely formatted as hashtag followed by 6 hexadecimal digits. Colors are checked for validity and invalid colors are not handled. * better input validation for own theme color * migration to unify theme color formats Fixes theme colors of other instances as well as the local instance. * add changelog entry Co-authored-by: syuilo --- CHANGELOG.md | 3 ++ .../migration/1652859567549-uniform-themecolor.js | 38 ++++++++++++++++++++++ .../src/server/api/endpoints/admin/update-meta.ts | 2 +- .../src/services/fetch-instance-metadata.ts | 14 +++----- 4 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 packages/backend/migration/1652859567549-uniform-themecolor.js (limited to 'packages/backend/src/services') diff --git a/CHANGELOG.md b/CHANGELOG.md index fb8b8fdee6..21ae948d0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ You should also include the user name that made the change. - update dependencies @syuilo - enhance: display URL of QR code for TOTP registration @syuilo - make CAPTCHA required for signin to improve security @syuilo +- The theme color is now better validated. @Johann150 + Your own theme color may be unset if it was in an invalid format. + Admins should check their instance settings if in doubt. - Perform port diagnosis at startup only when Listen fails @mei23 ### Bugfixes diff --git a/packages/backend/migration/1652859567549-uniform-themecolor.js b/packages/backend/migration/1652859567549-uniform-themecolor.js new file mode 100644 index 0000000000..bc47143e54 --- /dev/null +++ b/packages/backend/migration/1652859567549-uniform-themecolor.js @@ -0,0 +1,38 @@ +import tinycolor from 'tinycolor2'; + +export class uniformThemecolor1652859567549 { + name = 'uniformThemecolor1652859567549' + + async up(queryRunner) { + const formatColor = (color) => { + let tc = new tinycolor(color); + if (color.isValid()) { + return color.toHexString(); + } else { + return null; + } + }; + + await Promise.all(queryRunner.query('SELECT "id", "themeColor" FROM "instance" WHERE "themeColor" IS NOT NULL') + .then(instances => instances.map(instance => { + // update theme color to uniform format, e.g. #00ff00 + // invalid theme colors get set to null + instance.color = formatColor(instance.color); + + return queryRunner.query('UPDATE "instance" SET "themeColor" = :themeColor WHERE "id" = :id', instance); + }))); + + // also fix own theme color + await queryRunner.query('SELECT "themeColor" FROM "meta" WHERE "themeColor" IS NOT NULL LIMIT 1') + .then(metas => { + if (metas.length > 0) { + return queryRunner.query('UPDATE "meta" SET "themeColor" = :color', { color: formatColor(metas[0].color) }); + } + }); + } + + async down(queryRunner) { + // The original representation is not stored, so migrating back is not possible. + // The new format also works in older versions so this is not a problem. + } +} diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index b23ee9e3df..09e43301b7 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -27,7 +27,7 @@ export const paramDef = { blockedHosts: { type: 'array', nullable: true, items: { type: 'string', } }, - themeColor: { type: 'string', nullable: true }, + themeColor: { type: 'string', nullable: true, pattern: '^#[0-9a-fA-F]{6}$' }, mascotImageUrl: { type: 'string', nullable: true }, bannerUrl: { type: 'string', nullable: true }, errorImageUrl: { type: 'string', nullable: true }, diff --git a/packages/backend/src/services/fetch-instance-metadata.ts b/packages/backend/src/services/fetch-instance-metadata.ts index d5294c5fe8..029c388dc2 100644 --- a/packages/backend/src/services/fetch-instance-metadata.ts +++ b/packages/backend/src/services/fetch-instance-metadata.ts @@ -1,5 +1,6 @@ import { DOMWindow, JSDOM } from 'jsdom'; import fetch from 'node-fetch'; +import tinycolor from 'tinycolor2'; import { getJson, getHtml, getAgentByUrl } from '@/misc/fetch.js'; import { Instance } from '@/models/entities/instance.js'; import { Instances } from '@/models/index.js'; @@ -208,16 +209,11 @@ async function fetchIconUrl(instance: Instance, doc: DOMWindow['document'] | nul } async function getThemeColor(doc: DOMWindow['document'] | null, manifest: Record | null): Promise { - if (doc) { - const themeColor = doc.querySelector('meta[name="theme-color"]')?.getAttribute('content'); + const themeColor = doc?.querySelector('meta[name="theme-color"]')?.getAttribute('content') || manifest?.theme_color; - if (themeColor) { - return themeColor; - } - } - - if (manifest) { - return manifest.theme_color; + if (themeColor) { + const color = new tinycolor(themeColor); + if (color.isValid()) return color.toHexString(); } return null; -- cgit v1.2.3-freya From edfded7fb7e55a83b21256469fd3a58dec1bfe20 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Thu, 19 May 2022 13:40:16 +0200 Subject: fix(activitypub): add authorization checks (#8534) * fix spelling * fix(activitypub): add authorization checks --- .../src/remote/activitypub/kernel/announce/note.ts | 3 +++ .../src/remote/activitypub/kernel/delete/index.ts | 22 +++++++++++----------- .../src/remote/activitypub/kernel/undo/announce.ts | 1 + .../backend/src/services/note/reaction/create.ts | 5 +++++ 4 files changed, 20 insertions(+), 11 deletions(-) (limited to 'packages/backend/src/services') diff --git a/packages/backend/src/remote/activitypub/kernel/announce/note.ts b/packages/backend/src/remote/activitypub/kernel/announce/note.ts index 680749f4d8..052751c654 100644 --- a/packages/backend/src/remote/activitypub/kernel/announce/note.ts +++ b/packages/backend/src/remote/activitypub/kernel/announce/note.ts @@ -9,6 +9,7 @@ import { fetchMeta } from '@/misc/fetch-meta.js'; import { getApLock } from '@/misc/app-lock.js'; import { parseAudience } from '../../audience.js'; import { StatusError } from '@/misc/fetch.js'; +import { Notes } from '@/models/index.js'; const logger = apLogger; @@ -52,6 +53,8 @@ export default async function(resolver: Resolver, actor: CacheableRemoteUser, ac throw e; } + if (!await Notes.isVisibleForMe(renote, actor)) return 'skip: invalid actor for this activity'; + logger.info(`Creating the (Re)Note: ${uri}`); const activityAudience = await parseAudience(actor, activity.to, activity.cc); diff --git a/packages/backend/src/remote/activitypub/kernel/delete/index.ts b/packages/backend/src/remote/activitypub/kernel/delete/index.ts index 4c06a9de0b..c7064f553b 100644 --- a/packages/backend/src/remote/activitypub/kernel/delete/index.ts +++ b/packages/backend/src/remote/activitypub/kernel/delete/index.ts @@ -13,37 +13,37 @@ export default async (actor: CacheableRemoteUser, activity: IDelete): Promise Date: Tue, 24 May 2022 10:12:42 +0200 Subject: fix: wrong type for isVisibleForMe --- packages/backend/src/remote/activitypub/kernel/announce/note.ts | 2 +- packages/backend/src/services/note/reaction/create.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/services') diff --git a/packages/backend/src/remote/activitypub/kernel/announce/note.ts b/packages/backend/src/remote/activitypub/kernel/announce/note.ts index 052751c654..759cb4ae83 100644 --- a/packages/backend/src/remote/activitypub/kernel/announce/note.ts +++ b/packages/backend/src/remote/activitypub/kernel/announce/note.ts @@ -53,7 +53,7 @@ export default async function(resolver: Resolver, actor: CacheableRemoteUser, ac throw e; } - if (!await Notes.isVisibleForMe(renote, actor)) return 'skip: invalid actor for this activity'; + if (!await Notes.isVisibleForMe(renote, actor.id)) return 'skip: invalid actor for this activity'; logger.info(`Creating the (Re)Note: ${uri}`); diff --git a/packages/backend/src/services/note/reaction/create.ts b/packages/backend/src/services/note/reaction/create.ts index 5cb7ebdcd1..83d302826a 100644 --- a/packages/backend/src/services/note/reaction/create.ts +++ b/packages/backend/src/services/note/reaction/create.ts @@ -28,7 +28,7 @@ export default async (user: { id: User['id']; host: User['host']; }, note: Note, } // check visibility - if (!await Notes.isVisibleForMe(note, user)) { + if (!await Notes.isVisibleForMe(note, user.id)) { throw new IdentifiableError('68e9d2d1-48bf-42c2-b90a-b20e09fd3d48', 'Note not accessible for you.'); } -- cgit v1.2.3-freya From e27c6abaeaf0e0e0be9fba7ffc6fd165474d8592 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Wed, 25 May 2022 09:50:22 +0200 Subject: refactor: temporary files (#8713) * simplify temporary files for thumbnails Because only a single file will be written to the directory, creating a separate directory seems unnecessary. If only a temporary file is created, the code from `createTemp` can be reused here as well. * refactor: deduplicate code for temporary files/directories To follow the DRY principle, the same code should not be duplicated across different files. Instead an already existing function is used. Because temporary directories are also create in multiple locations, a function for this is also newly added to reduce duplication. * fix: clean up identicon temp files The temporary files for identicons are not reused and can be deleted after they are fully read. This condition is met when the stream is closed and so the file can be cleaned up using the events API of the stream. * fix: ensure cleanup is called when download fails * fix: ensure cleanup is called in error conditions This covers import/export queue jobs and is mostly just wrapping all code in a try...finally statement where the finally runs the cleanup. * fix: use correct type instead of `any` --- packages/backend/src/misc/create-temp.ts | 13 ++- .../src/queue/processors/db/export-blocking.ts | 111 +++++++++---------- .../queue/processors/db/export-custom-emojis.ts | 17 +-- .../src/queue/processors/db/export-following.ts | 117 ++++++++++---------- .../backend/src/queue/processors/db/export-mute.ts | 113 ++++++++++--------- .../src/queue/processors/db/export-notes.ts | 123 ++++++++++----------- .../src/queue/processors/db/export-user-lists.ts | 63 ++++++----- .../queue/processors/db/import-custom-emojis.ts | 10 +- .../backend/src/server/file/send-drive-file.ts | 9 +- packages/backend/src/server/index.ts | 4 +- .../src/services/drive/generate-video-thumbnail.ts | 53 ++++----- .../backend/src/services/drive/upload-from-url.ts | 25 ++--- 12 files changed, 312 insertions(+), 346 deletions(-) (limited to 'packages/backend/src/services') diff --git a/packages/backend/src/misc/create-temp.ts b/packages/backend/src/misc/create-temp.ts index 04604cf7d0..f07be634fb 100644 --- a/packages/backend/src/misc/create-temp.ts +++ b/packages/backend/src/misc/create-temp.ts @@ -1,10 +1,19 @@ import * as tmp from 'tmp'; -export function createTemp(): Promise<[string, any]> { - return new Promise<[string, any]>((res, rej) => { +export function createTemp(): Promise<[string, () => void]> { + return new Promise<[string, () => void]>((res, rej) => { tmp.file((e, path, fd, cleanup) => { if (e) return rej(e); res([path, cleanup]); }); }); } + +export function createTempDir(): Promise<[string, () => void]> { + return new Promise<[string, () => void]>((res, rej) => { + tmp.dir((e, path, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); +} diff --git a/packages/backend/src/queue/processors/db/export-blocking.ts b/packages/backend/src/queue/processors/db/export-blocking.ts index 166c9e4cd3..f5e0424a79 100644 --- a/packages/backend/src/queue/processors/db/export-blocking.ts +++ b/packages/backend/src/queue/processors/db/export-blocking.ts @@ -1,11 +1,11 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { queueLogger } from '../../logger.js'; import { addFile } from '@/services/drive/add-file.js'; import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host.js'; +import { createTemp } from '@/misc/create-temp.js'; import { Users, Blockings } from '@/models/index.js'; import { MoreThan } from 'typeorm'; import { DbUserJobData } from '@/queue/types.js'; @@ -22,73 +22,72 @@ export async function exportBlocking(job: Bull.Job, done: any): P } // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - const stream = fs.createWriteStream(path, { flags: 'a' }); - - let exportedCount = 0; - let cursor: any = null; - - while (true) { - const blockings = await Blockings.find({ - where: { - blockerId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }); - - if (blockings.length === 0) { - job.progress(100); - break; - } - - cursor = blockings[blockings.length - 1].id; + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); + + let exportedCount = 0; + let cursor: any = null; + + while (true) { + const blockings = await Blockings.find({ + where: { + blockerId: user.id, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }); - for (const block of blockings) { - const u = await Users.findOneBy({ id: block.blockeeId }); - if (u == null) { - exportedCount++; continue; + if (blockings.length === 0) { + job.progress(100); + break; } - const content = getFullApAccount(u.username, u.host); - await new Promise((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } + cursor = blockings[blockings.length - 1].id; + + for (const block of blockings) { + const u = await Users.findOneBy({ id: block.blockeeId }); + if (u == null) { + exportedCount++; continue; + } + + const content = getFullApAccount(u.username, u.host); + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); }); + exportedCount++; + } + + const total = await Blockings.countBy({ + blockerId: user.id, }); - exportedCount++; - } - const total = await Blockings.countBy({ - blockerId: user.id, - }); + job.progress(exportedCount / total); + } - job.progress(exportedCount / total); - } + stream.end(); + logger.succ(`Exported to: ${path}`); - stream.end(); - logger.succ(`Exported to: ${path}`); + const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const driveFile = await addFile({ user, path, name: fileName, force: true }); - const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); + logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); done(); } diff --git a/packages/backend/src/queue/processors/db/export-custom-emojis.ts b/packages/backend/src/queue/processors/db/export-custom-emojis.ts index c2467fb5f0..97ba62dcf6 100644 --- a/packages/backend/src/queue/processors/db/export-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/export-custom-emojis.ts @@ -1,5 +1,4 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { ulid } from 'ulid'; @@ -10,6 +9,7 @@ import { addFile } from '@/services/drive/add-file.js'; import { format as dateFormat } from 'date-fns'; import { Users, Emojis } from '@/models/index.js'; import { } from '@/queue/types.js'; +import { createTempDir } from '@/misc/create-temp.js'; import { downloadUrl } from '@/misc/download-url.js'; import config from '@/config/index.js'; import { IsNull } from 'typeorm'; @@ -25,13 +25,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi return; } - // Create temp dir - const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => { - tmp.dir((e, path, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTempDir(); logger.info(`Temp dir is ${path}`); @@ -98,12 +92,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi metaStream.end(); // Create archive - const [archivePath, archiveCleanup] = await new Promise<[string, () => void]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [archivePath, archiveCleanup] = await createTemp(); const archiveStream = fs.createWriteStream(archivePath); const archive = archiver('zip', { zlib: { level: 0 }, diff --git a/packages/backend/src/queue/processors/db/export-following.ts b/packages/backend/src/queue/processors/db/export-following.ts index 965500ac27..4ac165567b 100644 --- a/packages/backend/src/queue/processors/db/export-following.ts +++ b/packages/backend/src/queue/processors/db/export-following.ts @@ -1,11 +1,11 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { queueLogger } from '../../logger.js'; import { addFile } from '@/services/drive/add-file.js'; import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host.js'; +import { createTemp } from '@/misc/create-temp.js'; import { Users, Followings, Mutings } from '@/models/index.js'; import { In, MoreThan, Not } from 'typeorm'; import { DbUserJobData } from '@/queue/types.js'; @@ -23,73 +23,72 @@ export async function exportFollowing(job: Bull.Job, done: () => } // Create temp file - const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - const stream = fs.createWriteStream(path, { flags: 'a' }); - - let cursor: Following['id'] | null = null; - - const mutings = job.data.excludeMuting ? await Mutings.findBy({ - muterId: user.id, - }) : []; - - while (true) { - const followings = await Followings.find({ - where: { - followerId: user.id, - ...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}), - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as Following[]; - - if (followings.length === 0) { - break; - } - - cursor = followings[followings.length - 1].id; - - for (const following of followings) { - const u = await Users.findOneBy({ id: following.followeeId }); - if (u == null) { - continue; - } - - if (job.data.excludeInactive && u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) { - continue; + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); + + let cursor: Following['id'] | null = null; + + const mutings = job.data.excludeMuting ? await Mutings.findBy({ + muterId: user.id, + }) : []; + + while (true) { + const followings = await Followings.find({ + where: { + followerId: user.id, + ...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}), + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }) as Following[]; + + if (followings.length === 0) { + break; } - const content = getFullApAccount(u.username, u.host); - await new Promise((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } + cursor = followings[followings.length - 1].id; + + for (const following of followings) { + const u = await Users.findOneBy({ id: following.followeeId }); + if (u == null) { + continue; + } + + if (job.data.excludeInactive && u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) { + continue; + } + + const content = getFullApAccount(u.username, u.host); + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); }); - }); + } } - } - stream.end(); - logger.succ(`Exported to: ${path}`); + stream.end(); + logger.succ(`Exported to: ${path}`); - const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); + const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const driveFile = await addFile({ user, path, name: fileName, force: true }); + + logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); done(); } diff --git a/packages/backend/src/queue/processors/db/export-mute.ts b/packages/backend/src/queue/processors/db/export-mute.ts index 0ef81971f1..6a36cfa072 100644 --- a/packages/backend/src/queue/processors/db/export-mute.ts +++ b/packages/backend/src/queue/processors/db/export-mute.ts @@ -1,11 +1,11 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { queueLogger } from '../../logger.js'; import { addFile } from '@/services/drive/add-file.js'; import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host.js'; +import { createTemp } from '@/misc/create-temp.js'; import { Users, Mutings } from '@/models/index.js'; import { IsNull, MoreThan } from 'typeorm'; import { DbUserJobData } from '@/queue/types.js'; @@ -22,74 +22,73 @@ export async function exportMute(job: Bull.Job, done: any): Promi } // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - const stream = fs.createWriteStream(path, { flags: 'a' }); - - let exportedCount = 0; - let cursor: any = null; - - while (true) { - const mutes = await Mutings.find({ - where: { - muterId: user.id, - expiresAt: IsNull(), - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }); - - if (mutes.length === 0) { - job.progress(100); - break; - } - - cursor = mutes[mutes.length - 1].id; + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); + + let exportedCount = 0; + let cursor: any = null; + + while (true) { + const mutes = await Mutings.find({ + where: { + muterId: user.id, + expiresAt: IsNull(), + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }); - for (const mute of mutes) { - const u = await Users.findOneBy({ id: mute.muteeId }); - if (u == null) { - exportedCount++; continue; + if (mutes.length === 0) { + job.progress(100); + break; } - const content = getFullApAccount(u.username, u.host); - await new Promise((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } + cursor = mutes[mutes.length - 1].id; + + for (const mute of mutes) { + const u = await Users.findOneBy({ id: mute.muteeId }); + if (u == null) { + exportedCount++; continue; + } + + const content = getFullApAccount(u.username, u.host); + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); }); + exportedCount++; + } + + const total = await Mutings.countBy({ + muterId: user.id, }); - exportedCount++; - } - const total = await Mutings.countBy({ - muterId: user.id, - }); + job.progress(exportedCount / total); + } - job.progress(exportedCount / total); - } + stream.end(); + logger.succ(`Exported to: ${path}`); - stream.end(); - logger.succ(`Exported to: ${path}`); + const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const driveFile = await addFile({ user, path, name: fileName, force: true }); - const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); + logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); done(); } diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts index 7e12a6fac2..051fcdf385 100644 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ b/packages/backend/src/queue/processors/db/export-notes.ts @@ -1,5 +1,4 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { queueLogger } from '../../logger.js'; @@ -10,6 +9,7 @@ import { MoreThan } from 'typeorm'; import { Note } from '@/models/entities/note.js'; import { Poll } from '@/models/entities/poll.js'; import { DbUserJobData } from '@/queue/types.js'; +import { createTemp } from '@/misc/create-temp.js'; const logger = queueLogger.createSubLogger('export-notes'); @@ -23,82 +23,81 @@ export async function exportNotes(job: Bull.Job, done: any): Prom } // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - const stream = fs.createWriteStream(path, { flags: 'a' }); - - const write = (text: string): Promise => { - return new Promise((res, rej) => { - stream.write(text, err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); + + const write = (text: string): Promise => { + return new Promise((res, rej) => { + stream.write(text, err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); }); - }); - }; + }; + + await write('['); + + let exportedNotesCount = 0; + let cursor: Note['id'] | null = null; + + while (true) { + const notes = await Notes.find({ + where: { + userId: user.id, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }) as Note[]; + + if (notes.length === 0) { + job.progress(100); + break; + } - await write('['); + cursor = notes[notes.length - 1].id; - let exportedNotesCount = 0; - let cursor: Note['id'] | null = null; + for (const note of notes) { + let poll: Poll | undefined; + if (note.hasPoll) { + poll = await Polls.findOneByOrFail({ noteId: note.id }); + } + const content = JSON.stringify(serialize(note, poll)); + const isFirst = exportedNotesCount === 0; + await write(isFirst ? content : ',\n' + content); + exportedNotesCount++; + } - while (true) { - const notes = await Notes.find({ - where: { + const total = await Notes.countBy({ userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as Note[]; - - if (notes.length === 0) { - job.progress(100); - break; - } - - cursor = notes[notes.length - 1].id; + }); - for (const note of notes) { - let poll: Poll | undefined; - if (note.hasPoll) { - poll = await Polls.findOneByOrFail({ noteId: note.id }); - } - const content = JSON.stringify(serialize(note, poll)); - const isFirst = exportedNotesCount === 0; - await write(isFirst ? content : ',\n' + content); - exportedNotesCount++; + job.progress(exportedNotesCount / total); } - const total = await Notes.countBy({ - userId: user.id, - }); + await write(']'); - job.progress(exportedNotesCount / total); - } - - await write(']'); + stream.end(); + logger.succ(`Exported to: ${path}`); - stream.end(); - logger.succ(`Exported to: ${path}`); + const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; + const driveFile = await addFile({ user, path, name: fileName, force: true }); - const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); + logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); done(); } diff --git a/packages/backend/src/queue/processors/db/export-user-lists.ts b/packages/backend/src/queue/processors/db/export-user-lists.ts index 45852a6038..71dd72df27 100644 --- a/packages/backend/src/queue/processors/db/export-user-lists.ts +++ b/packages/backend/src/queue/processors/db/export-user-lists.ts @@ -1,11 +1,11 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import { queueLogger } from '../../logger.js'; import { addFile } from '@/services/drive/add-file.js'; import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host.js'; +import { createTemp } from '@/misc/create-temp.js'; import { Users, UserLists, UserListJoinings } from '@/models/index.js'; import { In } from 'typeorm'; import { DbUserJobData } from '@/queue/types.js'; @@ -26,46 +26,45 @@ export async function exportUserLists(job: Bull.Job, done: any): }); // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - const stream = fs.createWriteStream(path, { flags: 'a' }); + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); - for (const list of lists) { - const joinings = await UserListJoinings.findBy({ userListId: list.id }); - const users = await Users.findBy({ - id: In(joinings.map(j => j.userId)), - }); + for (const list of lists) { + const joinings = await UserListJoinings.findBy({ userListId: list.id }); + const users = await Users.findBy({ + id: In(joinings.map(j => j.userId)), + }); - for (const u of users) { - const acct = getFullApAccount(u.username, u.host); - const content = `${list.name},${acct}`; - await new Promise((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } + for (const u of users) { + const acct = getFullApAccount(u.username, u.host); + const content = `${list.name},${acct}`; + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); }); - }); + } } - } - stream.end(); - logger.succ(`Exported to: ${path}`); + stream.end(); + logger.succ(`Exported to: ${path}`); + + const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const driveFile = await addFile({ user, path, name: fileName, force: true }); - const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); + logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); done(); } diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index 28e0b867a4..64dfe85374 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -1,9 +1,9 @@ import Bull from 'bull'; -import * as tmp from 'tmp'; import * as fs from 'node:fs'; import unzipper from 'unzipper'; import { queueLogger } from '../../logger.js'; +import { createTempDir } from '@/misc/create-temp.js'; import { downloadUrl } from '@/misc/download-url.js'; import { DriveFiles, Emojis } from '@/models/index.js'; import { DbUserImportJobData } from '@/queue/types.js'; @@ -25,13 +25,7 @@ export async function importCustomEmojis(job: Bull.Job, don return; } - // Create temp dir - const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => { - tmp.dir((e, path, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTempDir(); logger.info(`Temp dir is ${path}`); diff --git a/packages/backend/src/server/file/send-drive-file.ts b/packages/backend/src/server/file/send-drive-file.ts index 027d078ce1..c34e043145 100644 --- a/packages/backend/src/server/file/send-drive-file.ts +++ b/packages/backend/src/server/file/send-drive-file.ts @@ -4,11 +4,11 @@ import { dirname } from 'node:path'; import Koa from 'koa'; import send from 'koa-send'; import rename from 'rename'; -import * as tmp from 'tmp'; import { serverLogger } from '../index.js'; import { contentDisposition } from '@/misc/content-disposition.js'; import { DriveFiles } from '@/models/index.js'; import { InternalStorage } from '@/services/drive/internal-storage.js'; +import { createTemp } from '@/misc/create-temp.js'; import { downloadUrl } from '@/misc/download-url.js'; import { detectType } from '@/misc/get-file-info.js'; import { convertToWebp, convertToJpeg, convertToPng } from '@/services/drive/image-processor.js'; @@ -50,12 +50,7 @@ export default async function(ctx: Koa.Context) { if (!file.storedInternal) { if (file.isLink && file.uri) { // 期限切れリモートファイル - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); try { await downloadUrl(file.uri, path); diff --git a/packages/backend/src/server/index.ts b/packages/backend/src/server/index.ts index cd061da725..f31de2b7f4 100644 --- a/packages/backend/src/server/index.ts +++ b/packages/backend/src/server/index.ts @@ -89,10 +89,10 @@ router.get('/avatar/@:acct', async ctx => { }); router.get('/identicon/:x', async ctx => { - const [temp] = await createTemp(); + const [temp, cleanup] = await createTemp(); await genIdenticon(ctx.params.x, fs.createWriteStream(temp)); ctx.set('Content-Type', 'image/png'); - ctx.body = fs.createReadStream(temp); + ctx.body = fs.createReadStream(temp).on('close', () => cleanup()); }); router.get('/verify-email/:code', async ctx => { diff --git a/packages/backend/src/services/drive/generate-video-thumbnail.ts b/packages/backend/src/services/drive/generate-video-thumbnail.ts index ef75a9f585..ca12ab8d3d 100644 --- a/packages/backend/src/services/drive/generate-video-thumbnail.ts +++ b/packages/backend/src/services/drive/generate-video-thumbnail.ts @@ -1,38 +1,31 @@ import * as fs from 'node:fs'; -import * as tmp from 'tmp'; +import * as path from 'node:path'; +import { createTemp } from '@/misc/create-temp.js'; import { IImage, convertToJpeg } from './image-processor.js'; import FFmpeg from 'fluent-ffmpeg'; -export async function GenerateVideoThumbnail(path: string): Promise { - const [outDir, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.dir((e, path, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); +export async function GenerateVideoThumbnail(source: string): Promise { + const [file, cleanup] = await createTemp(); + const parsed = path.parse(file); - await new Promise((res, rej) => { - FFmpeg({ - source: path, - }) - .on('end', res) - .on('error', rej) - .screenshot({ - folder: outDir, - filename: 'output.png', - count: 1, - timestamps: ['5%'], + try { + await new Promise((res, rej) => { + FFmpeg({ + source, + }) + .on('end', res) + .on('error', rej) + .screenshot({ + folder: parsed.dir, + filename: parsed.base, + count: 1, + timestamps: ['5%'], + }); }); - }); - - const outPath = `${outDir}/output.png`; - - // JPEGに変換 (Webpでもいいが、MastodonはWebpをサポートせず表示できなくなる) - const thumbnail = await convertToJpeg(outPath, 498, 280); - - // cleanup - await fs.promises.unlink(outPath); - cleanup(); - return thumbnail; + // JPEGに変換 (Webpでもいいが、MastodonはWebpをサポートせず表示できなくなる) + return await convertToJpeg(498, 280); + } finally { + cleanup(); + } } diff --git a/packages/backend/src/services/drive/upload-from-url.ts b/packages/backend/src/services/drive/upload-from-url.ts index 79b1b8c2e1..001fc49ee4 100644 --- a/packages/backend/src/services/drive/upload-from-url.ts +++ b/packages/backend/src/services/drive/upload-from-url.ts @@ -45,29 +45,20 @@ export async function uploadFromUrl({ // Create temp file const [path, cleanup] = await createTemp(); - // write content at URL to temp file - await downloadUrl(url, path); - - let driveFile: DriveFile; - let error; - try { - driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive }); + // write content at URL to temp file + await downloadUrl(url, path); + + const driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive }); logger.succ(`Got: ${driveFile.id}`); + return driveFile!; } catch (e) { - error = e; logger.error(`Failed to create drive file: ${e}`, { url: url, e: e, }); - } - - // clean-up - cleanup(); - - if (error) { - throw error; - } else { - return driveFile!; + throw e; + } finally { + cleanup(); } } -- cgit v1.2.3-freya