diff options
| author | syuilo <Syuilotan@yahoo.co.jp> | 2022-09-18 03:27:08 +0900 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-09-18 03:27:08 +0900 |
| commit | b75184ec8e3436200bacdcd832e3324702553d20 (patch) | |
| tree | 8b7e316f29e95df921db57289c8b8da476d18f07 /packages/backend/src/queue/processors/db | |
| parent | Update ROADMAP.md (diff) | |
| download | sharkey-b75184ec8e3436200bacdcd832e3324702553d20.tar.gz sharkey-b75184ec8e3436200bacdcd832e3324702553d20.tar.bz2 sharkey-b75184ec8e3436200bacdcd832e3324702553d20.zip | |
なんかもうめっちゃ変えた
Diffstat (limited to 'packages/backend/src/queue/processors/db')
14 files changed, 0 insertions, 1164 deletions
diff --git a/packages/backend/src/queue/processors/db/delete-account.ts b/packages/backend/src/queue/processors/db/delete-account.ts deleted file mode 100644 index c1657b4be6..0000000000 --- a/packages/backend/src/queue/processors/db/delete-account.ts +++ /dev/null @@ -1,94 +0,0 @@ -import Bull from 'bull'; -import { queueLogger } from '../../logger.js'; -import { DriveFiles, Notes, UserProfiles, Users } from '@/models/index.js'; -import { DbUserDeleteJobData } from '@/queue/types.js'; -import { Note } from '@/models/entities/note.js'; -import { DriveFile } from '@/models/entities/drive-file.js'; -import { MoreThan } from 'typeorm'; -import { deleteFileSync } from '@/services/drive/delete-file.js'; -import { sendEmail } from '@/services/send-email.js'; - -const logger = queueLogger.createSubLogger('delete-account'); - -export async function deleteAccount(job: Bull.Job<DbUserDeleteJobData>): Promise<string | void> { - logger.info(`Deleting account of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - return; - } - - { // Delete notes - let cursor: Note['id'] | null = null; - - while (true) { - const notes = await Notes.find({ - where: { - userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as Note[]; - - if (notes.length === 0) { - break; - } - - cursor = notes[notes.length - 1].id; - - await Notes.delete(notes.map(note => note.id)); - } - - logger.succ(`All of notes deleted`); - } - - { // Delete files - let cursor: DriveFile['id'] | null = null; - - while (true) { - const files = await DriveFiles.find({ - where: { - userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 10, - order: { - id: 1, - }, - }) as DriveFile[]; - - if (files.length === 0) { - break; - } - - cursor = files[files.length - 1].id; - - for (const file of files) { - await deleteFileSync(file); - } - } - - logger.succ(`All of files deleted`); - } - - { // Send email notification - const profile = await UserProfiles.findOneByOrFail({ userId: user.id }); - if (profile.email && profile.emailVerified) { - sendEmail(profile.email, 'Account deleted', - `Your account has been deleted.`, - `Your account has been deleted.`); - } - } - - // soft指定されている場合は物理削除しない - if (job.data.soft) { - // nop - } else { - await Users.delete(job.data.user.id); - } - - return 'Account deleted'; -} diff --git a/packages/backend/src/queue/processors/db/delete-drive-files.ts b/packages/backend/src/queue/processors/db/delete-drive-files.ts deleted file mode 100644 index b3832d9f04..0000000000 --- a/packages/backend/src/queue/processors/db/delete-drive-files.ts +++ /dev/null @@ -1,56 +0,0 @@ -import Bull from 'bull'; - -import { queueLogger } from '../../logger.js'; -import { deleteFileSync } from '@/services/drive/delete-file.js'; -import { Users, DriveFiles } from '@/models/index.js'; -import { MoreThan } from 'typeorm'; -import { DbUserJobData } from '@/queue/types.js'; - -const logger = queueLogger.createSubLogger('delete-drive-files'); - -export async function deleteDriveFiles(job: Bull.Job<DbUserJobData>, done: any): Promise<void> { - logger.info(`Deleting drive files of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - let deletedCount = 0; - let cursor: any = null; - - while (true) { - const files = await DriveFiles.find({ - where: { - userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }); - - if (files.length === 0) { - job.progress(100); - break; - } - - cursor = files[files.length - 1].id; - - for (const file of files) { - await deleteFileSync(file); - deletedCount++; - } - - const total = await DriveFiles.countBy({ - userId: user.id, - }); - - job.progress(deletedCount / total); - } - - logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`); - done(); -} diff --git a/packages/backend/src/queue/processors/db/export-blocking.ts b/packages/backend/src/queue/processors/db/export-blocking.ts deleted file mode 100644 index f5e0424a79..0000000000 --- a/packages/backend/src/queue/processors/db/export-blocking.ts +++ /dev/null @@ -1,93 +0,0 @@ -import Bull from 'bull'; -import * as fs from 'node:fs'; - -import { queueLogger } from '../../logger.js'; -import { addFile } from '@/services/drive/add-file.js'; -import { format as dateFormat } from 'date-fns'; -import { getFullApAccount } from '@/misc/convert-host.js'; -import { createTemp } from '@/misc/create-temp.js'; -import { Users, Blockings } from '@/models/index.js'; -import { MoreThan } from 'typeorm'; -import { DbUserJobData } from '@/queue/types.js'; - -const logger = queueLogger.createSubLogger('export-blocking'); - -export async function exportBlocking(job: Bull.Job<DbUserJobData>, done: any): Promise<void> { - logger.info(`Exporting blocking of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - // Create temp file - const [path, cleanup] = await createTemp(); - - logger.info(`Temp file is ${path}`); - - try { - const stream = fs.createWriteStream(path, { flags: 'a' }); - - let exportedCount = 0; - let cursor: any = null; - - while (true) { - const blockings = await Blockings.find({ - where: { - blockerId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }); - - if (blockings.length === 0) { - job.progress(100); - break; - } - - cursor = blockings[blockings.length - 1].id; - - for (const block of blockings) { - const u = await Users.findOneBy({ id: block.blockeeId }); - if (u == null) { - exportedCount++; continue; - } - - const content = getFullApAccount(u.username, u.host); - await new Promise<void>((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } - }); - }); - exportedCount++; - } - - const total = await Blockings.countBy({ - blockerId: user.id, - }); - - job.progress(exportedCount / total); - } - - stream.end(); - logger.succ(`Exported to: ${path}`); - - const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); - - logger.succ(`Exported to: ${driveFile.id}`); - } finally { - cleanup(); - } - - done(); -} diff --git a/packages/backend/src/queue/processors/db/export-custom-emojis.ts b/packages/backend/src/queue/processors/db/export-custom-emojis.ts deleted file mode 100644 index 3da887cda2..0000000000 --- a/packages/backend/src/queue/processors/db/export-custom-emojis.ts +++ /dev/null @@ -1,114 +0,0 @@ -import Bull from 'bull'; -import * as fs from 'node:fs'; - -import { ulid } from 'ulid'; -import mime from 'mime-types'; -import archiver from 'archiver'; -import { queueLogger } from '../../logger.js'; -import { addFile } from '@/services/drive/add-file.js'; -import { format as dateFormat } from 'date-fns'; -import { Users, Emojis } from '@/models/index.js'; -import { } from '@/queue/types.js'; -import { createTemp, createTempDir } from '@/misc/create-temp.js'; -import { downloadUrl } from '@/misc/download-url.js'; -import config from '@/config/index.js'; -import { IsNull } from 'typeorm'; - -const logger = queueLogger.createSubLogger('export-custom-emojis'); - -export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promise<void> { - logger.info(`Exporting custom emojis ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - const [path, cleanup] = await createTempDir(); - - logger.info(`Temp dir is ${path}`); - - const metaPath = path + '/meta.json'; - - fs.writeFileSync(metaPath, '', 'utf-8'); - - const metaStream = fs.createWriteStream(metaPath, { flags: 'a' }); - - const writeMeta = (text: string): Promise<void> => { - return new Promise<void>((res, rej) => { - metaStream.write(text, err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } - }); - }); - }; - - await writeMeta(`{"metaVersion":2,"host":"${config.host}","exportedAt":"${new Date().toString()}","emojis":[`); - - const customEmojis = await Emojis.find({ - where: { - host: IsNull(), - }, - order: { - id: 'ASC', - }, - }); - - for (const emoji of customEmojis) { - const ext = mime.extension(emoji.type); - const fileName = emoji.name + (ext ? '.' + ext : ''); - const emojiPath = path + '/' + fileName; - fs.writeFileSync(emojiPath, '', 'binary'); - let downloaded = false; - - try { - await downloadUrl(emoji.originalUrl, emojiPath); - downloaded = true; - } catch (e) { // TODO: 何度か再試行 - logger.error(e instanceof Error ? e : new Error(e as string)); - } - - if (!downloaded) { - fs.unlinkSync(emojiPath); - } - - const content = JSON.stringify({ - fileName: fileName, - downloaded: downloaded, - emoji: emoji, - }); - const isFirst = customEmojis.indexOf(emoji) === 0; - - await writeMeta(isFirst ? content : ',\n' + content); - } - - await writeMeta(']}'); - - metaStream.end(); - - // Create archive - const [archivePath, archiveCleanup] = await createTemp(); - const archiveStream = fs.createWriteStream(archivePath); - const archive = archiver('zip', { - zlib: { level: 0 }, - }); - archiveStream.on('close', async () => { - logger.succ(`Exported to: ${archivePath}`); - - const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; - const driveFile = await addFile({ user, path: archivePath, name: fileName, force: true }); - - logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); - archiveCleanup(); - done(); - }); - archive.pipe(archiveStream); - archive.directory(path, false); - archive.finalize(); -} diff --git a/packages/backend/src/queue/processors/db/export-following.ts b/packages/backend/src/queue/processors/db/export-following.ts deleted file mode 100644 index 4ac165567b..0000000000 --- a/packages/backend/src/queue/processors/db/export-following.ts +++ /dev/null @@ -1,94 +0,0 @@ -import Bull from 'bull'; -import * as fs from 'node:fs'; - -import { queueLogger } from '../../logger.js'; -import { addFile } from '@/services/drive/add-file.js'; -import { format as dateFormat } from 'date-fns'; -import { getFullApAccount } from '@/misc/convert-host.js'; -import { createTemp } from '@/misc/create-temp.js'; -import { Users, Followings, Mutings } from '@/models/index.js'; -import { In, MoreThan, Not } from 'typeorm'; -import { DbUserJobData } from '@/queue/types.js'; -import { Following } from '@/models/entities/following.js'; - -const logger = queueLogger.createSubLogger('export-following'); - -export async function exportFollowing(job: Bull.Job<DbUserJobData>, done: () => void): Promise<void> { - logger.info(`Exporting following of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - // Create temp file - const [path, cleanup] = await createTemp(); - - logger.info(`Temp file is ${path}`); - - try { - const stream = fs.createWriteStream(path, { flags: 'a' }); - - let cursor: Following['id'] | null = null; - - const mutings = job.data.excludeMuting ? await Mutings.findBy({ - muterId: user.id, - }) : []; - - while (true) { - const followings = await Followings.find({ - where: { - followerId: user.id, - ...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}), - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as Following[]; - - if (followings.length === 0) { - break; - } - - cursor = followings[followings.length - 1].id; - - for (const following of followings) { - const u = await Users.findOneBy({ id: following.followeeId }); - if (u == null) { - continue; - } - - if (job.data.excludeInactive && u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) { - continue; - } - - const content = getFullApAccount(u.username, u.host); - await new Promise<void>((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } - }); - }); - } - } - - stream.end(); - logger.succ(`Exported to: ${path}`); - - const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); - - logger.succ(`Exported to: ${driveFile.id}`); - } finally { - cleanup(); - } - - done(); -} diff --git a/packages/backend/src/queue/processors/db/export-mute.ts b/packages/backend/src/queue/processors/db/export-mute.ts deleted file mode 100644 index 6a36cfa072..0000000000 --- a/packages/backend/src/queue/processors/db/export-mute.ts +++ /dev/null @@ -1,94 +0,0 @@ -import Bull from 'bull'; -import * as fs from 'node:fs'; - -import { queueLogger } from '../../logger.js'; -import { addFile } from '@/services/drive/add-file.js'; -import { format as dateFormat } from 'date-fns'; -import { getFullApAccount } from '@/misc/convert-host.js'; -import { createTemp } from '@/misc/create-temp.js'; -import { Users, Mutings } from '@/models/index.js'; -import { IsNull, MoreThan } from 'typeorm'; -import { DbUserJobData } from '@/queue/types.js'; - -const logger = queueLogger.createSubLogger('export-mute'); - -export async function exportMute(job: Bull.Job<DbUserJobData>, done: any): Promise<void> { - logger.info(`Exporting mute of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - // Create temp file - const [path, cleanup] = await createTemp(); - - logger.info(`Temp file is ${path}`); - - try { - const stream = fs.createWriteStream(path, { flags: 'a' }); - - let exportedCount = 0; - let cursor: any = null; - - while (true) { - const mutes = await Mutings.find({ - where: { - muterId: user.id, - expiresAt: IsNull(), - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }); - - if (mutes.length === 0) { - job.progress(100); - break; - } - - cursor = mutes[mutes.length - 1].id; - - for (const mute of mutes) { - const u = await Users.findOneBy({ id: mute.muteeId }); - if (u == null) { - exportedCount++; continue; - } - - const content = getFullApAccount(u.username, u.host); - await new Promise<void>((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } - }); - }); - exportedCount++; - } - - const total = await Mutings.countBy({ - muterId: user.id, - }); - - job.progress(exportedCount / total); - } - - stream.end(); - logger.succ(`Exported to: ${path}`); - - const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); - - logger.succ(`Exported to: ${driveFile.id}`); - } finally { - cleanup(); - } - - done(); -} diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts deleted file mode 100644 index 051fcdf385..0000000000 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ /dev/null @@ -1,118 +0,0 @@ -import Bull from 'bull'; -import * as fs from 'node:fs'; - -import { queueLogger } from '../../logger.js'; -import { addFile } from '@/services/drive/add-file.js'; -import { format as dateFormat } from 'date-fns'; -import { Users, Notes, Polls } from '@/models/index.js'; -import { MoreThan } from 'typeorm'; -import { Note } from '@/models/entities/note.js'; -import { Poll } from '@/models/entities/poll.js'; -import { DbUserJobData } from '@/queue/types.js'; -import { createTemp } from '@/misc/create-temp.js'; - -const logger = queueLogger.createSubLogger('export-notes'); - -export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Promise<void> { - logger.info(`Exporting notes of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - // Create temp file - const [path, cleanup] = await createTemp(); - - logger.info(`Temp file is ${path}`); - - try { - const stream = fs.createWriteStream(path, { flags: 'a' }); - - const write = (text: string): Promise<void> => { - return new Promise<void>((res, rej) => { - stream.write(text, err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } - }); - }); - }; - - await write('['); - - let exportedNotesCount = 0; - let cursor: Note['id'] | null = null; - - while (true) { - const notes = await Notes.find({ - where: { - userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as Note[]; - - if (notes.length === 0) { - job.progress(100); - break; - } - - cursor = notes[notes.length - 1].id; - - for (const note of notes) { - let poll: Poll | undefined; - if (note.hasPoll) { - poll = await Polls.findOneByOrFail({ noteId: note.id }); - } - const content = JSON.stringify(serialize(note, poll)); - const isFirst = exportedNotesCount === 0; - await write(isFirst ? content : ',\n' + content); - exportedNotesCount++; - } - - const total = await Notes.countBy({ - userId: user.id, - }); - - job.progress(exportedNotesCount / total); - } - - await write(']'); - - stream.end(); - logger.succ(`Exported to: ${path}`); - - const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); - - logger.succ(`Exported to: ${driveFile.id}`); - } finally { - cleanup(); - } - - done(); -} - -function serialize(note: Note, poll: Poll | null = null): Record<string, unknown> { - return { - id: note.id, - text: note.text, - createdAt: note.createdAt, - fileIds: note.fileIds, - replyId: note.replyId, - renoteId: note.renoteId, - poll: poll, - cw: note.cw, - visibility: note.visibility, - visibleUserIds: note.visibleUserIds, - localOnly: note.localOnly, - }; -} diff --git a/packages/backend/src/queue/processors/db/export-user-lists.ts b/packages/backend/src/queue/processors/db/export-user-lists.ts deleted file mode 100644 index 71dd72df27..0000000000 --- a/packages/backend/src/queue/processors/db/export-user-lists.ts +++ /dev/null @@ -1,70 +0,0 @@ -import Bull from 'bull'; -import * as fs from 'node:fs'; - -import { queueLogger } from '../../logger.js'; -import { addFile } from '@/services/drive/add-file.js'; -import { format as dateFormat } from 'date-fns'; -import { getFullApAccount } from '@/misc/convert-host.js'; -import { createTemp } from '@/misc/create-temp.js'; -import { Users, UserLists, UserListJoinings } from '@/models/index.js'; -import { In } from 'typeorm'; -import { DbUserJobData } from '@/queue/types.js'; - -const logger = queueLogger.createSubLogger('export-user-lists'); - -export async function exportUserLists(job: Bull.Job<DbUserJobData>, done: any): Promise<void> { - logger.info(`Exporting user lists of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - const lists = await UserLists.findBy({ - userId: user.id, - }); - - // Create temp file - const [path, cleanup] = await createTemp(); - - logger.info(`Temp file is ${path}`); - - try { - const stream = fs.createWriteStream(path, { flags: 'a' }); - - for (const list of lists) { - const joinings = await UserListJoinings.findBy({ userListId: list.id }); - const users = await Users.findBy({ - id: In(joinings.map(j => j.userId)), - }); - - for (const u of users) { - const acct = getFullApAccount(u.username, u.host); - const content = `${list.name},${acct}`; - await new Promise<void>((res, rej) => { - stream.write(content + '\n', err => { - if (err) { - logger.error(err); - rej(err); - } else { - res(); - } - }); - }); - } - } - - stream.end(); - logger.succ(`Exported to: ${path}`); - - const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; - const driveFile = await addFile({ user, path, name: fileName, force: true }); - - logger.succ(`Exported to: ${driveFile.id}`); - } finally { - cleanup(); - } - - done(); -} diff --git a/packages/backend/src/queue/processors/db/import-blocking.ts b/packages/backend/src/queue/processors/db/import-blocking.ts deleted file mode 100644 index 8bddf34bc2..0000000000 --- a/packages/backend/src/queue/processors/db/import-blocking.ts +++ /dev/null @@ -1,75 +0,0 @@ -import Bull from 'bull'; - -import { queueLogger } from '../../logger.js'; -import * as Acct from '@/misc/acct.js'; -import { resolveUser } from '@/remote/resolve-user.js'; -import { downloadTextFile } from '@/misc/download-text-file.js'; -import { isSelfHost, toPuny } from '@/misc/convert-host.js'; -import { Users, DriveFiles, Blockings } from '@/models/index.js'; -import { DbUserImportJobData } from '@/queue/types.js'; -import block from '@/services/blocking/create.js'; -import { IsNull } from 'typeorm'; - -const logger = queueLogger.createSubLogger('import-blocking'); - -export async function importBlocking(job: Bull.Job<DbUserImportJobData>, done: any): Promise<void> { - logger.info(`Importing blocking of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - const file = await DriveFiles.findOneBy({ - id: job.data.fileId, - }); - if (file == null) { - done(); - return; - } - - const csv = await downloadTextFile(file.url); - - let linenum = 0; - - for (const line of csv.trim().split('\n')) { - linenum++; - - try { - const acct = line.split(',')[0].trim(); - const { username, host } = Acct.parse(acct); - - let target = isSelfHost(host!) ? await Users.findOneBy({ - host: IsNull(), - usernameLower: username.toLowerCase(), - }) : await Users.findOneBy({ - host: toPuny(host!), - usernameLower: username.toLowerCase(), - }); - - if (host == null && target == null) continue; - - if (target == null) { - target = await resolveUser(username, host); - } - - if (target == null) { - throw `cannot resolve user: @${username}@${host}`; - } - - // skip myself - if (target.id === job.data.user.id) continue; - - logger.info(`Block[${linenum}] ${target.id} ...`); - - await block(user, target); - } catch (e) { - logger.warn(`Error in line:${linenum} ${e}`); - } - } - - logger.succ('Imported'); - done(); -} - diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts deleted file mode 100644 index 64dfe85374..0000000000 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ /dev/null @@ -1,81 +0,0 @@ -import Bull from 'bull'; -import * as fs from 'node:fs'; -import unzipper from 'unzipper'; - -import { queueLogger } from '../../logger.js'; -import { createTempDir } from '@/misc/create-temp.js'; -import { downloadUrl } from '@/misc/download-url.js'; -import { DriveFiles, Emojis } from '@/models/index.js'; -import { DbUserImportJobData } from '@/queue/types.js'; -import { addFile } from '@/services/drive/add-file.js'; -import { genId } from '@/misc/gen-id.js'; -import { db } from '@/db/postgre.js'; - -const logger = queueLogger.createSubLogger('import-custom-emojis'); - -// TODO: 名前衝突時の動作を選べるようにする -export async function importCustomEmojis(job: Bull.Job<DbUserImportJobData>, done: any): Promise<void> { - logger.info(`Importing custom emojis ...`); - - const file = await DriveFiles.findOneBy({ - id: job.data.fileId, - }); - if (file == null) { - done(); - return; - } - - const [path, cleanup] = await createTempDir(); - - logger.info(`Temp dir is ${path}`); - - const destPath = path + '/emojis.zip'; - - try { - fs.writeFileSync(destPath, '', 'binary'); - await downloadUrl(file.url, destPath); - } catch (e) { // TODO: 何度か再試行 - if (e instanceof Error || typeof e === 'string') { - logger.error(e); - } - throw e; - } - - const outputPath = path + '/emojis'; - const unzipStream = fs.createReadStream(destPath); - const extractor = unzipper.Extract({ path: outputPath }); - extractor.on('close', async () => { - const metaRaw = fs.readFileSync(outputPath + '/meta.json', 'utf-8'); - const meta = JSON.parse(metaRaw); - - for (const record of meta.emojis) { - if (!record.downloaded) continue; - const emojiInfo = record.emoji; - const emojiPath = outputPath + '/' + record.fileName; - await Emojis.delete({ - name: emojiInfo.name, - }); - const driveFile = await addFile({ user: null, path: emojiPath, name: record.fileName, force: true }); - const emoji = await Emojis.insert({ - id: genId(), - updatedAt: new Date(), - name: emojiInfo.name, - category: emojiInfo.category, - host: null, - aliases: emojiInfo.aliases, - originalUrl: driveFile.url, - publicUrl: driveFile.webpublicUrl ?? driveFile.url, - type: driveFile.webpublicType ?? driveFile.type, - }).then(x => Emojis.findOneByOrFail(x.identifiers[0])); - } - - await db.queryResultCache!.remove(['meta_emojis']); - - cleanup(); - - logger.succ('Imported'); - done(); - }); - unzipStream.pipe(extractor); - logger.succ(`Unzipping to ${outputPath}`); -} diff --git a/packages/backend/src/queue/processors/db/import-following.ts b/packages/backend/src/queue/processors/db/import-following.ts deleted file mode 100644 index 8ce2c367d6..0000000000 --- a/packages/backend/src/queue/processors/db/import-following.ts +++ /dev/null @@ -1,74 +0,0 @@ -import Bull from 'bull'; - -import { queueLogger } from '../../logger.js'; -import follow from '@/services/following/create.js'; -import * as Acct from '@/misc/acct.js'; -import { resolveUser } from '@/remote/resolve-user.js'; -import { downloadTextFile } from '@/misc/download-text-file.js'; -import { isSelfHost, toPuny } from '@/misc/convert-host.js'; -import { Users, DriveFiles } from '@/models/index.js'; -import { DbUserImportJobData } from '@/queue/types.js'; -import { IsNull } from 'typeorm'; - -const logger = queueLogger.createSubLogger('import-following'); - -export async function importFollowing(job: Bull.Job<DbUserImportJobData>, done: any): Promise<void> { - logger.info(`Importing following of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - const file = await DriveFiles.findOneBy({ - id: job.data.fileId, - }); - if (file == null) { - done(); - return; - } - - const csv = await downloadTextFile(file.url); - - let linenum = 0; - - for (const line of csv.trim().split('\n')) { - linenum++; - - try { - const acct = line.split(',')[0].trim(); - const { username, host } = Acct.parse(acct); - - let target = isSelfHost(host!) ? await Users.findOneBy({ - host: IsNull(), - usernameLower: username.toLowerCase(), - }) : await Users.findOneBy({ - host: toPuny(host!), - usernameLower: username.toLowerCase(), - }); - - if (host == null && target == null) continue; - - if (target == null) { - target = await resolveUser(username, host); - } - - if (target == null) { - throw `cannot resolve user: @${username}@${host}`; - } - - // skip myself - if (target.id === job.data.user.id) continue; - - logger.info(`Follow[${linenum}] ${target.id} ...`); - - follow(user, target); - } catch (e) { - logger.warn(`Error in line:${linenum} ${e}`); - } - } - - logger.succ('Imported'); - done(); -} diff --git a/packages/backend/src/queue/processors/db/import-muting.ts b/packages/backend/src/queue/processors/db/import-muting.ts deleted file mode 100644 index 8552b797be..0000000000 --- a/packages/backend/src/queue/processors/db/import-muting.ts +++ /dev/null @@ -1,84 +0,0 @@ -import Bull from 'bull'; - -import { queueLogger } from '../../logger.js'; -import * as Acct from '@/misc/acct.js'; -import { resolveUser } from '@/remote/resolve-user.js'; -import { downloadTextFile } from '@/misc/download-text-file.js'; -import { isSelfHost, toPuny } from '@/misc/convert-host.js'; -import { Users, DriveFiles, Mutings } from '@/models/index.js'; -import { DbUserImportJobData } from '@/queue/types.js'; -import { User } from '@/models/entities/user.js'; -import { genId } from '@/misc/gen-id.js'; -import { IsNull } from 'typeorm'; - -const logger = queueLogger.createSubLogger('import-muting'); - -export async function importMuting(job: Bull.Job<DbUserImportJobData>, done: any): Promise<void> { - logger.info(`Importing muting of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - const file = await DriveFiles.findOneBy({ - id: job.data.fileId, - }); - if (file == null) { - done(); - return; - } - - const csv = await downloadTextFile(file.url); - - let linenum = 0; - - for (const line of csv.trim().split('\n')) { - linenum++; - - try { - const acct = line.split(',')[0].trim(); - const { username, host } = Acct.parse(acct); - - let target = isSelfHost(host!) ? await Users.findOneBy({ - host: IsNull(), - usernameLower: username.toLowerCase(), - }) : await Users.findOneBy({ - host: toPuny(host!), - usernameLower: username.toLowerCase(), - }); - - if (host == null && target == null) continue; - - if (target == null) { - target = await resolveUser(username, host); - } - - if (target == null) { - throw `cannot resolve user: @${username}@${host}`; - } - - // skip myself - if (target.id === job.data.user.id) continue; - - logger.info(`Mute[${linenum}] ${target.id} ...`); - - await mute(user, target); - } catch (e) { - logger.warn(`Error in line:${linenum} ${e}`); - } - } - - logger.succ('Imported'); - done(); -} - -async function mute(user: User, target: User) { - await Mutings.insert({ - id: genId(), - createdAt: new Date(), - muterId: user.id, - muteeId: target.id, - }); -} diff --git a/packages/backend/src/queue/processors/db/import-user-lists.ts b/packages/backend/src/queue/processors/db/import-user-lists.ts deleted file mode 100644 index 9919b7c53c..0000000000 --- a/packages/backend/src/queue/processors/db/import-user-lists.ts +++ /dev/null @@ -1,80 +0,0 @@ -import Bull from 'bull'; - -import { queueLogger } from '../../logger.js'; -import * as Acct from '@/misc/acct.js'; -import { resolveUser } from '@/remote/resolve-user.js'; -import { pushUserToUserList } from '@/services/user-list/push.js'; -import { downloadTextFile } from '@/misc/download-text-file.js'; -import { isSelfHost, toPuny } from '@/misc/convert-host.js'; -import { DriveFiles, Users, UserLists, UserListJoinings } from '@/models/index.js'; -import { genId } from '@/misc/gen-id.js'; -import { DbUserImportJobData } from '@/queue/types.js'; -import { IsNull } from 'typeorm'; - -const logger = queueLogger.createSubLogger('import-user-lists'); - -export async function importUserLists(job: Bull.Job<DbUserImportJobData>, done: any): Promise<void> { - logger.info(`Importing user lists of ${job.data.user.id} ...`); - - const user = await Users.findOneBy({ id: job.data.user.id }); - if (user == null) { - done(); - return; - } - - const file = await DriveFiles.findOneBy({ - id: job.data.fileId, - }); - if (file == null) { - done(); - return; - } - - const csv = await downloadTextFile(file.url); - - let linenum = 0; - - for (const line of csv.trim().split('\n')) { - linenum++; - - try { - const listName = line.split(',')[0].trim(); - const { username, host } = Acct.parse(line.split(',')[1].trim()); - - let list = await UserLists.findOneBy({ - userId: user.id, - name: listName, - }); - - if (list == null) { - list = await UserLists.insert({ - id: genId(), - createdAt: new Date(), - userId: user.id, - name: listName, - }).then(x => UserLists.findOneByOrFail(x.identifiers[0])); - } - - let target = isSelfHost(host!) ? await Users.findOneBy({ - host: IsNull(), - usernameLower: username.toLowerCase(), - }) : await Users.findOneBy({ - host: toPuny(host!), - usernameLower: username.toLowerCase(), - }); - - if (target == null) { - target = await resolveUser(username, host); - } - - if (await UserListJoinings.findOneBy({ userListId: list!.id, userId: target.id }) != null) continue; - - pushUserToUserList(target, list!); - } catch (e) { - logger.warn(`Error in line:${linenum} ${e}`); - } - } - - logger.succ('Imported'); - done(); -} diff --git a/packages/backend/src/queue/processors/db/index.ts b/packages/backend/src/queue/processors/db/index.ts deleted file mode 100644 index e91d569779..0000000000 --- a/packages/backend/src/queue/processors/db/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -import Bull from 'bull'; -import { DbJobData } from '@/queue/types.js'; -import { deleteDriveFiles } from './delete-drive-files.js'; -import { exportCustomEmojis } from './export-custom-emojis.js'; -import { exportNotes } from './export-notes.js'; -import { exportFollowing } from './export-following.js'; -import { exportMute } from './export-mute.js'; -import { exportBlocking } from './export-blocking.js'; -import { exportUserLists } from './export-user-lists.js'; -import { importFollowing } from './import-following.js'; -import { importUserLists } from './import-user-lists.js'; -import { deleteAccount } from './delete-account.js'; -import { importMuting } from './import-muting.js'; -import { importBlocking } from './import-blocking.js'; -import { importCustomEmojis } from './import-custom-emojis.js'; - -const jobs = { - deleteDriveFiles, - exportCustomEmojis, - exportNotes, - exportFollowing, - exportMute, - exportBlocking, - exportUserLists, - importFollowing, - importMuting, - importBlocking, - importUserLists, - importCustomEmojis, - deleteAccount, -} as Record<string, Bull.ProcessCallbackFunction<DbJobData> | Bull.ProcessPromiseFunction<DbJobData>>; - -export default function(dbQueue: Bull.Queue<DbJobData>) { - for (const [k, v] of Object.entries(jobs)) { - dbQueue.process(k, v); - } -} |