diff options
| author | tamaina <tamaina@hotmail.co.jp> | 2022-03-04 15:56:43 +0900 |
|---|---|---|
| committer | tamaina <tamaina@hotmail.co.jp> | 2022-03-04 15:56:43 +0900 |
| commit | a4de5aa94b9b318b1a286d36406e11e0e7f27590 (patch) | |
| tree | 6c0cd908b7e8f68fc300b9cb102f69b8ec2f5adc /packages/backend/src/queue/processors | |
| parent | Merge branch 'develop' into pizzax-indexeddb (diff) | |
| parent | fix federation chart pubsub (diff) | |
| download | misskey-a4de5aa94b9b318b1a286d36406e11e0e7f27590.tar.gz misskey-a4de5aa94b9b318b1a286d36406e11e0e7f27590.tar.bz2 misskey-a4de5aa94b9b318b1a286d36406e11e0e7f27590.zip | |
Merge branch 'develop' into pizzax-indexeddb
Diffstat (limited to 'packages/backend/src/queue/processors')
23 files changed, 262 insertions, 193 deletions
diff --git a/packages/backend/src/queue/processors/db/delete-account.ts b/packages/backend/src/queue/processors/db/delete-account.ts index 4b620842a4..dbc1f16a46 100644 --- a/packages/backend/src/queue/processors/db/delete-account.ts +++ b/packages/backend/src/queue/processors/db/delete-account.ts @@ -1,12 +1,12 @@ -import * as Bull from 'bull'; -import { queueLogger } from '../../logger'; -import { DriveFiles, Notes, UserProfiles, Users } from '@/models/index'; -import { DbUserDeleteJobData } from '@/queue/types'; -import { Note } from '@/models/entities/note'; -import { DriveFile } from '@/models/entities/drive-file'; +import Bull from 'bull'; +import { queueLogger } from '../../logger.js'; +import { DriveFiles, Notes, UserProfiles, Users } from '@/models/index.js'; +import { DbUserDeleteJobData } from '@/queue/types.js'; +import { Note } from '@/models/entities/note.js'; +import { DriveFile } from '@/models/entities/drive-file.js'; import { MoreThan } from 'typeorm'; -import { deleteFileSync } from '@/services/drive/delete-file'; -import { sendEmail } from '@/services/send-email'; +import { deleteFileSync } from '@/services/drive/delete-file.js'; +import { sendEmail } from '@/services/send-email.js'; const logger = queueLogger.createSubLogger('delete-account'); @@ -31,7 +31,7 @@ export async function deleteAccount(job: Bull.Job<DbUserDeleteJobData>): Promise order: { id: 1, }, - }); + }) as Note[]; if (notes.length === 0) { break; @@ -58,7 +58,7 @@ export async function deleteAccount(job: Bull.Job<DbUserDeleteJobData>): Promise order: { id: 1, }, - }); + }) as DriveFile[]; if (files.length === 0) { break; diff --git a/packages/backend/src/queue/processors/db/delete-drive-files.ts b/packages/backend/src/queue/processors/db/delete-drive-files.ts index d26f5a47c4..f6a8699855 100644 --- a/packages/backend/src/queue/processors/db/delete-drive-files.ts +++ b/packages/backend/src/queue/processors/db/delete-drive-files.ts @@ -1,10 +1,10 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; -import { queueLogger } from '../../logger'; -import { deleteFileSync } from '@/services/drive/delete-file'; -import { Users, DriveFiles } from '@/models/index'; +import { queueLogger } from '../../logger.js'; +import { deleteFileSync } from '@/services/drive/delete-file.js'; +import { Users, DriveFiles } from '@/models/index.js'; import { MoreThan } from 'typeorm'; -import { DbUserJobData } from '@/queue/types'; +import { DbUserJobData } from '@/queue/types.js'; const logger = queueLogger.createSubLogger('delete-drive-files'); diff --git a/packages/backend/src/queue/processors/db/export-blocking.ts b/packages/backend/src/queue/processors/db/export-blocking.ts index 01edaaeb63..83f1ec8fd6 100644 --- a/packages/backend/src/queue/processors/db/export-blocking.ts +++ b/packages/backend/src/queue/processors/db/export-blocking.ts @@ -1,14 +1,14 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; import * as tmp from 'tmp'; -import * as fs from 'fs'; +import * as fs from 'node:fs'; -import { queueLogger } from '../../logger'; -import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; -import { getFullApAccount } from '@/misc/convert-host'; -import { Users, Blockings } from '@/models/index'; +import { queueLogger } from '../../logger.js'; +import { addFile } from '@/services/drive/add-file.js'; +import { format as dateFormat } from 'date-fns'; +import { getFullApAccount } from '@/misc/convert-host.js'; +import { Users, Blockings } from '@/models/index.js'; import { MoreThan } from 'typeorm'; -import { DbUserJobData } from '@/queue/types'; +import { DbUserJobData } from '@/queue/types.js'; const logger = queueLogger.createSubLogger('export-blocking'); @@ -85,7 +85,7 @@ export async function exportBlocking(job: Bull.Job<DbUserJobData>, done: any): P stream.end(); logger.succ(`Exported to: ${path}`); - const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; + const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/export-custom-emojis.ts b/packages/backend/src/queue/processors/db/export-custom-emojis.ts index 240a542fec..a65b46cc00 100644 --- a/packages/backend/src/queue/processors/db/export-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/export-custom-emojis.ts @@ -1,17 +1,17 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; import * as tmp from 'tmp'; -import * as fs from 'fs'; +import * as fs from 'node:fs'; import { ulid } from 'ulid'; -const mime = require('mime-types'); -const archiver = require('archiver'); -import { queueLogger } from '../../logger'; -import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; -import { Users, Emojis } from '@/models/index'; -import { } from '@/queue/types'; -import { downloadUrl } from '@/misc/download-url'; -import config from '@/config/index'; +import mime from 'mime-types'; +import archiver from 'archiver'; +import { queueLogger } from '../../logger.js'; +import { addFile } from '@/services/drive/add-file.js'; +import { format as dateFormat } from 'date-fns'; +import { Users, Emojis } from '@/models/index.js'; +import { } from '@/queue/types.js'; +import { downloadUrl } from '@/misc/download-url.js'; +import config from '@/config/index.js'; const logger = queueLogger.createSubLogger('export-custom-emojis'); @@ -75,7 +75,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi await downloadUrl(emoji.originalUrl, emojiPath); downloaded = true; } catch (e) { // TODO: 何度か再試行 - logger.error(e); + logger.error(e instanceof Error ? e : new Error(e as string)); } if (!downloaded) { @@ -110,7 +110,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi archiveStream.on('close', async () => { logger.succ(`Exported to: ${archivePath}`); - const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.zip'; + const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; const driveFile = await addFile({ user, path: archivePath, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/export-following.ts b/packages/backend/src/queue/processors/db/export-following.ts index 06572acec1..162862180b 100644 --- a/packages/backend/src/queue/processors/db/export-following.ts +++ b/packages/backend/src/queue/processors/db/export-following.ts @@ -1,15 +1,15 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; import * as tmp from 'tmp'; -import * as fs from 'fs'; +import * as fs from 'node:fs'; -import { queueLogger } from '../../logger'; -import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; -import { getFullApAccount } from '@/misc/convert-host'; -import { Users, Followings, Mutings } from '@/models/index'; +import { queueLogger } from '../../logger.js'; +import { addFile } from '@/services/drive/add-file.js'; +import { format as dateFormat } from 'date-fns'; +import { getFullApAccount } from '@/misc/convert-host.js'; +import { Users, Followings, Mutings } from '@/models/index.js'; import { In, MoreThan, Not } from 'typeorm'; -import { DbUserJobData } from '@/queue/types'; -import { Following } from '@/models/entities/following'; +import { DbUserJobData } from '@/queue/types.js'; +import { Following } from '@/models/entities/following.js'; const logger = queueLogger.createSubLogger('export-following'); @@ -51,7 +51,7 @@ export async function exportFollowing(job: Bull.Job<DbUserJobData>, done: () => order: { id: 1, }, - }); + }) as Following[]; if (followings.length === 0) { break; @@ -86,7 +86,7 @@ export async function exportFollowing(job: Bull.Job<DbUserJobData>, done: () => stream.end(); logger.succ(`Exported to: ${path}`); - const fileName = 'following-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; + const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/export-mute.ts b/packages/backend/src/queue/processors/db/export-mute.ts index 4a856f8ef9..8602e00bff 100644 --- a/packages/backend/src/queue/processors/db/export-mute.ts +++ b/packages/backend/src/queue/processors/db/export-mute.ts @@ -1,14 +1,14 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; import * as tmp from 'tmp'; -import * as fs from 'fs'; +import * as fs from 'node:fs'; -import { queueLogger } from '../../logger'; -import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; -import { getFullApAccount } from '@/misc/convert-host'; -import { Users, Mutings } from '@/models/index'; +import { queueLogger } from '../../logger.js'; +import { addFile } from '@/services/drive/add-file.js'; +import { format as dateFormat } from 'date-fns'; +import { getFullApAccount } from '@/misc/convert-host.js'; +import { Users, Mutings } from '@/models/index.js'; import { MoreThan } from 'typeorm'; -import { DbUserJobData } from '@/queue/types'; +import { DbUserJobData } from '@/queue/types.js'; const logger = queueLogger.createSubLogger('export-mute'); @@ -85,7 +85,7 @@ export async function exportMute(job: Bull.Job<DbUserJobData>, done: any): Promi stream.end(); logger.succ(`Exported to: ${path}`); - const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; + const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts index 305abf44cf..c79679366c 100644 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ b/packages/backend/src/queue/processors/db/export-notes.ts @@ -1,15 +1,15 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; import * as tmp from 'tmp'; -import * as fs from 'fs'; +import * as fs from 'node:fs'; -import { queueLogger } from '../../logger'; -import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; -import { Users, Notes, Polls } from '@/models/index'; +import { queueLogger } from '../../logger.js'; +import { addFile } from '@/services/drive/add-file.js'; +import { format as dateFormat } from 'date-fns'; +import { Users, Notes, Polls } from '@/models/index.js'; import { MoreThan } from 'typeorm'; -import { Note } from '@/models/entities/note'; -import { Poll } from '@/models/entities/poll'; -import { DbUserJobData } from '@/queue/types'; +import { Note } from '@/models/entities/note.js'; +import { Poll } from '@/models/entities/poll.js'; +import { DbUserJobData } from '@/queue/types.js'; const logger = queueLogger.createSubLogger('export-notes'); @@ -62,7 +62,7 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom order: { id: 1, }, - }); + }) as Note[]; if (notes.length === 0) { job.progress(100); @@ -94,7 +94,7 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom stream.end(); logger.succ(`Exported to: ${path}`); - const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json'; + const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/export-user-lists.ts b/packages/backend/src/queue/processors/db/export-user-lists.ts index f907cf9526..1c04c36789 100644 --- a/packages/backend/src/queue/processors/db/export-user-lists.ts +++ b/packages/backend/src/queue/processors/db/export-user-lists.ts @@ -1,14 +1,14 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; import * as tmp from 'tmp'; -import * as fs from 'fs'; +import * as fs from 'node:fs'; -import { queueLogger } from '../../logger'; -import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; -import { getFullApAccount } from '@/misc/convert-host'; -import { Users, UserLists, UserListJoinings } from '@/models/index'; +import { queueLogger } from '../../logger.js'; +import { addFile } from '@/services/drive/add-file.js'; +import { format as dateFormat } from 'date-fns'; +import { getFullApAccount } from '@/misc/convert-host.js'; +import { Users, UserLists, UserListJoinings } from '@/models/index.js'; import { In } from 'typeorm'; -import { DbUserJobData } from '@/queue/types'; +import { DbUserJobData } from '@/queue/types.js'; const logger = queueLogger.createSubLogger('export-user-lists'); @@ -62,7 +62,7 @@ export async function exportUserLists(job: Bull.Job<DbUserJobData>, done: any): stream.end(); logger.succ(`Exported to: ${path}`); - const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; + const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/import-blocking.ts b/packages/backend/src/queue/processors/db/import-blocking.ts index 42c3cd0a40..857c2629e3 100644 --- a/packages/backend/src/queue/processors/db/import-blocking.ts +++ b/packages/backend/src/queue/processors/db/import-blocking.ts @@ -1,13 +1,13 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; -import { queueLogger } from '../../logger'; -import * as Acct from 'misskey-js/built/acct'; -import { resolveUser } from '@/remote/resolve-user'; -import { downloadTextFile } from '@/misc/download-text-file'; -import { isSelfHost, toPuny } from '@/misc/convert-host'; -import { Users, DriveFiles, Blockings } from '@/models/index'; -import { DbUserImportJobData } from '@/queue/types'; -import block from '@/services/blocking/create'; +import { queueLogger } from '../../logger.js'; +import * as Acct from '@/misc/acct.js'; +import { resolveUser } from '@/remote/resolve-user.js'; +import { downloadTextFile } from '@/misc/download-text-file.js'; +import { isSelfHost, toPuny } from '@/misc/convert-host.js'; +import { Users, DriveFiles, Blockings } from '@/models/index.js'; +import { DbUserImportJobData } from '@/queue/types.js'; +import block from '@/services/blocking/create.js'; const logger = queueLogger.createSubLogger('import-blocking'); diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index 04e93671ed..f862276b47 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -1,15 +1,15 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; import * as tmp from 'tmp'; -import * as fs from 'fs'; -const unzipper = require('unzipper'); +import * as fs from 'node:fs'; +import unzipper from 'unzipper'; import { getConnection } from 'typeorm'; -import { queueLogger } from '../../logger'; -import { downloadUrl } from '@/misc/download-url'; -import { DriveFiles, Emojis } from '@/models/index'; -import { DbUserImportJobData } from '@/queue/types'; -import { addFile } from '@/services/drive/add-file'; -import { genId } from '@/misc/gen-id'; +import { queueLogger } from '../../logger.js'; +import { downloadUrl } from '@/misc/download-url.js'; +import { DriveFiles, Emojis } from '@/models/index.js'; +import { DbUserImportJobData } from '@/queue/types.js'; +import { addFile } from '@/services/drive/add-file.js'; +import { genId } from '@/misc/gen-id.js'; const logger = queueLogger.createSubLogger('import-custom-emojis'); @@ -41,7 +41,9 @@ export async function importCustomEmojis(job: Bull.Job<DbUserImportJobData>, don fs.writeFileSync(destPath, '', 'binary'); await downloadUrl(file.url, destPath); } catch (e) { // TODO: 何度か再試行 - logger.error(e); + if (e instanceof Error || typeof e === 'string') { + logger.error(e); + } throw e; } diff --git a/packages/backend/src/queue/processors/db/import-following.ts b/packages/backend/src/queue/processors/db/import-following.ts index f19296832e..235fc28394 100644 --- a/packages/backend/src/queue/processors/db/import-following.ts +++ b/packages/backend/src/queue/processors/db/import-following.ts @@ -1,13 +1,13 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; -import { queueLogger } from '../../logger'; -import follow from '@/services/following/create'; -import * as Acct from 'misskey-js/built/acct'; -import { resolveUser } from '@/remote/resolve-user'; -import { downloadTextFile } from '@/misc/download-text-file'; -import { isSelfHost, toPuny } from '@/misc/convert-host'; -import { Users, DriveFiles } from '@/models/index'; -import { DbUserImportJobData } from '@/queue/types'; +import { queueLogger } from '../../logger.js'; +import follow from '@/services/following/create.js'; +import * as Acct from '@/misc/acct.js'; +import { resolveUser } from '@/remote/resolve-user.js'; +import { downloadTextFile } from '@/misc/download-text-file.js'; +import { isSelfHost, toPuny } from '@/misc/convert-host.js'; +import { Users, DriveFiles } from '@/models/index.js'; +import { DbUserImportJobData } from '@/queue/types.js'; const logger = queueLogger.createSubLogger('import-following'); diff --git a/packages/backend/src/queue/processors/db/import-muting.ts b/packages/backend/src/queue/processors/db/import-muting.ts index 189740c292..32f5f6bbee 100644 --- a/packages/backend/src/queue/processors/db/import-muting.ts +++ b/packages/backend/src/queue/processors/db/import-muting.ts @@ -1,14 +1,14 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; -import { queueLogger } from '../../logger'; -import * as Acct from 'misskey-js/built/acct'; -import { resolveUser } from '@/remote/resolve-user'; -import { downloadTextFile } from '@/misc/download-text-file'; -import { isSelfHost, toPuny } from '@/misc/convert-host'; -import { Users, DriveFiles, Mutings } from '@/models/index'; -import { DbUserImportJobData } from '@/queue/types'; -import { User } from '@/models/entities/user'; -import { genId } from '@/misc/gen-id'; +import { queueLogger } from '../../logger.js'; +import * as Acct from '@/misc/acct.js'; +import { resolveUser } from '@/remote/resolve-user.js'; +import { downloadTextFile } from '@/misc/download-text-file.js'; +import { isSelfHost, toPuny } from '@/misc/convert-host.js'; +import { Users, DriveFiles, Mutings } from '@/models/index.js'; +import { DbUserImportJobData } from '@/queue/types.js'; +import { User } from '@/models/entities/user.js'; +import { genId } from '@/misc/gen-id.js'; const logger = queueLogger.createSubLogger('import-muting'); diff --git a/packages/backend/src/queue/processors/db/import-user-lists.ts b/packages/backend/src/queue/processors/db/import-user-lists.ts index e060e86dd8..ae263e19b0 100644 --- a/packages/backend/src/queue/processors/db/import-user-lists.ts +++ b/packages/backend/src/queue/processors/db/import-user-lists.ts @@ -1,14 +1,14 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; -import { queueLogger } from '../../logger'; -import * as Acct from 'misskey-js/built/acct'; -import { resolveUser } from '@/remote/resolve-user'; -import { pushUserToUserList } from '@/services/user-list/push'; -import { downloadTextFile } from '@/misc/download-text-file'; -import { isSelfHost, toPuny } from '@/misc/convert-host'; -import { DriveFiles, Users, UserLists, UserListJoinings } from '@/models/index'; -import { genId } from '@/misc/gen-id'; -import { DbUserImportJobData } from '@/queue/types'; +import { queueLogger } from '../../logger.js'; +import * as Acct from '@/misc/acct.js'; +import { resolveUser } from '@/remote/resolve-user.js'; +import { pushUserToUserList } from '@/services/user-list/push.js'; +import { downloadTextFile } from '@/misc/download-text-file.js'; +import { isSelfHost, toPuny } from '@/misc/convert-host.js'; +import { DriveFiles, Users, UserLists, UserListJoinings } from '@/models/index.js'; +import { genId } from '@/misc/gen-id.js'; +import { DbUserImportJobData } from '@/queue/types.js'; const logger = queueLogger.createSubLogger('import-user-lists'); @@ -51,7 +51,6 @@ export async function importUserLists(job: Bull.Job<DbUserImportJobData>, done: createdAt: new Date(), userId: user.id, name: listName, - userIds: [], }).then(x => UserLists.findOneOrFail(x.identifiers[0])); } @@ -67,9 +66,9 @@ export async function importUserLists(job: Bull.Job<DbUserImportJobData>, done: target = await resolveUser(username, host); } - if (await UserListJoinings.findOne({ userListId: list.id, userId: target.id }) != null) continue; + if (await UserListJoinings.findOne({ userListId: list!.id, userId: target.id }) != null) continue; - pushUserToUserList(target, list); + pushUserToUserList(target, list!); } catch (e) { logger.warn(`Error in line:${linenum} ${e}`); } diff --git a/packages/backend/src/queue/processors/db/index.ts b/packages/backend/src/queue/processors/db/index.ts index 5fffa378f5..e91d569779 100644 --- a/packages/backend/src/queue/processors/db/index.ts +++ b/packages/backend/src/queue/processors/db/index.ts @@ -1,18 +1,18 @@ -import * as Bull from 'bull'; -import { DbJobData } from '@/queue/types'; -import { deleteDriveFiles } from './delete-drive-files'; -import { exportCustomEmojis } from './export-custom-emojis'; -import { exportNotes } from './export-notes'; -import { exportFollowing } from './export-following'; -import { exportMute } from './export-mute'; -import { exportBlocking } from './export-blocking'; -import { exportUserLists } from './export-user-lists'; -import { importFollowing } from './import-following'; -import { importUserLists } from './import-user-lists'; -import { deleteAccount } from './delete-account'; -import { importMuting } from './import-muting'; -import { importBlocking } from './import-blocking'; -import { importCustomEmojis } from './import-custom-emojis'; +import Bull from 'bull'; +import { DbJobData } from '@/queue/types.js'; +import { deleteDriveFiles } from './delete-drive-files.js'; +import { exportCustomEmojis } from './export-custom-emojis.js'; +import { exportNotes } from './export-notes.js'; +import { exportFollowing } from './export-following.js'; +import { exportMute } from './export-mute.js'; +import { exportBlocking } from './export-blocking.js'; +import { exportUserLists } from './export-user-lists.js'; +import { importFollowing } from './import-following.js'; +import { importUserLists } from './import-user-lists.js'; +import { deleteAccount } from './delete-account.js'; +import { importMuting } from './import-muting.js'; +import { importBlocking } from './import-blocking.js'; +import { importCustomEmojis } from './import-custom-emojis.js'; const jobs = { deleteDriveFiles, diff --git a/packages/backend/src/queue/processors/deliver.ts b/packages/backend/src/queue/processors/deliver.ts index 46aeb8cb7e..291c05766e 100644 --- a/packages/backend/src/queue/processors/deliver.ts +++ b/packages/backend/src/queue/processors/deliver.ts @@ -1,17 +1,17 @@ -import { URL } from 'url'; -import * as Bull from 'bull'; -import request from '@/remote/activitypub/request'; -import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc'; -import Logger from '@/services/logger'; -import { Instances } from '@/models/index'; -import { instanceChart } from '@/services/chart/index'; -import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata'; -import { fetchMeta } from '@/misc/fetch-meta'; -import { toPuny } from '@/misc/convert-host'; -import { Cache } from '@/misc/cache'; -import { Instance } from '@/models/entities/instance'; -import { DeliverJobData } from '../types'; -import { StatusError } from '@/misc/fetch'; +import { URL } from 'node:url'; +import Bull from 'bull'; +import request from '@/remote/activitypub/request.js'; +import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc.js'; +import Logger from '@/services/logger.js'; +import { Instances } from '@/models/index.js'; +import { apRequestChart, federationChart, instanceChart } from '@/services/chart/index.js'; +import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js'; +import { fetchMeta } from '@/misc/fetch-meta.js'; +import { toPuny } from '@/misc/convert-host.js'; +import { Cache } from '@/misc/cache.js'; +import { Instance } from '@/models/entities/instance.js'; +import { DeliverJobData } from '../types.js'; +import { StatusError } from '@/misc/fetch.js'; const logger = new Logger('deliver'); @@ -61,6 +61,8 @@ export default async (job: Bull.Job<DeliverJobData>) => { fetchInstanceMetadata(i); instanceChart.requestSent(i.host, true); + apRequestChart.deliverSucc(); + federationChart.deliverd(i.host, true); }); return 'Success'; @@ -74,6 +76,8 @@ export default async (job: Bull.Job<DeliverJobData>) => { }); instanceChart.requestSent(i.host, false); + apRequestChart.deliverFail(); + federationChart.deliverd(i.host, false); }); if (res instanceof StatusError) { diff --git a/packages/backend/src/queue/processors/inbox.ts b/packages/backend/src/queue/processors/inbox.ts index bfdebc0077..1b3f94b700 100644 --- a/packages/backend/src/queue/processors/inbox.ts +++ b/packages/backend/src/queue/processors/inbox.ts @@ -1,20 +1,20 @@ -import { URL } from 'url'; -import * as Bull from 'bull'; -import * as httpSignature from 'http-signature'; -import perform from '@/remote/activitypub/perform'; -import Logger from '@/services/logger'; -import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc'; -import { Instances } from '@/models/index'; -import { instanceChart } from '@/services/chart/index'; -import { fetchMeta } from '@/misc/fetch-meta'; -import { toPuny, extractDbHost } from '@/misc/convert-host'; -import { getApId } from '@/remote/activitypub/type'; -import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata'; -import { InboxJobData } from '../types'; -import DbResolver from '@/remote/activitypub/db-resolver'; -import { resolvePerson } from '@/remote/activitypub/models/person'; -import { LdSignature } from '@/remote/activitypub/misc/ld-signature'; -import { StatusError } from '@/misc/fetch'; +import { URL } from 'node:url'; +import Bull from 'bull'; +import httpSignature from 'http-signature'; +import perform from '@/remote/activitypub/perform.js'; +import Logger from '@/services/logger.js'; +import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc.js'; +import { Instances } from '@/models/index.js'; +import { apRequestChart, federationChart, instanceChart } from '@/services/chart/index.js'; +import { fetchMeta } from '@/misc/fetch-meta.js'; +import { toPuny, extractDbHost } from '@/misc/convert-host.js'; +import { getApId } from '@/remote/activitypub/type.js'; +import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js'; +import { InboxJobData } from '../types.js'; +import DbResolver from '@/remote/activitypub/db-resolver.js'; +import { resolvePerson } from '@/remote/activitypub/models/person.js'; +import { LdSignature } from '@/remote/activitypub/misc/ld-signature.js'; +import { StatusError } from '@/misc/fetch.js'; const logger = new Logger('inbox'); @@ -54,10 +54,12 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => { authUser = await dbResolver.getAuthUserFromApId(getApId(activity.actor)); } catch (e) { // 対象が4xxならスキップ - if (e instanceof StatusError && e.isClientError) { - return `skip: Ignored deleted actors on both ends ${activity.actor} - ${e.statusCode}`; + if (e instanceof StatusError) { + if (e.isClientError) { + return `skip: Ignored deleted actors on both ends ${activity.actor} - ${e.statusCode}`; + } + throw `Error in actor ${activity.actor} - ${e.statusCode || e}`; } - throw `Error in actor ${activity.actor} - ${e.statusCode || e}`; } } @@ -141,6 +143,8 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => { fetchInstanceMetadata(i); instanceChart.requestReceived(i.host); + apRequestChart.inbox(); + federationChart.inbox(i.host); }); // アクティビティを処理 diff --git a/packages/backend/src/queue/processors/object-storage/clean-remote-files.ts b/packages/backend/src/queue/processors/object-storage/clean-remote-files.ts index 788383a0a3..7d71a20adb 100644 --- a/packages/backend/src/queue/processors/object-storage/clean-remote-files.ts +++ b/packages/backend/src/queue/processors/object-storage/clean-remote-files.ts @@ -1,8 +1,8 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; -import { queueLogger } from '../../logger'; -import { deleteFileSync } from '@/services/drive/delete-file'; -import { DriveFiles } from '@/models/index'; +import { queueLogger } from '../../logger.js'; +import { deleteFileSync } from '@/services/drive/delete-file.js'; +import { DriveFiles } from '@/models/index.js'; import { MoreThan, Not, IsNull } from 'typeorm'; const logger = queueLogger.createSubLogger('clean-remote-files'); diff --git a/packages/backend/src/queue/processors/object-storage/delete-file.ts b/packages/backend/src/queue/processors/object-storage/delete-file.ts index ed22968a27..c271e3ddd4 100644 --- a/packages/backend/src/queue/processors/object-storage/delete-file.ts +++ b/packages/backend/src/queue/processors/object-storage/delete-file.ts @@ -1,6 +1,6 @@ -import { ObjectStorageFileJobData } from '@/queue/types'; -import * as Bull from 'bull'; -import { deleteObjectStorageFile } from '@/services/drive/delete-file'; +import { ObjectStorageFileJobData } from '@/queue/types.js'; +import Bull from 'bull'; +import { deleteObjectStorageFile } from '@/services/drive/delete-file.js'; export default async (job: Bull.Job<ObjectStorageFileJobData>) => { const key: string = job.data.key; diff --git a/packages/backend/src/queue/processors/object-storage/index.ts b/packages/backend/src/queue/processors/object-storage/index.ts index 0d9570e179..ae6c481fea 100644 --- a/packages/backend/src/queue/processors/object-storage/index.ts +++ b/packages/backend/src/queue/processors/object-storage/index.ts @@ -1,7 +1,7 @@ -import * as Bull from 'bull'; -import { ObjectStorageJobData } from '@/queue/types'; -import deleteFile from './delete-file'; -import cleanRemoteFiles from './clean-remote-files'; +import Bull from 'bull'; +import { ObjectStorageJobData } from '@/queue/types.js'; +import deleteFile from './delete-file.js'; +import cleanRemoteFiles from './clean-remote-files.js'; const jobs = { deleteFile, diff --git a/packages/backend/src/queue/processors/system/clean-charts.ts b/packages/backend/src/queue/processors/system/clean-charts.ts new file mode 100644 index 0000000000..c9169d5acf --- /dev/null +++ b/packages/backend/src/queue/processors/system/clean-charts.ts @@ -0,0 +1,28 @@ +import Bull from 'bull'; + +import { queueLogger } from '../../logger.js'; +import { activeUsersChart, driveChart, federationChart, hashtagChart, instanceChart, notesChart, perUserDriveChart, perUserFollowingChart, perUserNotesChart, perUserReactionsChart, usersChart, apRequestChart } from '@/services/chart/index.js'; + +const logger = queueLogger.createSubLogger('clean-charts'); + +export async function cleanCharts(job: Bull.Job<Record<string, unknown>>, done: any): Promise<void> { + logger.info(`Clean charts...`); + + await Promise.all([ + federationChart.clean(), + notesChart.clean(), + usersChart.clean(), + activeUsersChart.clean(), + instanceChart.clean(), + perUserNotesChart.clean(), + driveChart.clean(), + perUserReactionsChart.clean(), + hashtagChart.clean(), + perUserFollowingChart.clean(), + perUserDriveChart.clean(), + apRequestChart.clean(), + ]); + + logger.succ(`All charts successfully cleaned.`); + done(); +} diff --git a/packages/backend/src/queue/processors/system/index.ts b/packages/backend/src/queue/processors/system/index.ts index 8460ea0a9b..dca3249e82 100644 --- a/packages/backend/src/queue/processors/system/index.ts +++ b/packages/backend/src/queue/processors/system/index.ts @@ -1,8 +1,12 @@ -import * as Bull from 'bull'; -import { resyncCharts } from './resync-charts'; +import Bull from 'bull'; +import { tickCharts } from './tick-charts.js'; +import { resyncCharts } from './resync-charts.js'; +import { cleanCharts } from './clean-charts.js'; const jobs = { + tickCharts, resyncCharts, + cleanCharts, } as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>> | Bull.ProcessPromiseFunction<Record<string, unknown>>>; export default function(dbQueue: Bull.Queue<Record<string, unknown>>) { diff --git a/packages/backend/src/queue/processors/system/resync-charts.ts b/packages/backend/src/queue/processors/system/resync-charts.ts index 78a70bb981..20012513af 100644 --- a/packages/backend/src/queue/processors/system/resync-charts.ts +++ b/packages/backend/src/queue/processors/system/resync-charts.ts @@ -1,7 +1,7 @@ -import * as Bull from 'bull'; +import Bull from 'bull'; -import { queueLogger } from '../../logger'; -import { driveChart, notesChart, usersChart } from '@/services/chart/index'; +import { queueLogger } from '../../logger.js'; +import { driveChart, notesChart, usersChart } from '@/services/chart/index.js'; const logger = queueLogger.createSubLogger('resync-charts'); diff --git a/packages/backend/src/queue/processors/system/tick-charts.ts b/packages/backend/src/queue/processors/system/tick-charts.ts new file mode 100644 index 0000000000..13403f8f73 --- /dev/null +++ b/packages/backend/src/queue/processors/system/tick-charts.ts @@ -0,0 +1,28 @@ +import Bull from 'bull'; + +import { queueLogger } from '../../logger.js'; +import { activeUsersChart, driveChart, federationChart, hashtagChart, instanceChart, notesChart, perUserDriveChart, perUserFollowingChart, perUserNotesChart, perUserReactionsChart, usersChart, apRequestChart } from '@/services/chart/index.js'; + +const logger = queueLogger.createSubLogger('tick-charts'); + +export async function tickCharts(job: Bull.Job<Record<string, unknown>>, done: any): Promise<void> { + logger.info(`Tick charts...`); + + await Promise.all([ + federationChart.tick(false), + notesChart.tick(false), + usersChart.tick(false), + activeUsersChart.tick(false), + instanceChart.tick(false), + perUserNotesChart.tick(false), + driveChart.tick(false), + perUserReactionsChart.tick(false), + hashtagChart.tick(false), + perUserFollowingChart.tick(false), + perUserDriveChart.tick(false), + apRequestChart.tick(false), + ]); + + logger.succ(`All charts successfully ticked.`); + done(); +} |