diff options
Diffstat (limited to 'packages/backend/src/queue/processors')
12 files changed, 60 insertions, 21 deletions
diff --git a/packages/backend/src/queue/processors/db/export-blocking.ts b/packages/backend/src/queue/processors/db/export-blocking.ts index 01edaaeb63..f4de9ce005 100644 --- a/packages/backend/src/queue/processors/db/export-blocking.ts +++ b/packages/backend/src/queue/processors/db/export-blocking.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import { queueLogger } from '../../logger'; import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; +import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host'; import { Users, Blockings } from '@/models/index'; import { MoreThan } from 'typeorm'; @@ -85,7 +85,7 @@ export async function exportBlocking(job: Bull.Job<DbUserJobData>, done: any): P stream.end(); logger.succ(`Exported to: ${path}`); - const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; + const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/export-custom-emojis.ts b/packages/backend/src/queue/processors/db/export-custom-emojis.ts index 240a542fec..2f7505f158 100644 --- a/packages/backend/src/queue/processors/db/export-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/export-custom-emojis.ts @@ -7,7 +7,7 @@ const mime = require('mime-types'); const archiver = require('archiver'); import { queueLogger } from '../../logger'; import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; +import { format as dateFormat } from 'date-fns'; import { Users, Emojis } from '@/models/index'; import { } from '@/queue/types'; import { downloadUrl } from '@/misc/download-url'; @@ -110,7 +110,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi archiveStream.on('close', async () => { logger.succ(`Exported to: ${archivePath}`); - const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.zip'; + const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; const driveFile = await addFile({ user, path: archivePath, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/export-following.ts b/packages/backend/src/queue/processors/db/export-following.ts index 06572acec1..2ac558381b 100644 --- a/packages/backend/src/queue/processors/db/export-following.ts +++ b/packages/backend/src/queue/processors/db/export-following.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import { queueLogger } from '../../logger'; import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; +import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host'; import { Users, Followings, Mutings } from '@/models/index'; import { In, MoreThan, Not } from 'typeorm'; @@ -86,7 +86,7 @@ export async function exportFollowing(job: Bull.Job<DbUserJobData>, done: () => stream.end(); logger.succ(`Exported to: ${path}`); - const fileName = 'following-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; + const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/export-mute.ts b/packages/backend/src/queue/processors/db/export-mute.ts index 4a856f8ef9..9e917ccbf3 100644 --- a/packages/backend/src/queue/processors/db/export-mute.ts +++ b/packages/backend/src/queue/processors/db/export-mute.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import { queueLogger } from '../../logger'; import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; +import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host'; import { Users, Mutings } from '@/models/index'; import { MoreThan } from 'typeorm'; @@ -85,7 +85,7 @@ export async function exportMute(job: Bull.Job<DbUserJobData>, done: any): Promi stream.end(); logger.succ(`Exported to: ${path}`); - const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; + const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts index 305abf44cf..0e65cb8849 100644 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ b/packages/backend/src/queue/processors/db/export-notes.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import { queueLogger } from '../../logger'; import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; +import { format as dateFormat } from 'date-fns'; import { Users, Notes, Polls } from '@/models/index'; import { MoreThan } from 'typeorm'; import { Note } from '@/models/entities/note'; @@ -94,7 +94,7 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom stream.end(); logger.succ(`Exported to: ${path}`); - const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json'; + const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/export-user-lists.ts b/packages/backend/src/queue/processors/db/export-user-lists.ts index f907cf9526..fcd2ba336c 100644 --- a/packages/backend/src/queue/processors/db/export-user-lists.ts +++ b/packages/backend/src/queue/processors/db/export-user-lists.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import { queueLogger } from '../../logger'; import { addFile } from '@/services/drive/add-file'; -import * as dateFormat from 'dateformat'; +import { format as dateFormat } from 'date-fns'; import { getFullApAccount } from '@/misc/convert-host'; import { Users, UserLists, UserListJoinings } from '@/models/index'; import { In } from 'typeorm'; @@ -62,7 +62,7 @@ export async function exportUserLists(job: Bull.Job<DbUserJobData>, done: any): stream.end(); logger.succ(`Exported to: ${path}`); - const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; + const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index 04e93671ed..b6c0126534 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -41,7 +41,9 @@ export async function importCustomEmojis(job: Bull.Job<DbUserImportJobData>, don fs.writeFileSync(destPath, '', 'binary'); await downloadUrl(file.url, destPath); } catch (e) { // TODO: 何度か再試行 - logger.error(e); + if (e instanceof Error || typeof e === 'string') { + logger.error(e); + } throw e; } diff --git a/packages/backend/src/queue/processors/db/import-user-lists.ts b/packages/backend/src/queue/processors/db/import-user-lists.ts index e060e86dd8..9b3c0ed60e 100644 --- a/packages/backend/src/queue/processors/db/import-user-lists.ts +++ b/packages/backend/src/queue/processors/db/import-user-lists.ts @@ -51,7 +51,6 @@ export async function importUserLists(job: Bull.Job<DbUserImportJobData>, done: createdAt: new Date(), userId: user.id, name: listName, - userIds: [], }).then(x => UserLists.findOneOrFail(x.identifiers[0])); } @@ -67,9 +66,9 @@ export async function importUserLists(job: Bull.Job<DbUserImportJobData>, done: target = await resolveUser(username, host); } - if (await UserListJoinings.findOne({ userListId: list.id, userId: target.id }) != null) continue; + if (await UserListJoinings.findOne({ userListId: list!.id, userId: target.id }) != null) continue; - pushUserToUserList(target, list); + pushUserToUserList(target, list!); } catch (e) { logger.warn(`Error in line:${linenum} ${e}`); } diff --git a/packages/backend/src/queue/processors/deliver.ts b/packages/backend/src/queue/processors/deliver.ts index 46aeb8cb7e..bd91dfc3b5 100644 --- a/packages/backend/src/queue/processors/deliver.ts +++ b/packages/backend/src/queue/processors/deliver.ts @@ -4,7 +4,7 @@ import request from '@/remote/activitypub/request'; import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc'; import Logger from '@/services/logger'; import { Instances } from '@/models/index'; -import { instanceChart } from '@/services/chart/index'; +import { apRequestChart, federationChart, instanceChart } from '@/services/chart/index'; import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata'; import { fetchMeta } from '@/misc/fetch-meta'; import { toPuny } from '@/misc/convert-host'; @@ -61,6 +61,8 @@ export default async (job: Bull.Job<DeliverJobData>) => { fetchInstanceMetadata(i); instanceChart.requestSent(i.host, true); + apRequestChart.deliverSucc(); + federationChart.deliverd(i.host, true); }); return 'Success'; @@ -74,6 +76,8 @@ export default async (job: Bull.Job<DeliverJobData>) => { }); instanceChart.requestSent(i.host, false); + apRequestChart.deliverFail(); + federationChart.deliverd(i.host, false); }); if (res instanceof StatusError) { diff --git a/packages/backend/src/queue/processors/inbox.ts b/packages/backend/src/queue/processors/inbox.ts index bfdebc0077..c189256c33 100644 --- a/packages/backend/src/queue/processors/inbox.ts +++ b/packages/backend/src/queue/processors/inbox.ts @@ -5,7 +5,7 @@ import perform from '@/remote/activitypub/perform'; import Logger from '@/services/logger'; import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc'; import { Instances } from '@/models/index'; -import { instanceChart } from '@/services/chart/index'; +import { apRequestChart, federationChart, instanceChart } from '@/services/chart/index'; import { fetchMeta } from '@/misc/fetch-meta'; import { toPuny, extractDbHost } from '@/misc/convert-host'; import { getApId } from '@/remote/activitypub/type'; @@ -54,10 +54,12 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => { authUser = await dbResolver.getAuthUserFromApId(getApId(activity.actor)); } catch (e) { // 対象が4xxならスキップ - if (e instanceof StatusError && e.isClientError) { - return `skip: Ignored deleted actors on both ends ${activity.actor} - ${e.statusCode}`; + if (e instanceof StatusError) { + if (e.isClientError) { + return `skip: Ignored deleted actors on both ends ${activity.actor} - ${e.statusCode}`; + } + throw `Error in actor ${activity.actor} - ${e.statusCode || e}`; } - throw `Error in actor ${activity.actor} - ${e.statusCode || e}`; } } @@ -141,6 +143,8 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => { fetchInstanceMetadata(i); instanceChart.requestReceived(i.host); + apRequestChart.inbox(); + federationChart.inbox(i.host); }); // アクティビティを処理 diff --git a/packages/backend/src/queue/processors/system/clean-charts.ts b/packages/backend/src/queue/processors/system/clean-charts.ts new file mode 100644 index 0000000000..3ae0f495f8 --- /dev/null +++ b/packages/backend/src/queue/processors/system/clean-charts.ts @@ -0,0 +1,28 @@ +import * as Bull from 'bull'; + +import { queueLogger } from '../../logger'; +import { activeUsersChart, driveChart, federationChart, hashtagChart, instanceChart, notesChart, perUserDriveChart, perUserFollowingChart, perUserNotesChart, perUserReactionsChart, usersChart, apRequestChart } from '@/services/chart/index'; + +const logger = queueLogger.createSubLogger('clean-charts'); + +export async function cleanCharts(job: Bull.Job<Record<string, unknown>>, done: any): Promise<void> { + logger.info(`Clean charts...`); + + await Promise.all([ + federationChart.clean(), + notesChart.clean(), + usersChart.clean(), + activeUsersChart.clean(), + instanceChart.clean(), + perUserNotesChart.clean(), + driveChart.clean(), + perUserReactionsChart.clean(), + hashtagChart.clean(), + perUserFollowingChart.clean(), + perUserDriveChart.clean(), + apRequestChart.clean(), + ]); + + logger.succ(`All charts successfully cleaned.`); + done(); +} diff --git a/packages/backend/src/queue/processors/system/index.ts b/packages/backend/src/queue/processors/system/index.ts index 8460ea0a9b..636fefc402 100644 --- a/packages/backend/src/queue/processors/system/index.ts +++ b/packages/backend/src/queue/processors/system/index.ts @@ -1,8 +1,10 @@ import * as Bull from 'bull'; import { resyncCharts } from './resync-charts'; +import { cleanCharts } from './clean-charts'; const jobs = { resyncCharts, + cleanCharts, } as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>> | Bull.ProcessPromiseFunction<Record<string, unknown>>>; export default function(dbQueue: Bull.Queue<Record<string, unknown>>) { |