From 0e4a111f81cceed275d9bec2695f6e401fb654d8 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 12 Nov 2021 02:02:25 +0900 Subject: refactoring Resolve #7779 --- .../services/chart/charts/classes/active-users.ts | 47 ++ .../src/services/chart/charts/classes/drive.ts | 91 ++++ .../services/chart/charts/classes/federation.ts | 62 +++ .../src/services/chart/charts/classes/hashtag.ts | 47 ++ .../src/services/chart/charts/classes/instance.ts | 217 ++++++++ .../src/services/chart/charts/classes/network.ts | 45 ++ .../src/services/chart/charts/classes/notes.ts | 97 ++++ .../chart/charts/classes/per-user-drive.ts | 64 +++ .../chart/charts/classes/per-user-following.ts | 121 +++++ .../chart/charts/classes/per-user-notes.ts | 72 +++ .../chart/charts/classes/per-user-reactions.ts | 44 ++ .../services/chart/charts/classes/test-grouped.ts | 58 +++ .../services/chart/charts/classes/test-unique.ts | 36 ++ .../src/services/chart/charts/classes/test.ts | 69 +++ .../src/services/chart/charts/classes/users.ts | 76 +++ .../services/chart/charts/schemas/active-users.ts | 35 ++ .../src/services/chart/charts/schemas/drive.ts | 68 +++ .../services/chart/charts/schemas/federation.ts | 29 ++ .../src/services/chart/charts/schemas/hashtag.ts | 35 ++ .../src/services/chart/charts/schemas/instance.ts | 157 ++++++ .../src/services/chart/charts/schemas/network.ts | 31 ++ .../src/services/chart/charts/schemas/notes.ts | 56 ++ .../chart/charts/schemas/per-user-drive.ts | 55 ++ .../chart/charts/schemas/per-user-following.ts | 86 ++++ .../chart/charts/schemas/per-user-notes.ts | 43 ++ .../chart/charts/schemas/per-user-reactions.ts | 31 ++ .../services/chart/charts/schemas/test-grouped.ts | 28 + .../services/chart/charts/schemas/test-unique.ts | 16 + .../src/services/chart/charts/schemas/test.ts | 28 + .../src/services/chart/charts/schemas/users.ts | 44 ++ packages/backend/src/services/chart/core.ts | 563 +++++++++++++++++++++ packages/backend/src/services/chart/entities.ts | 15 + packages/backend/src/services/chart/index.ts | 50 ++ 33 files changed, 2516 insertions(+) create mode 100644 packages/backend/src/services/chart/charts/classes/active-users.ts create mode 100644 packages/backend/src/services/chart/charts/classes/drive.ts create mode 100644 packages/backend/src/services/chart/charts/classes/federation.ts create mode 100644 packages/backend/src/services/chart/charts/classes/hashtag.ts create mode 100644 packages/backend/src/services/chart/charts/classes/instance.ts create mode 100644 packages/backend/src/services/chart/charts/classes/network.ts create mode 100644 packages/backend/src/services/chart/charts/classes/notes.ts create mode 100644 packages/backend/src/services/chart/charts/classes/per-user-drive.ts create mode 100644 packages/backend/src/services/chart/charts/classes/per-user-following.ts create mode 100644 packages/backend/src/services/chart/charts/classes/per-user-notes.ts create mode 100644 packages/backend/src/services/chart/charts/classes/per-user-reactions.ts create mode 100644 packages/backend/src/services/chart/charts/classes/test-grouped.ts create mode 100644 packages/backend/src/services/chart/charts/classes/test-unique.ts create mode 100644 packages/backend/src/services/chart/charts/classes/test.ts create mode 100644 packages/backend/src/services/chart/charts/classes/users.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/active-users.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/drive.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/federation.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/hashtag.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/instance.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/network.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/notes.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/per-user-drive.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/per-user-following.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/per-user-notes.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/per-user-reactions.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/test-grouped.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/test-unique.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/test.ts create mode 100644 packages/backend/src/services/chart/charts/schemas/users.ts create mode 100644 packages/backend/src/services/chart/core.ts create mode 100644 packages/backend/src/services/chart/entities.ts create mode 100644 packages/backend/src/services/chart/index.ts (limited to 'packages/backend/src/services/chart') diff --git a/packages/backend/src/services/chart/charts/classes/active-users.ts b/packages/backend/src/services/chart/charts/classes/active-users.ts new file mode 100644 index 0000000000..f80d8a3322 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/active-users.ts @@ -0,0 +1,47 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { User } from '@/models/entities/user'; +import { SchemaType } from '@/misc/schema'; +import { Users } from '@/models/index'; +import { name, schema } from '../schemas/active-users'; + +type ActiveUsersLog = SchemaType; + +export default class ActiveUsersChart extends Chart { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: ActiveUsersLog): DeepPartial { + return {}; + } + + @autobind + protected aggregate(logs: ActiveUsersLog[]): ActiveUsersLog { + return { + local: { + users: logs.reduce((a, b) => a.concat(b.local.users), [] as ActiveUsersLog['local']['users']), + }, + remote: { + users: logs.reduce((a, b) => a.concat(b.remote.users), [] as ActiveUsersLog['remote']['users']), + }, + }; + } + + @autobind + protected async fetchActual(): Promise> { + return {}; + } + + @autobind + public async update(user: { id: User['id'], host: User['host'] }) { + const update: Obj = { + users: [user.id] + }; + + await this.inc({ + [Users.isLocalUser(user) ? 'local' : 'remote']: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/drive.ts b/packages/backend/src/services/chart/charts/classes/drive.ts new file mode 100644 index 0000000000..93eabf3096 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/drive.ts @@ -0,0 +1,91 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { DriveFiles } from '@/models/index'; +import { Not, IsNull } from 'typeorm'; +import { DriveFile } from '@/models/entities/drive-file'; +import { name, schema } from '../schemas/drive'; + +type DriveLog = SchemaType; + +export default class DriveChart extends Chart { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: DriveLog): DeepPartial { + return { + local: { + totalCount: latest.local.totalCount, + totalSize: latest.local.totalSize, + }, + remote: { + totalCount: latest.remote.totalCount, + totalSize: latest.remote.totalSize, + } + }; + } + + @autobind + protected aggregate(logs: DriveLog[]): DriveLog { + return { + local: { + totalCount: logs[0].local.totalCount, + totalSize: logs[0].local.totalSize, + incCount: logs.reduce((a, b) => a + b.local.incCount, 0), + incSize: logs.reduce((a, b) => a + b.local.incSize, 0), + decCount: logs.reduce((a, b) => a + b.local.decCount, 0), + decSize: logs.reduce((a, b) => a + b.local.decSize, 0), + }, + remote: { + totalCount: logs[0].remote.totalCount, + totalSize: logs[0].remote.totalSize, + incCount: logs.reduce((a, b) => a + b.remote.incCount, 0), + incSize: logs.reduce((a, b) => a + b.remote.incSize, 0), + decCount: logs.reduce((a, b) => a + b.remote.decCount, 0), + decSize: logs.reduce((a, b) => a + b.remote.decSize, 0), + }, + }; + } + + @autobind + protected async fetchActual(): Promise> { + const [localCount, remoteCount, localSize, remoteSize] = await Promise.all([ + DriveFiles.count({ userHost: null }), + DriveFiles.count({ userHost: Not(IsNull()) }), + DriveFiles.calcDriveUsageOfLocal(), + DriveFiles.calcDriveUsageOfRemote() + ]); + + return { + local: { + totalCount: localCount, + totalSize: localSize, + }, + remote: { + totalCount: remoteCount, + totalSize: remoteSize, + } + }; + } + + @autobind + public async update(file: DriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalCount = isAdditional ? 1 : -1; + update.totalSize = isAdditional ? file.size : -file.size; + if (isAdditional) { + update.incCount = 1; + update.incSize = file.size; + } else { + update.decCount = 1; + update.decSize = file.size; + } + + await this.inc({ + [file.userHost === null ? 'local' : 'remote']: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/federation.ts b/packages/backend/src/services/chart/charts/classes/federation.ts new file mode 100644 index 0000000000..5f918b294f --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/federation.ts @@ -0,0 +1,62 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { Instances } from '@/models/index'; +import { name, schema } from '../schemas/federation'; + +type FederationLog = SchemaType; + +export default class FederationChart extends Chart { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: FederationLog): DeepPartial { + return { + instance: { + total: latest.instance.total, + } + }; + } + + @autobind + protected aggregate(logs: FederationLog[]): FederationLog { + return { + instance: { + total: logs[0].instance.total, + inc: logs.reduce((a, b) => a + b.instance.inc, 0), + dec: logs.reduce((a, b) => a + b.instance.dec, 0), + }, + }; + } + + @autobind + protected async fetchActual(): Promise> { + const [total] = await Promise.all([ + Instances.count({}) + ]); + + return { + instance: { + total: total, + } + }; + } + + @autobind + public async update(isAdditional: boolean) { + const update: Obj = {}; + + update.total = isAdditional ? 1 : -1; + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + await this.inc({ + instance: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/hashtag.ts b/packages/backend/src/services/chart/charts/classes/hashtag.ts new file mode 100644 index 0000000000..f7f5e17dec --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/hashtag.ts @@ -0,0 +1,47 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { User } from '@/models/entities/user'; +import { SchemaType } from '@/misc/schema'; +import { Users } from '@/models/index'; +import { name, schema } from '../schemas/hashtag'; + +type HashtagLog = SchemaType; + +export default class HashtagChart extends Chart { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: HashtagLog): DeepPartial { + return {}; + } + + @autobind + protected aggregate(logs: HashtagLog[]): HashtagLog { + return { + local: { + users: logs.reduce((a, b) => a.concat(b.local.users), [] as HashtagLog['local']['users']), + }, + remote: { + users: logs.reduce((a, b) => a.concat(b.remote.users), [] as HashtagLog['remote']['users']), + }, + }; + } + + @autobind + protected async fetchActual(): Promise> { + return {}; + } + + @autobind + public async update(hashtag: string, user: { id: User['id'], host: User['host'] }) { + const update: Obj = { + users: [user.id] + }; + + await this.inc({ + [Users.isLocalUser(user) ? 'local' : 'remote']: update + }, hashtag); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/instance.ts b/packages/backend/src/services/chart/charts/classes/instance.ts new file mode 100644 index 0000000000..1032de7bc0 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/instance.ts @@ -0,0 +1,217 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { DriveFiles, Followings, Users, Notes } from '@/models/index'; +import { DriveFile } from '@/models/entities/drive-file'; +import { name, schema } from '../schemas/instance'; +import { Note } from '@/models/entities/note'; +import { toPuny } from '@/misc/convert-host'; + +type InstanceLog = SchemaType; + +export default class InstanceChart extends Chart { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: InstanceLog): DeepPartial { + return { + notes: { + total: latest.notes.total, + }, + users: { + total: latest.users.total, + }, + following: { + total: latest.following.total, + }, + followers: { + total: latest.followers.total, + }, + drive: { + totalFiles: latest.drive.totalFiles, + totalUsage: latest.drive.totalUsage, + } + }; + } + + @autobind + protected aggregate(logs: InstanceLog[]): InstanceLog { + return { + requests: { + failed: logs.reduce((a, b) => a + b.requests.failed, 0), + succeeded: logs.reduce((a, b) => a + b.requests.succeeded, 0), + received: logs.reduce((a, b) => a + b.requests.received, 0), + }, + notes: { + total: logs[0].notes.total, + inc: logs.reduce((a, b) => a + b.notes.inc, 0), + dec: logs.reduce((a, b) => a + b.notes.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.notes.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.notes.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.notes.diffs.normal, 0), + }, + }, + users: { + total: logs[0].users.total, + inc: logs.reduce((a, b) => a + b.users.inc, 0), + dec: logs.reduce((a, b) => a + b.users.dec, 0), + }, + following: { + total: logs[0].following.total, + inc: logs.reduce((a, b) => a + b.following.inc, 0), + dec: logs.reduce((a, b) => a + b.following.dec, 0), + }, + followers: { + total: logs[0].followers.total, + inc: logs.reduce((a, b) => a + b.followers.inc, 0), + dec: logs.reduce((a, b) => a + b.followers.dec, 0), + }, + drive: { + totalFiles: logs[0].drive.totalFiles, + totalUsage: logs[0].drive.totalUsage, + incFiles: logs.reduce((a, b) => a + b.drive.incFiles, 0), + incUsage: logs.reduce((a, b) => a + b.drive.incUsage, 0), + decFiles: logs.reduce((a, b) => a + b.drive.decFiles, 0), + decUsage: logs.reduce((a, b) => a + b.drive.decUsage, 0), + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise> { + const [ + notesCount, + usersCount, + followingCount, + followersCount, + driveFiles, + driveUsage, + ] = await Promise.all([ + Notes.count({ userHost: group }), + Users.count({ host: group }), + Followings.count({ followerHost: group }), + Followings.count({ followeeHost: group }), + DriveFiles.count({ userHost: group }), + DriveFiles.calcDriveUsageOfHost(group), + ]); + + return { + notes: { + total: notesCount, + }, + users: { + total: usersCount, + }, + following: { + total: followingCount, + }, + followers: { + total: followersCount, + }, + drive: { + totalFiles: driveFiles, + totalUsage: driveUsage, + } + }; + } + + @autobind + public async requestReceived(host: string) { + await this.inc({ + requests: { + received: 1 + } + }, toPuny(host)); + } + + @autobind + public async requestSent(host: string, isSucceeded: boolean) { + const update: Obj = {}; + + if (isSucceeded) { + update.succeeded = 1; + } else { + update.failed = 1; + } + + await this.inc({ + requests: update + }, toPuny(host)); + } + + @autobind + public async newUser(host: string) { + await this.inc({ + users: { + total: 1, + inc: 1 + } + }, toPuny(host)); + } + + @autobind + public async updateNote(host: string, note: Note, isAdditional: boolean) { + const diffs = {} as any; + + if (note.replyId != null) { + diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + diffs.renote = isAdditional ? 1 : -1; + } else { + diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc({ + notes: { + total: isAdditional ? 1 : -1, + inc: isAdditional ? 1 : 0, + dec: isAdditional ? 0 : 1, + diffs: diffs + } + }, toPuny(host)); + } + + @autobind + public async updateFollowing(host: string, isAdditional: boolean) { + await this.inc({ + following: { + total: isAdditional ? 1 : -1, + inc: isAdditional ? 1 : 0, + dec: isAdditional ? 0 : 1, + } + }, toPuny(host)); + } + + @autobind + public async updateFollowers(host: string, isAdditional: boolean) { + await this.inc({ + followers: { + total: isAdditional ? 1 : -1, + inc: isAdditional ? 1 : 0, + dec: isAdditional ? 0 : 1, + } + }, toPuny(host)); + } + + @autobind + public async updateDrive(file: DriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalFiles = isAdditional ? 1 : -1; + update.totalUsage = isAdditional ? file.size : -file.size; + if (isAdditional) { + update.incFiles = 1; + update.incUsage = file.size; + } else { + update.decFiles = 1; + update.decUsage = file.size; + } + + await this.inc({ + drive: update + }, file.userHost); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/network.ts b/packages/backend/src/services/chart/charts/classes/network.ts new file mode 100644 index 0000000000..2ce75e0b34 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/network.ts @@ -0,0 +1,45 @@ +import autobind from 'autobind-decorator'; +import Chart, { DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { name, schema } from '../schemas/network'; + +type NetworkLog = SchemaType; + +export default class NetworkChart extends Chart { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: NetworkLog): DeepPartial { + return {}; + } + + @autobind + protected aggregate(logs: NetworkLog[]): NetworkLog { + return { + incomingRequests: logs.reduce((a, b) => a + b.incomingRequests, 0), + outgoingRequests: logs.reduce((a, b) => a + b.outgoingRequests, 0), + totalTime: logs.reduce((a, b) => a + b.totalTime, 0), + incomingBytes: logs.reduce((a, b) => a + b.incomingBytes, 0), + outgoingBytes: logs.reduce((a, b) => a + b.outgoingBytes, 0), + }; + } + + @autobind + protected async fetchActual(): Promise> { + return {}; + } + + @autobind + public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number) { + const inc: DeepPartial = { + incomingRequests: incomingRequests, + totalTime: time, + incomingBytes: incomingBytes, + outgoingBytes: outgoingBytes + }; + + await this.inc(inc); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/notes.ts b/packages/backend/src/services/chart/charts/classes/notes.ts new file mode 100644 index 0000000000..0675d346d1 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/notes.ts @@ -0,0 +1,97 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { Notes } from '@/models/index'; +import { Not, IsNull } from 'typeorm'; +import { Note } from '@/models/entities/note'; +import { name, schema } from '../schemas/notes'; + +type NotesLog = SchemaType; + +export default class NotesChart extends Chart { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: NotesLog): DeepPartial { + return { + local: { + total: latest.local.total, + }, + remote: { + total: latest.remote.total, + } + }; + } + + @autobind + protected aggregate(logs: NotesLog[]): NotesLog { + return { + local: { + total: logs[0].local.total, + inc: logs.reduce((a, b) => a + b.local.inc, 0), + dec: logs.reduce((a, b) => a + b.local.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.local.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.local.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.local.diffs.normal, 0), + }, + }, + remote: { + total: logs[0].remote.total, + inc: logs.reduce((a, b) => a + b.remote.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.remote.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.remote.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.remote.diffs.normal, 0), + }, + }, + }; + } + + @autobind + protected async fetchActual(): Promise> { + const [localCount, remoteCount] = await Promise.all([ + Notes.count({ userHost: null }), + Notes.count({ userHost: Not(IsNull()) }) + ]); + + return { + local: { + total: localCount, + }, + remote: { + total: remoteCount, + } + }; + } + + @autobind + public async update(note: Note, isAdditional: boolean) { + const update: Obj = { + diffs: {} + }; + + update.total = isAdditional ? 1 : -1; + + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + if (note.replyId != null) { + update.diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + update.diffs.renote = isAdditional ? 1 : -1; + } else { + update.diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc({ + [note.userHost === null ? 'local' : 'remote']: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/per-user-drive.ts b/packages/backend/src/services/chart/charts/classes/per-user-drive.ts new file mode 100644 index 0000000000..f28987191b --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/per-user-drive.ts @@ -0,0 +1,64 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { DriveFiles } from '@/models/index'; +import { DriveFile } from '@/models/entities/drive-file'; +import { name, schema } from '../schemas/per-user-drive'; + +type PerUserDriveLog = SchemaType; + +export default class PerUserDriveChart extends Chart { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserDriveLog): DeepPartial { + return { + totalCount: latest.totalCount, + totalSize: latest.totalSize, + }; + } + + @autobind + protected aggregate(logs: PerUserDriveLog[]): PerUserDriveLog { + return { + totalCount: logs[0].totalCount, + totalSize: logs[0].totalSize, + incCount: logs.reduce((a, b) => a + b.incCount, 0), + incSize: logs.reduce((a, b) => a + b.incSize, 0), + decCount: logs.reduce((a, b) => a + b.decCount, 0), + decSize: logs.reduce((a, b) => a + b.decSize, 0), + }; + } + + @autobind + protected async fetchActual(group: string): Promise> { + const [count, size] = await Promise.all([ + DriveFiles.count({ userId: group }), + DriveFiles.calcDriveUsageOf(group) + ]); + + return { + totalCount: count, + totalSize: size, + }; + } + + @autobind + public async update(file: DriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalCount = isAdditional ? 1 : -1; + update.totalSize = isAdditional ? file.size : -file.size; + if (isAdditional) { + update.incCount = 1; + update.incSize = file.size; + } else { + update.decCount = 1; + update.decSize = file.size; + } + + await this.inc(update, file.userId); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/per-user-following.ts b/packages/backend/src/services/chart/charts/classes/per-user-following.ts new file mode 100644 index 0000000000..08a9ad1d2b --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/per-user-following.ts @@ -0,0 +1,121 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { Followings, Users } from '@/models/index'; +import { Not, IsNull } from 'typeorm'; +import { User } from '@/models/entities/user'; +import { name, schema } from '../schemas/per-user-following'; + +type PerUserFollowingLog = SchemaType; + +export default class PerUserFollowingChart extends Chart { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserFollowingLog): DeepPartial { + return { + local: { + followings: { + total: latest.local.followings.total, + }, + followers: { + total: latest.local.followers.total, + } + }, + remote: { + followings: { + total: latest.remote.followings.total, + }, + followers: { + total: latest.remote.followers.total, + } + } + }; + } + + @autobind + protected aggregate(logs: PerUserFollowingLog[]): PerUserFollowingLog { + return { + local: { + followings: { + total: logs[0].local.followings.total, + inc: logs.reduce((a, b) => a + b.local.followings.inc, 0), + dec: logs.reduce((a, b) => a + b.local.followings.dec, 0), + }, + followers: { + total: logs[0].local.followers.total, + inc: logs.reduce((a, b) => a + b.local.followers.inc, 0), + dec: logs.reduce((a, b) => a + b.local.followers.dec, 0), + }, + }, + remote: { + followings: { + total: logs[0].remote.followings.total, + inc: logs.reduce((a, b) => a + b.remote.followings.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.followings.dec, 0), + }, + followers: { + total: logs[0].remote.followers.total, + inc: logs.reduce((a, b) => a + b.remote.followers.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.followers.dec, 0), + }, + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise> { + const [ + localFollowingsCount, + localFollowersCount, + remoteFollowingsCount, + remoteFollowersCount + ] = await Promise.all([ + Followings.count({ followerId: group, followeeHost: null }), + Followings.count({ followeeId: group, followerHost: null }), + Followings.count({ followerId: group, followeeHost: Not(IsNull()) }), + Followings.count({ followeeId: group, followerHost: Not(IsNull()) }) + ]); + + return { + local: { + followings: { + total: localFollowingsCount, + }, + followers: { + total: localFollowersCount, + } + }, + remote: { + followings: { + total: remoteFollowingsCount, + }, + followers: { + total: remoteFollowersCount, + } + } + }; + } + + @autobind + public async update(follower: { id: User['id']; host: User['host']; }, followee: { id: User['id']; host: User['host']; }, isFollow: boolean) { + const update: Obj = {}; + + update.total = isFollow ? 1 : -1; + + if (isFollow) { + update.inc = 1; + } else { + update.dec = 1; + } + + this.inc({ + [Users.isLocalUser(follower) ? 'local' : 'remote']: { followings: update } + }, follower.id); + this.inc({ + [Users.isLocalUser(followee) ? 'local' : 'remote']: { followers: update } + }, followee.id); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/per-user-notes.ts b/packages/backend/src/services/chart/charts/classes/per-user-notes.ts new file mode 100644 index 0000000000..0e808766f5 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/per-user-notes.ts @@ -0,0 +1,72 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { User } from '@/models/entities/user'; +import { SchemaType } from '@/misc/schema'; +import { Notes } from '@/models/index'; +import { Note } from '@/models/entities/note'; +import { name, schema } from '../schemas/per-user-notes'; + +type PerUserNotesLog = SchemaType; + +export default class PerUserNotesChart extends Chart { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserNotesLog): DeepPartial { + return { + total: latest.total, + }; + } + + @autobind + protected aggregate(logs: PerUserNotesLog[]): PerUserNotesLog { + return { + total: logs[0].total, + inc: logs.reduce((a, b) => a + b.inc, 0), + dec: logs.reduce((a, b) => a + b.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.diffs.normal, 0), + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise> { + const [count] = await Promise.all([ + Notes.count({ userId: group }), + ]); + + return { + total: count, + }; + } + + @autobind + public async update(user: { id: User['id'] }, note: Note, isAdditional: boolean) { + const update: Obj = { + diffs: {} + }; + + update.total = isAdditional ? 1 : -1; + + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + if (note.replyId != null) { + update.diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + update.diffs.renote = isAdditional ? 1 : -1; + } else { + update.diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc(update, user.id); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/per-user-reactions.ts b/packages/backend/src/services/chart/charts/classes/per-user-reactions.ts new file mode 100644 index 0000000000..e71bcb71c4 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/per-user-reactions.ts @@ -0,0 +1,44 @@ +import autobind from 'autobind-decorator'; +import Chart, { DeepPartial } from '../../core'; +import { User } from '@/models/entities/user'; +import { Note } from '@/models/entities/note'; +import { SchemaType } from '@/misc/schema'; +import { Users } from '@/models/index'; +import { name, schema } from '../schemas/per-user-reactions'; + +type PerUserReactionsLog = SchemaType; + +export default class PerUserReactionsChart extends Chart { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserReactionsLog): DeepPartial { + return {}; + } + + @autobind + protected aggregate(logs: PerUserReactionsLog[]): PerUserReactionsLog { + return { + local: { + count: logs.reduce((a, b) => a + b.local.count, 0), + }, + remote: { + count: logs.reduce((a, b) => a + b.remote.count, 0), + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise> { + return {}; + } + + @autobind + public async update(user: { id: User['id'], host: User['host'] }, note: Note) { + this.inc({ + [Users.isLocalUser(user) ? 'local' : 'remote']: { count: 1 } + }, note.userId); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/test-grouped.ts b/packages/backend/src/services/chart/charts/classes/test-grouped.ts new file mode 100644 index 0000000000..84e6d5e33f --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/test-grouped.ts @@ -0,0 +1,58 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { name, schema } from '../schemas/test-grouped'; + +type TestGroupedLog = SchemaType; + +export default class TestGroupedChart extends Chart { + private total = {} as Record; + + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: TestGroupedLog): DeepPartial { + return { + foo: { + total: latest.foo.total, + }, + }; + } + + @autobind + protected aggregate(logs: TestGroupedLog[]): TestGroupedLog { + return { + foo: { + total: logs[0].foo.total, + inc: logs.reduce((a, b) => a + b.foo.inc, 0), + dec: logs.reduce((a, b) => a + b.foo.dec, 0), + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise> { + return { + foo: { + total: this.total[group], + }, + }; + } + + @autobind + public async increment(group: string) { + if (this.total[group] == null) this.total[group] = 0; + + const update: Obj = {}; + + update.total = 1; + update.inc = 1; + this.total[group]++; + + await this.inc({ + foo: update + }, group); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/test-unique.ts b/packages/backend/src/services/chart/charts/classes/test-unique.ts new file mode 100644 index 0000000000..559fda13c9 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/test-unique.ts @@ -0,0 +1,36 @@ +import autobind from 'autobind-decorator'; +import Chart, { DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { name, schema } from '../schemas/test-unique'; + +type TestUniqueLog = SchemaType; + +export default class TestUniqueChart extends Chart { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: TestUniqueLog): DeepPartial { + return {}; + } + + @autobind + protected aggregate(logs: TestUniqueLog[]): TestUniqueLog { + return { + foo: logs.reduce((a, b) => a.concat(b.foo), [] as TestUniqueLog['foo']), + }; + } + + @autobind + protected async fetchActual(): Promise> { + return {}; + } + + @autobind + public async uniqueIncrement(key: string) { + await this.inc({ + foo: [key] + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/test.ts b/packages/backend/src/services/chart/charts/classes/test.ts new file mode 100644 index 0000000000..a91d5e1895 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/test.ts @@ -0,0 +1,69 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { name, schema } from '../schemas/test'; + +type TestLog = SchemaType; + +export default class TestChart extends Chart { + public total = 0; // publicにするのはテストのため + + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: TestLog): DeepPartial { + return { + foo: { + total: latest.foo.total, + }, + }; + } + + @autobind + protected aggregate(logs: TestLog[]): TestLog { + return { + foo: { + total: logs[0].foo.total, + inc: logs.reduce((a, b) => a + b.foo.inc, 0), + dec: logs.reduce((a, b) => a + b.foo.dec, 0), + }, + }; + } + + @autobind + protected async fetchActual(): Promise> { + return { + foo: { + total: this.total, + }, + }; + } + + @autobind + public async increment() { + const update: Obj = {}; + + update.total = 1; + update.inc = 1; + this.total++; + + await this.inc({ + foo: update + }); + } + + @autobind + public async decrement() { + const update: Obj = {}; + + update.total = -1; + update.dec = 1; + this.total--; + + await this.inc({ + foo: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/users.ts b/packages/backend/src/services/chart/charts/classes/users.ts new file mode 100644 index 0000000000..89b480ef77 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/users.ts @@ -0,0 +1,76 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { Users } from '@/models/index'; +import { Not, IsNull } from 'typeorm'; +import { User } from '@/models/entities/user'; +import { name, schema } from '../schemas/users'; + +type UsersLog = SchemaType; + +export default class UsersChart extends Chart { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: UsersLog): DeepPartial { + return { + local: { + total: latest.local.total, + }, + remote: { + total: latest.remote.total, + } + }; + } + + @autobind + protected aggregate(logs: UsersLog[]): UsersLog { + return { + local: { + total: logs[0].local.total, + inc: logs.reduce((a, b) => a + b.local.inc, 0), + dec: logs.reduce((a, b) => a + b.local.dec, 0), + }, + remote: { + total: logs[0].remote.total, + inc: logs.reduce((a, b) => a + b.remote.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.dec, 0), + }, + }; + } + + @autobind + protected async fetchActual(): Promise> { + const [localCount, remoteCount] = await Promise.all([ + Users.count({ host: null }), + Users.count({ host: Not(IsNull()) }) + ]); + + return { + local: { + total: localCount, + }, + remote: { + total: remoteCount, + } + }; + } + + @autobind + public async update(user: { id: User['id'], host: User['host'] }, isAdditional: boolean) { + const update: Obj = {}; + + update.total = isAdditional ? 1 : -1; + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + await this.inc({ + [Users.isLocalUser(user) ? 'local' : 'remote']: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/schemas/active-users.ts b/packages/backend/src/services/chart/charts/schemas/active-users.ts new file mode 100644 index 0000000000..1d65f280b0 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/active-users.ts @@ -0,0 +1,35 @@ +export const logSchema = { + /** + * アクティブユーザー + */ + users: { + type: 'array' as const, + optional: false as const, nullable: false as const, + items: { + type: 'string' as const, + optional: false as const, nullable: false as const, + } + }, +}; + +/** + * アクティブユーザーに関するチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'activeUsers'; diff --git a/packages/backend/src/services/chart/charts/schemas/drive.ts b/packages/backend/src/services/chart/charts/schemas/drive.ts new file mode 100644 index 0000000000..133b47846a --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/drive.ts @@ -0,0 +1,68 @@ +const logSchema = { + /** + * 集計期間時点での、全ドライブファイル数 + */ + totalCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 集計期間時点での、全ドライブファイルの合計サイズ + */ + totalSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 増加したドライブファイル数 + */ + incCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 増加したドライブ使用量 + */ + incSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 減少したドライブファイル数 + */ + decCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 減少したドライブ使用量 + */ + decSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, +}; + +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'drive'; diff --git a/packages/backend/src/services/chart/charts/schemas/federation.ts b/packages/backend/src/services/chart/charts/schemas/federation.ts new file mode 100644 index 0000000000..dca4587cac --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/federation.ts @@ -0,0 +1,29 @@ +/** + * フェデレーションに関するチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + instance: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + } + } +}; + +export const name = 'federation'; diff --git a/packages/backend/src/services/chart/charts/schemas/hashtag.ts b/packages/backend/src/services/chart/charts/schemas/hashtag.ts new file mode 100644 index 0000000000..4e7c542bbc --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/hashtag.ts @@ -0,0 +1,35 @@ +export const logSchema = { + /** + * 投稿したユーザー + */ + users: { + type: 'array' as const, + optional: false as const, nullable: false as const, + items: { + type: 'string' as const, + optional: false as const, nullable: false as const, + } + }, +}; + +/** + * ハッシュタグに関するチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'hashtag'; diff --git a/packages/backend/src/services/chart/charts/schemas/instance.ts b/packages/backend/src/services/chart/charts/schemas/instance.ts new file mode 100644 index 0000000000..785d6ae7ce --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/instance.ts @@ -0,0 +1,157 @@ +/** + * インスタンスごとのチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + requests: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + failed: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + succeeded: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + received: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + + notes: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + diffs: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + normal: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + reply: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + renote: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + } + }, + + users: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + + following: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + + followers: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + + drive: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + totalFiles: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + totalUsage: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + incFiles: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + incUsage: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + decFiles: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + decUsage: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + } +}; + +export const name = 'instance'; diff --git a/packages/backend/src/services/chart/charts/schemas/network.ts b/packages/backend/src/services/chart/charts/schemas/network.ts new file mode 100644 index 0000000000..49a364debc --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/network.ts @@ -0,0 +1,31 @@ +/** + * ネットワークに関するチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + incomingRequests: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + outgoingRequests: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + totalTime: { // TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + incomingBytes: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + outgoingBytes: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } +}; + +export const name = 'network'; diff --git a/packages/backend/src/services/chart/charts/schemas/notes.ts b/packages/backend/src/services/chart/charts/schemas/notes.ts new file mode 100644 index 0000000000..2b5105348c --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/notes.ts @@ -0,0 +1,56 @@ +const logSchema = { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + diffs: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + normal: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + reply: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + renote: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, +}; + +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'notes'; diff --git a/packages/backend/src/services/chart/charts/schemas/per-user-drive.ts b/packages/backend/src/services/chart/charts/schemas/per-user-drive.ts new file mode 100644 index 0000000000..856f1e0439 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/per-user-drive.ts @@ -0,0 +1,55 @@ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + /** + * 集計期間時点での、全ドライブファイル数 + */ + totalCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 集計期間時点での、全ドライブファイルの合計サイズ + */ + totalSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 増加したドライブファイル数 + */ + incCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 増加したドライブ使用量 + */ + incSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 減少したドライブファイル数 + */ + decCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 減少したドライブ使用量 + */ + decSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } +}; + +export const name = 'perUserDrive'; diff --git a/packages/backend/src/services/chart/charts/schemas/per-user-following.ts b/packages/backend/src/services/chart/charts/schemas/per-user-following.ts new file mode 100644 index 0000000000..eaf74aaf77 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/per-user-following.ts @@ -0,0 +1,86 @@ +export const logSchema = { + /** + * フォローしている + */ + followings: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + /** + * フォローしている合計 + */ + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * フォローした数 + */ + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * フォロー解除した数 + */ + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + + /** + * フォローされている + */ + followers: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + /** + * フォローされている合計 + */ + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * フォローされた数 + */ + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * フォロー解除された数 + */ + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, +}; + +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'perUserFollowing'; diff --git a/packages/backend/src/services/chart/charts/schemas/per-user-notes.ts b/packages/backend/src/services/chart/charts/schemas/per-user-notes.ts new file mode 100644 index 0000000000..72b3ff0210 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/per-user-notes.ts @@ -0,0 +1,43 @@ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + diffs: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + normal: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + reply: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + renote: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + } +}; + +export const name = 'perUserNotes'; diff --git a/packages/backend/src/services/chart/charts/schemas/per-user-reactions.ts b/packages/backend/src/services/chart/charts/schemas/per-user-reactions.ts new file mode 100644 index 0000000000..2a8520db37 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/per-user-reactions.ts @@ -0,0 +1,31 @@ +export const logSchema = { + /** + * フォローしている合計 + */ + count: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, +}; + +/** + * ユーザーごとのリアクションに関するチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'perUserReaction'; diff --git a/packages/backend/src/services/chart/charts/schemas/test-grouped.ts b/packages/backend/src/services/chart/charts/schemas/test-grouped.ts new file mode 100644 index 0000000000..f8c8250e79 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/test-grouped.ts @@ -0,0 +1,28 @@ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + foo: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + } + } +}; + +export const name = 'testGrouped'; diff --git a/packages/backend/src/services/chart/charts/schemas/test-unique.ts b/packages/backend/src/services/chart/charts/schemas/test-unique.ts new file mode 100644 index 0000000000..51280400ac --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/test-unique.ts @@ -0,0 +1,16 @@ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + foo: { + type: 'array' as const, + optional: false as const, nullable: false as const, + items: { + type: 'string' as const, + optional: false as const, nullable: false as const, + } + }, + } +}; + +export const name = 'testUnique'; diff --git a/packages/backend/src/services/chart/charts/schemas/test.ts b/packages/backend/src/services/chart/charts/schemas/test.ts new file mode 100644 index 0000000000..4b48d4d417 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/test.ts @@ -0,0 +1,28 @@ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + foo: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + } + } +}; + +export const name = 'test'; diff --git a/packages/backend/src/services/chart/charts/schemas/users.ts b/packages/backend/src/services/chart/charts/schemas/users.ts new file mode 100644 index 0000000000..2bf9d3c50f --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/users.ts @@ -0,0 +1,44 @@ +const logSchema = { + /** + * 集計期間時点での、全ユーザー数 + */ + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 増加したユーザー数 + */ + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 減少したユーザー数 + */ + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, +}; + +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'users'; diff --git a/packages/backend/src/services/chart/core.ts b/packages/backend/src/services/chart/core.ts new file mode 100644 index 0000000000..c0d3280c2b --- /dev/null +++ b/packages/backend/src/services/chart/core.ts @@ -0,0 +1,563 @@ +/** + * チャートエンジン + * + * Tests located in test/chart + */ + +import * as nestedProperty from 'nested-property'; +import autobind from 'autobind-decorator'; +import Logger from '../logger'; +import { SimpleSchema } from '@/misc/simple-schema'; +import { EntitySchema, getRepository, Repository, LessThan, Between } from 'typeorm'; +import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from '@/prelude/time'; +import { getChartInsertLock } from '@/misc/app-lock'; + +const logger = new Logger('chart', 'white', process.env.NODE_ENV !== 'test'); + +export type Obj = { [key: string]: any }; + +export type DeepPartial = { + [P in keyof T]?: DeepPartial; +}; + +type ArrayValue = { + [P in keyof T]: T[P] extends number ? T[P][] : ArrayValue; +}; + +type Log = { + id: number; + + /** + * 集計のグループ + */ + group: string | null; + + /** + * 集計日時のUnixタイムスタンプ(秒) + */ + date: number; +}; + +const camelToSnake = (str: string) => { + return str.replace(/([A-Z])/g, s => '_' + s.charAt(0).toLowerCase()); +}; + +const removeDuplicates = (array: any[]) => Array.from(new Set(array)); + +/** + * 様々なチャートの管理を司るクラス + */ +export default abstract class Chart> { + private static readonly columnPrefix = '___'; + private static readonly columnDot = '_'; + + private name: string; + private buffer: { + diff: DeepPartial; + group: string | null; + }[] = []; + public schema: SimpleSchema; + protected repository: Repository; + + protected abstract genNewLog(latest: T): DeepPartial; + + /** + * @param logs 日時が新しい方が先頭 + */ + protected abstract aggregate(logs: T[]): T; + + protected abstract fetchActual(group: string | null): Promise>; + + @autobind + private static convertSchemaToFlatColumnDefinitions(schema: SimpleSchema) { + const columns = {} as any; + const flatColumns = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}${this.columnDot}${k}` : k; + if (v.type === 'object') { + flatColumns(v.properties, p); + } else if (v.type === 'number') { + columns[this.columnPrefix + p] = { + type: 'bigint', + }; + } else if (v.type === 'array' && v.items.type === 'string') { + columns[this.columnPrefix + p] = { + type: 'varchar', + array: true, + }; + } + } + }; + flatColumns(schema.properties!); + return columns; + } + + @autobind + private static convertFlattenColumnsToObject(x: Record): Record { + const obj = {} as any; + for (const k of Object.keys(x).filter(k => k.startsWith(Chart.columnPrefix))) { + // now k is ___x_y_z + const path = k.substr(Chart.columnPrefix.length).split(Chart.columnDot).join('.'); + nestedProperty.set(obj, path, x[k]); + } + return obj; + } + + @autobind + private static convertObjectToFlattenColumns(x: Record) { + const columns = {} as Record; + const flatten = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}${this.columnDot}${k}` : k; + if (typeof v === 'object' && !Array.isArray(v)) { + flatten(v, p); + } else { + columns[this.columnPrefix + p] = v; + } + } + }; + flatten(x); + return columns; + } + + @autobind + private static countUniqueFields(x: Record) { + const exec = (x: Obj) => { + const res = {} as Record; + for (const [k, v] of Object.entries(x)) { + if (typeof v === 'object' && !Array.isArray(v)) { + res[k] = exec(v); + } else if (Array.isArray(v)) { + res[k] = Array.from(new Set(v)).length; + } else { + res[k] = v; + } + } + return res; + }; + return exec(x); + } + + @autobind + private static convertQuery(diff: Record) { + const query: Record = {}; + + for (const [k, v] of Object.entries(diff)) { + if (typeof v === 'number') { + if (v > 0) query[k] = () => `"${k}" + ${v}`; + if (v < 0) query[k] = () => `"${k}" - ${Math.abs(v)}`; + } else if (Array.isArray(v)) { + // TODO: item が文字列以外の場合も対応 + // TODO: item をSQLエスケープ + const items = v.map(item => `"${item}"`).join(','); + query[k] = () => `array_cat("${k}", '{${items}}'::varchar[])`; + } + } + + return query; + } + + @autobind + private static dateToTimestamp(x: Date): Log['date'] { + return Math.floor(x.getTime() / 1000); + } + + @autobind + private static parseDate(date: Date): [number, number, number, number, number, number, number] { + const y = date.getUTCFullYear(); + const m = date.getUTCMonth(); + const d = date.getUTCDate(); + const h = date.getUTCHours(); + const _m = date.getUTCMinutes(); + const _s = date.getUTCSeconds(); + const _ms = date.getUTCMilliseconds(); + + return [y, m, d, h, _m, _s, _ms]; + } + + @autobind + private static getCurrentDate() { + return Chart.parseDate(new Date()); + } + + @autobind + public static schemaToEntity(name: string, schema: SimpleSchema): EntitySchema { + return new EntitySchema({ + name: `__chart__${camelToSnake(name)}`, + columns: { + id: { + type: 'integer', + primary: true, + generated: true + }, + date: { + type: 'integer', + }, + group: { + type: 'varchar', + length: 128, + nullable: true + }, + ...Chart.convertSchemaToFlatColumnDefinitions(schema) + }, + indices: [{ + columns: ['date', 'group'], + unique: true, + }, { // groupにnullが含まれると↑のuniqueは機能しないので↓の部分インデックスでカバー + columns: ['date'], + unique: true, + where: '"group" IS NULL' + }] + }); + } + + constructor(name: string, schema: SimpleSchema, grouped = false) { + this.name = name; + this.schema = schema; + const entity = Chart.schemaToEntity(name, schema); + + const keys = ['date']; + if (grouped) keys.push('group'); + + entity.options.uniques = [{ + columns: keys + }]; + + this.repository = getRepository(entity); + } + + @autobind + private getNewLog(latest: T | null): T { + const log = latest ? this.genNewLog(latest) : {}; + const flatColumns = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}.${k}` : k; + if (v.type === 'object') { + flatColumns(v.properties, p); + } else { + if (nestedProperty.get(log, p) == null) { + const emptyValue = v.type === 'number' ? 0 : []; + nestedProperty.set(log, p, emptyValue); + } + } + } + }; + flatColumns(this.schema.properties!); + return log as T; + } + + @autobind + private getLatestLog(group: string | null = null): Promise { + return this.repository.findOne({ + group: group, + }, { + order: { + date: -1 + } + }).then(x => x || null); + } + + @autobind + private async getCurrentLog(group: string | null = null): Promise { + const [y, m, d, h] = Chart.getCurrentDate(); + + const current = dateUTC([y, m, d, h]); + + // 現在(=今のHour)のログ + const currentLog = await this.repository.findOne({ + date: Chart.dateToTimestamp(current), + ...(group ? { group: group } : {}) + }); + + // ログがあればそれを返して終了 + if (currentLog != null) { + return currentLog; + } + + let log: Log; + let data: T; + + // 集計期間が変わってから、初めてのチャート更新なら + // 最も最近のログを持ってくる + // * 例えば集計期間が「日」である場合で考えると、 + // * 昨日何もチャートを更新するような出来事がなかった場合は、 + // * ログがそもそも作られずドキュメントが存在しないということがあり得るため、 + // * 「昨日の」と決め打ちせずに「もっとも最近の」とします + const latest = await this.getLatestLog(group); + + if (latest != null) { + const obj = Chart.convertFlattenColumnsToObject(latest) as T; + + // 空ログデータを作成 + data = this.getNewLog(obj); + } else { + // ログが存在しなかったら + // (Misskeyインスタンスを建てて初めてのチャート更新時など) + + // 初期ログデータを作成 + data = this.getNewLog(null); + + logger.info(`${this.name + (group ? `:${group}` : '')}: Initial commit created`); + } + + const date = Chart.dateToTimestamp(current); + const lockKey = `${this.name}:${date}:${group}`; + + const unlock = await getChartInsertLock(lockKey); + try { + // ロック内でもう1回チェックする + const currentLog = await this.repository.findOne({ + date: date, + ...(group ? { group: group } : {}) + }); + + // ログがあればそれを返して終了 + if (currentLog != null) return currentLog; + + // 新規ログ挿入 + log = await this.repository.insert({ + group: group, + date: date, + ...Chart.convertObjectToFlattenColumns(data) + }).then(x => this.repository.findOneOrFail(x.identifiers[0])); + + logger.info(`${this.name + (group ? `:${group}` : '')}: New commit created`); + + return log; + } finally { + unlock(); + } + } + + @autobind + protected commit(diff: DeepPartial, group: string | null = null): void { + this.buffer.push({ + diff, group, + }); + } + + @autobind + public async save() { + if (this.buffer.length === 0) { + logger.info(`${this.name}: Write skipped`); + return; + } + + // TODO: 前の時間のログがbufferにあった場合のハンドリング + // 例えば、save が20分ごとに行われるとして、前回行われたのは 01:50 だったとする。 + // 次に save が行われるのは 02:10 ということになるが、もし 01:55 に新規ログが buffer に追加されたとすると、 + // そのログは本来は 01:00~ のログとしてDBに保存されて欲しいのに、02:00~ のログ扱いになってしまう。 + // これを回避するための実装は複雑になりそうなため、一旦保留。 + + const update = async (log: Log) => { + const finalDiffs = {} as Record; + + for (const diff of this.buffer.filter(q => q.group === log.group).map(q => q.diff)) { + const columns = Chart.convertObjectToFlattenColumns(diff); + + for (const [k, v] of Object.entries(columns)) { + if (finalDiffs[k] == null) { + finalDiffs[k] = v; + } else { + if (typeof finalDiffs[k] === 'number') { + (finalDiffs[k] as number) += v as number; + } else { + (finalDiffs[k] as unknown[]) = (finalDiffs[k] as unknown[]).concat(v); + } + } + } + } + + const query = Chart.convertQuery(finalDiffs); + + // ログ更新 + await this.repository.createQueryBuilder() + .update() + .set(query) + .where('id = :id', { id: log.id }) + .execute(); + + logger.info(`${this.name + (log.group ? `:${log.group}` : '')}: Updated`); + + // TODO: この一連の処理が始まった後に新たにbufferに入ったものは消さないようにする + this.buffer = this.buffer.filter(q => q.group !== log.group); + }; + + const groups = removeDuplicates(this.buffer.map(log => log.group)); + + await Promise.all(groups.map(group => this.getCurrentLog(group).then(log => update(log)))); + } + + @autobind + public async resync(group: string | null = null): Promise { + const data = await this.fetchActual(group); + + const update = async (log: Log) => { + await this.repository.createQueryBuilder() + .update() + .set(Chart.convertObjectToFlattenColumns(data)) + .where('id = :id', { id: log.id }) + .execute(); + }; + + return this.getCurrentLog(group).then(log => update(log)); + } + + @autobind + protected async inc(inc: DeepPartial, group: string | null = null): Promise { + await this.commit(inc, group); + } + + @autobind + public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise> { + const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate(); + const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [] as never; + + const lt = dateUTC([y, m, d, h, _m, _s, _ms]); + + const gt = + span === 'day' ? subtractTime(cursor ? dateUTC([y2, m2, d2, 0]) : dateUTC([y, m, d, 0]), amount - 1, 'day') : + span === 'hour' ? subtractTime(cursor ? dateUTC([y2, m2, d2, h2]) : dateUTC([y, m, d, h]), amount - 1, 'hour') : + null as never; + + // ログ取得 + let logs = await this.repository.find({ + where: { + group: group, + date: Between(Chart.dateToTimestamp(gt), Chart.dateToTimestamp(lt)) + }, + order: { + date: -1 + }, + }); + + // 要求された範囲にログがひとつもなかったら + if (logs.length === 0) { + // もっとも新しいログを持ってくる + // (すくなくともひとつログが無いと隙間埋めできないため) + const recentLog = await this.repository.findOne({ + group: group, + }, { + order: { + date: -1 + }, + }); + + if (recentLog) { + logs = [recentLog]; + } + + // 要求された範囲の最も古い箇所に位置するログが存在しなかったら + } else if (!isTimeSame(new Date(logs[logs.length - 1].date * 1000), gt)) { + // 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する + // (隙間埋めできないため) + const outdatedLog = await this.repository.findOne({ + group: group, + date: LessThan(Chart.dateToTimestamp(gt)) + }, { + order: { + date: -1 + }, + }); + + if (outdatedLog) { + logs.push(outdatedLog); + } + } + + const chart: T[] = []; + + if (span === 'hour') { + for (let i = (amount - 1); i >= 0; i--) { + const current = subtractTime(dateUTC([y, m, d, h]), i, 'hour'); + + const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current)); + + if (log) { + const data = Chart.convertFlattenColumnsToObject(log); + chart.unshift(Chart.countUniqueFields(data) as T); + } else { + // 隙間埋め + const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current)); + const data = latest ? Chart.convertFlattenColumnsToObject(latest) as T : null; + chart.unshift(Chart.countUniqueFields(this.getNewLog(data)) as T); + } + } + } else if (span === 'day') { + const logsForEachDays: T[][] = []; + let currentDay = -1; + let currentDayIndex = -1; + for (let i = ((amount - 1) * 24) + h; i >= 0; i--) { + const current = subtractTime(dateUTC([y, m, d, h]), i, 'hour'); + const _currentDay = Chart.parseDate(current)[2]; + if (currentDay != _currentDay) currentDayIndex++; + currentDay = _currentDay; + + const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current)); + + if (log) { + if (logsForEachDays[currentDayIndex]) { + logsForEachDays[currentDayIndex].unshift(Chart.convertFlattenColumnsToObject(log) as T); + } else { + logsForEachDays[currentDayIndex] = [Chart.convertFlattenColumnsToObject(log) as T]; + } + } else { + // 隙間埋め + const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current)); + const data = latest ? Chart.convertFlattenColumnsToObject(latest) as T : null; + const newLog = this.getNewLog(data); + if (logsForEachDays[currentDayIndex]) { + logsForEachDays[currentDayIndex].unshift(newLog); + } else { + logsForEachDays[currentDayIndex] = [newLog]; + } + } + } + + for (const logs of logsForEachDays) { + const log = this.aggregate(logs); + chart.unshift(Chart.countUniqueFields(log) as T); + } + } + + const res: ArrayValue = {} as any; + + /** + * [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }] + * を + * { foo: [1, 2, 3], bar: [5, 6, 7] } + * にする + */ + const compact = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}.${k}` : k; + if (typeof v === 'object' && !Array.isArray(v)) { + compact(v, p); + } else { + const values = chart.map(s => nestedProperty.get(s, p)); + nestedProperty.set(res, p, values); + } + } + }; + + compact(chart[0]); + + return res; + } +} + +export function convertLog(logSchema: SimpleSchema): SimpleSchema { + const v: SimpleSchema = JSON.parse(JSON.stringify(logSchema)); // copy + if (v.type === 'number') { + v.type = 'array'; + v.items = { + type: 'number' as const, + optional: false as const, nullable: false as const, + }; + } else if (v.type === 'object') { + for (const k of Object.keys(v.properties!)) { + v.properties![k] = convertLog(v.properties![k]); + } + } + return v; +} diff --git a/packages/backend/src/services/chart/entities.ts b/packages/backend/src/services/chart/entities.ts new file mode 100644 index 0000000000..23a97607eb --- /dev/null +++ b/packages/backend/src/services/chart/entities.ts @@ -0,0 +1,15 @@ +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; +import Chart from './core'; + +//const _filename = fileURLToPath(import.meta.url); +const _filename = __filename; +const _dirname = dirname(_filename); + +export const entities = Object.values(require('require-all')({ + dirname: _dirname + '/charts/schemas', + filter: /^.+\.[jt]s$/, + resolve: (x: any) => { + return Chart.schemaToEntity(x.name, x.schema); + } +})); diff --git a/packages/backend/src/services/chart/index.ts b/packages/backend/src/services/chart/index.ts new file mode 100644 index 0000000000..61eb431ea3 --- /dev/null +++ b/packages/backend/src/services/chart/index.ts @@ -0,0 +1,50 @@ +import FederationChart from './charts/classes/federation'; +import NotesChart from './charts/classes/notes'; +import UsersChart from './charts/classes/users'; +import NetworkChart from './charts/classes/network'; +import ActiveUsersChart from './charts/classes/active-users'; +import InstanceChart from './charts/classes/instance'; +import PerUserNotesChart from './charts/classes/per-user-notes'; +import DriveChart from './charts/classes/drive'; +import PerUserReactionsChart from './charts/classes/per-user-reactions'; +import HashtagChart from './charts/classes/hashtag'; +import PerUserFollowingChart from './charts/classes/per-user-following'; +import PerUserDriveChart from './charts/classes/per-user-drive'; +import { beforeShutdown } from '@/misc/before-shutdown'; + +export const federationChart = new FederationChart(); +export const notesChart = new NotesChart(); +export const usersChart = new UsersChart(); +export const networkChart = new NetworkChart(); +export const activeUsersChart = new ActiveUsersChart(); +export const instanceChart = new InstanceChart(); +export const perUserNotesChart = new PerUserNotesChart(); +export const driveChart = new DriveChart(); +export const perUserReactionsChart = new PerUserReactionsChart(); +export const hashtagChart = new HashtagChart(); +export const perUserFollowingChart = new PerUserFollowingChart(); +export const perUserDriveChart = new PerUserDriveChart(); + +const charts = [ + federationChart, + notesChart, + usersChart, + networkChart, + activeUsersChart, + instanceChart, + perUserNotesChart, + driveChart, + perUserReactionsChart, + hashtagChart, + perUserFollowingChart, + perUserDriveChart, +]; + +// 20分おきにメモリ情報をDBに書き込み +setInterval(() => { + for (const chart of charts) { + chart.save(); + } +}, 1000 * 60 * 20); + +beforeShutdown(() => Promise.all(charts.map(chart => chart.save()))); -- cgit v1.3.1-freya From b9eaf906e7b7202d06c9fea72b6d3c422a03f81e Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 12 Nov 2021 10:52:10 +0900 Subject: fix lint errors --- packages/backend/.eslintrc.js | 3 ++ packages/backend/@types/jsrsasign.d.ts | 2 +- packages/backend/src/mfm/from-html.ts | 3 +- packages/backend/src/misc/cafy-id.ts | 1 + packages/backend/src/misc/gen-avatar.ts | 2 +- packages/backend/src/prelude/array.ts | 2 +- packages/backend/src/prelude/url.ts | 2 +- .../object-storage/clean-remote-files.ts | 2 +- .../backend/src/queue/processors/system/index.ts | 4 +-- .../src/queue/processors/system/resync-charts.ts | 2 +- packages/backend/src/queue/queues.ts | 2 +- packages/backend/src/queue/types.ts | 2 +- .../src/remote/activitypub/models/person.ts | 2 +- .../activitypub/renderer/ordered-collection.ts | 2 +- packages/backend/src/server/api/define.ts | 6 ++-- .../src/server/api/endpoints/channels/update.ts | 2 +- .../src/server/api/endpoints/i/2fa/key-done.ts | 2 +- packages/backend/src/server/api/limiter.ts | 8 ++--- .../server/api/stream/channels/games/reversi.ts | 3 +- packages/backend/src/server/api/stream/types.ts | 2 +- packages/backend/src/services/chart/core.ts | 16 +++++----- packages/backend/src/services/stream.ts | 34 +++++++++++----------- 22 files changed, 55 insertions(+), 49 deletions(-) (limited to 'packages/backend/src/services/chart') diff --git a/packages/backend/.eslintrc.js b/packages/backend/.eslintrc.js index 952a2ee9ef..bafd0c9b63 100644 --- a/packages/backend/.eslintrc.js +++ b/packages/backend/.eslintrc.js @@ -22,6 +22,7 @@ module.exports = { 'eol-last': ['error', 'always'], 'semi': ['error', 'always'], 'quotes': ['warn', 'single'], + 'comma-dangle': ['warn', 'always-multiline'], 'keyword-spacing': ['error', { 'before': true, 'after': true, @@ -44,6 +45,8 @@ module.exports = { 'no-multi-spaces': ['warn'], 'no-control-regex': ['warn'], 'no-empty': ['warn'], + 'no-inner-declarations': ['off'], + 'no-sparse-arrays': ['off'], '@typescript-eslint/no-var-requires': ['warn'], '@typescript-eslint/no-inferrable-types': ['warn'], '@typescript-eslint/no-empty-function': ['off'], diff --git a/packages/backend/@types/jsrsasign.d.ts b/packages/backend/@types/jsrsasign.d.ts index bc9d746f7e..bb52f8f64e 100644 --- a/packages/backend/@types/jsrsasign.d.ts +++ b/packages/backend/@types/jsrsasign.d.ts @@ -171,7 +171,7 @@ declare module 'jsrsasign' { public static getTLVbyList(h: ASN1S, currentIndex: Idx, nthList: Mutable, checkingTag?: string): ASN1TLV; - // tslint:disable-next-line:bool-param-default + // eslint:disable-next-line:bool-param-default public static getVbyList(h: ASN1S, currentIndex: Idx, nthList: Mutable, checkingTag?: string, removeUnusedbits?: boolean): ASN1V; public static hextooidstr(hex: ASN1OIDV): OID; diff --git a/packages/backend/src/mfm/from-html.ts b/packages/backend/src/mfm/from-html.ts index de6aa3d0cc..43e16d80c5 100644 --- a/packages/backend/src/mfm/from-html.ts +++ b/packages/backend/src/mfm/from-html.ts @@ -48,9 +48,10 @@ export function fromHtml(html: string, hashtagNames?: string[]): string | null { if (!treeAdapter.isElementNode(node)) return; switch (node.nodeName) { - case 'br': + case 'br': { text += '\n'; break; + } case 'a': { diff --git a/packages/backend/src/misc/cafy-id.ts b/packages/backend/src/misc/cafy-id.ts index 39886611e1..dd81c5c4cf 100644 --- a/packages/backend/src/misc/cafy-id.ts +++ b/packages/backend/src/misc/cafy-id.ts @@ -1,5 +1,6 @@ import { Context } from 'cafy'; +// eslint-disable-next-line @typescript-eslint/ban-types export class ID extends Context { public readonly name = 'ID'; diff --git a/packages/backend/src/misc/gen-avatar.ts b/packages/backend/src/misc/gen-avatar.ts index f03ca9f96d..8838ec8d15 100644 --- a/packages/backend/src/misc/gen-avatar.ts +++ b/packages/backend/src/misc/gen-avatar.ts @@ -56,7 +56,7 @@ export function genAvatar(seed: string, stream: WriteStream): Promise { // 1*n (filled by false) const center: boolean[] = new Array(n).fill(false); - // tslint:disable-next-line:prefer-for-of + // eslint:disable-next-line:prefer-for-of for (let x = 0; x < side.length; x++) { for (let y = 0; y < side[x].length; y++) { side[x][y] = rand(3) === 0; diff --git a/packages/backend/src/prelude/array.ts b/packages/backend/src/prelude/array.ts index d63f0475d0..1e9e62b895 100644 --- a/packages/backend/src/prelude/array.ts +++ b/packages/backend/src/prelude/array.ts @@ -87,7 +87,7 @@ export function groupOn(f: (x: T) => S, xs: T[]): T[][] { export function groupByX(collections: T[], keySelector: (x: T) => string) { return collections.reduce((obj: Record, item: T) => { const key = keySelector(item); - if (!obj.hasOwnProperty(key)) { + if (!Object.prototype.hasOwnProperty.call(obj, key)) { obj[key] = []; } diff --git a/packages/backend/src/prelude/url.ts b/packages/backend/src/prelude/url.ts index c7f2b7c1e7..a4f2f7f5a8 100644 --- a/packages/backend/src/prelude/url.ts +++ b/packages/backend/src/prelude/url.ts @@ -1,4 +1,4 @@ -export function query(obj: {}): string { +export function query(obj: Record): string { const params = Object.entries(obj) .filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined) .reduce((a, [k, v]) => (a[k] = v, a), {} as Record); diff --git a/packages/backend/src/queue/processors/object-storage/clean-remote-files.ts b/packages/backend/src/queue/processors/object-storage/clean-remote-files.ts index 3b2e4ea939..a094c39d5d 100644 --- a/packages/backend/src/queue/processors/object-storage/clean-remote-files.ts +++ b/packages/backend/src/queue/processors/object-storage/clean-remote-files.ts @@ -7,7 +7,7 @@ import { MoreThan, Not, IsNull } from 'typeorm'; const logger = queueLogger.createSubLogger('clean-remote-files'); -export default async function cleanRemoteFiles(job: Bull.Job<{}>, done: any): Promise { +export default async function cleanRemoteFiles(job: Bull.Job>, done: any): Promise { logger.info(`Deleting cached remote files...`); let deletedCount = 0; diff --git a/packages/backend/src/queue/processors/system/index.ts b/packages/backend/src/queue/processors/system/index.ts index 52b7868105..8460ea0a9b 100644 --- a/packages/backend/src/queue/processors/system/index.ts +++ b/packages/backend/src/queue/processors/system/index.ts @@ -3,9 +3,9 @@ import { resyncCharts } from './resync-charts'; const jobs = { resyncCharts, -} as Record | Bull.ProcessPromiseFunction<{}>>; +} as Record> | Bull.ProcessPromiseFunction>>; -export default function(dbQueue: Bull.Queue<{}>) { +export default function(dbQueue: Bull.Queue>) { for (const [k, v] of Object.entries(jobs)) { dbQueue.process(k, v); } diff --git a/packages/backend/src/queue/processors/system/resync-charts.ts b/packages/backend/src/queue/processors/system/resync-charts.ts index b36b024cfb..78a70bb981 100644 --- a/packages/backend/src/queue/processors/system/resync-charts.ts +++ b/packages/backend/src/queue/processors/system/resync-charts.ts @@ -5,7 +5,7 @@ import { driveChart, notesChart, usersChart } from '@/services/chart/index'; const logger = queueLogger.createSubLogger('resync-charts'); -export default async function resyncCharts(job: Bull.Job<{}>, done: any): Promise { +export async function resyncCharts(job: Bull.Job>, done: any): Promise { logger.info(`Resync charts...`); // TODO: ユーザーごとのチャートも更新する diff --git a/packages/backend/src/queue/queues.ts b/packages/backend/src/queue/queues.ts index a66a7ca451..b1d790fcb1 100644 --- a/packages/backend/src/queue/queues.ts +++ b/packages/backend/src/queue/queues.ts @@ -2,7 +2,7 @@ import config from '@/config/index'; import { initialize as initializeQueue } from './initialize'; import { DeliverJobData, InboxJobData, DbJobData, ObjectStorageJobData } from './types'; -export const systemQueue = initializeQueue<{}>('system'); +export const systemQueue = initializeQueue>('system'); export const deliverQueue = initializeQueue('deliver', config.deliverJobPerSec || 128); export const inboxQueue = initializeQueue('inbox', config.inboxJobPerSec || 16); export const dbQueue = initializeQueue('db'); diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 39cab29966..c8c7147152 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -33,7 +33,7 @@ export type DbUserImportJobData = { fileId: DriveFile['id']; }; -export type ObjectStorageJobData = ObjectStorageFileJobData | {}; +export type ObjectStorageJobData = ObjectStorageFileJobData | Record; export type ObjectStorageFileJobData = { key: string; diff --git a/packages/backend/src/remote/activitypub/models/person.ts b/packages/backend/src/remote/activitypub/models/person.ts index eb8c00a10b..95db46bff2 100644 --- a/packages/backend/src/remote/activitypub/models/person.ts +++ b/packages/backend/src/remote/activitypub/models/person.ts @@ -274,7 +274,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise { +export async function updatePerson(uri: string, resolver?: Resolver | null, hint?: Record): Promise { if (typeof uri !== 'string') throw new Error('uri is not string'); // URIがこのサーバーを指しているならスキップ diff --git a/packages/backend/src/remote/activitypub/renderer/ordered-collection.ts b/packages/backend/src/remote/activitypub/renderer/ordered-collection.ts index 68870a0ecd..c4b4337af8 100644 --- a/packages/backend/src/remote/activitypub/renderer/ordered-collection.ts +++ b/packages/backend/src/remote/activitypub/renderer/ordered-collection.ts @@ -6,7 +6,7 @@ * @param last URL of last page (optional) * @param orderedItems attached objects (optional) */ -export default function(id: string | null, totalItems: any, first?: string, last?: string, orderedItems?: object) { +export default function(id: string | null, totalItems: any, first?: string, last?: string, orderedItems?: Record) { const page: any = { id, type: 'OrderedCollection', diff --git a/packages/backend/src/server/api/define.ts b/packages/backend/src/server/api/define.ts index 4bd8f95e31..48253e78e0 100644 --- a/packages/backend/src/server/api/define.ts +++ b/packages/backend/src/server/api/define.ts @@ -20,7 +20,7 @@ type SimpleUserInfo = { }; type Params = { - [P in keyof T['params']]: NonNullable[P]['transform'] extends Function + [P in keyof T['params']]: NonNullable[P]['transform'] extends () => any ? ReturnType[P]['transform']> : NonNullable[P]['default'] extends null | number | string ? NonOptional[P]['validator']['get']>[0]> @@ -30,7 +30,7 @@ type Params = { export type Response = Record | void; type executor = - (params: Params, user: T['requireCredential'] extends true ? SimpleUserInfo : SimpleUserInfo | null, token: AccessToken | null, file?: any, cleanup?: Function) => + (params: Params, user: T['requireCredential'] extends true ? SimpleUserInfo : SimpleUserInfo | null, token: AccessToken | null, file?: any, cleanup?: () => any) => Promise>>; export default function (meta: T, cb: executor) @@ -74,7 +74,7 @@ function getParams(defs: T, params: any): [Params, A }); return true; } else { - if (v === undefined && def.hasOwnProperty('default')) { + if (v === undefined && Object.prototype.hasOwnProperty.call(def, 'default')) { x[k] = def.default; } else { x[k] = v; diff --git a/packages/backend/src/server/api/endpoints/channels/update.ts b/packages/backend/src/server/api/endpoints/channels/update.ts index 9b447bd04b..05f279d6ac 100644 --- a/packages/backend/src/server/api/endpoints/channels/update.ts +++ b/packages/backend/src/server/api/endpoints/channels/update.ts @@ -69,7 +69,7 @@ export default define(meta, async (ps, me) => { throw new ApiError(meta.errors.accessDenied); } - // tslint:disable-next-line:no-unnecessary-initializer + // eslint:disable-next-line:no-unnecessary-initializer let banner = undefined; if (ps.bannerId != null) { banner = await DriveFiles.findOne({ diff --git a/packages/backend/src/server/api/endpoints/i/2fa/key-done.ts b/packages/backend/src/server/api/endpoints/i/2fa/key-done.ts index b4d3af235a..e06d0a9f68 100644 --- a/packages/backend/src/server/api/endpoints/i/2fa/key-done.ts +++ b/packages/backend/src/server/api/endpoints/i/2fa/key-done.ts @@ -75,7 +75,7 @@ export default define(meta, async (ps, user) => { const flags = attestation.authData[32]; - // tslint:disable-next-line:no-bitwise + // eslint:disable-next-line:no-bitwise if (!(flags & 1)) { throw new Error('user not present'); } diff --git a/packages/backend/src/server/api/limiter.ts b/packages/backend/src/server/api/limiter.ts index 1e2fe5bcb3..82a8613c90 100644 --- a/packages/backend/src/server/api/limiter.ts +++ b/packages/backend/src/server/api/limiter.ts @@ -10,16 +10,16 @@ const logger = new Logger('limiter'); export default (endpoint: IEndpoint, user: User) => new Promise((ok, reject) => { const limitation = endpoint.meta.limit!; - const key = limitation.hasOwnProperty('key') + const key = Object.prototype.hasOwnProperty.call(limitation, 'key') ? limitation.key : endpoint.name; const hasShortTermLimit = - limitation.hasOwnProperty('minInterval'); + Object.prototype.hasOwnProperty.call(limitation, 'minInterval'); const hasLongTermLimit = - limitation.hasOwnProperty('duration') && - limitation.hasOwnProperty('max'); + Object.prototype.hasOwnProperty.call(limitation, 'duration') && + Object.prototype.hasOwnProperty.call(limitation, 'max'); if (hasShortTermLimit) { min(); diff --git a/packages/backend/src/server/api/stream/channels/games/reversi.ts b/packages/backend/src/server/api/stream/channels/games/reversi.ts index 3b89aac35c..399750c26a 100644 --- a/packages/backend/src/server/api/stream/channels/games/reversi.ts +++ b/packages/backend/src/server/api/stream/channels/games/reversi.ts @@ -19,7 +19,7 @@ export default class extends Channel { @autobind public async onMessage(type: string, body: any) { switch (type) { - case 'ping': + case 'ping': { if (body.id == null) return; const matching = await ReversiMatchings.findOne({ parentId: this.user!.id, @@ -28,6 +28,7 @@ export default class extends Channel { if (matching == null) return; publishMainStream(matching.childId, 'reversiInvited', await ReversiMatchings.pack(matching, { id: matching.childId })); break; + } } } } diff --git a/packages/backend/src/server/api/stream/types.ts b/packages/backend/src/server/api/stream/types.ts index 70eb5c5ce5..f4302f64a0 100644 --- a/packages/backend/src/server/api/stream/types.ts +++ b/packages/backend/src/server/api/stream/types.ts @@ -31,7 +31,7 @@ export interface BroadcastTypes { } export interface UserStreamTypes { - terminate: {}; + terminate: Record; followChannel: Channel; unfollowChannel: Channel; updateUserProfile: UserProfile; diff --git a/packages/backend/src/services/chart/core.ts b/packages/backend/src/services/chart/core.ts index c0d3280c2b..78b7dd1359 100644 --- a/packages/backend/src/services/chart/core.ts +++ b/packages/backend/src/services/chart/core.ts @@ -70,7 +70,7 @@ export default abstract class Chart> { @autobind private static convertSchemaToFlatColumnDefinitions(schema: SimpleSchema) { - const columns = {} as any; + const columns = {} as Record; const flatColumns = (x: Obj, path?: string) => { for (const [k, v] of Object.entries(x)) { const p = path ? `${path}${this.columnDot}${k}` : k; @@ -93,8 +93,8 @@ export default abstract class Chart> { } @autobind - private static convertFlattenColumnsToObject(x: Record): Record { - const obj = {} as any; + private static convertFlattenColumnsToObject(x: Record): Record { + const obj = {} as Record; for (const k of Object.keys(x).filter(k => k.startsWith(Chart.columnPrefix))) { // now k is ___x_y_z const path = k.substr(Chart.columnPrefix.length).split(Chart.columnDot).join('.'); @@ -104,7 +104,7 @@ export default abstract class Chart> { } @autobind - private static convertObjectToFlattenColumns(x: Record) { + private static convertObjectToFlattenColumns(x: Record) { const columns = {} as Record; const flatten = (x: Obj, path?: string) => { for (const [k, v] of Object.entries(x)) { @@ -121,9 +121,9 @@ export default abstract class Chart> { } @autobind - private static countUniqueFields(x: Record) { + private static countUniqueFields(x: Record) { const exec = (x: Obj) => { - const res = {} as Record; + const res = {} as Record; for (const [k, v] of Object.entries(x)) { if (typeof v === 'object' && !Array.isArray(v)) { res[k] = exec(v); @@ -140,7 +140,7 @@ export default abstract class Chart> { @autobind private static convertQuery(diff: Record) { - const query: Record = {}; + const query: Record string> = {}; for (const [k, v] of Object.entries(diff)) { if (typeof v === 'number') { @@ -337,7 +337,7 @@ export default abstract class Chart> { } @autobind - public async save() { + public async save(): Promise { if (this.buffer.length === 0) { logger.info(`${this.name}: Write skipped`); return; diff --git a/packages/backend/src/services/stream.ts b/packages/backend/src/services/stream.ts index 2c308a1b54..0901857c33 100644 --- a/packages/backend/src/services/stream.ts +++ b/packages/backend/src/services/stream.ts @@ -37,74 +37,74 @@ class Publisher { channel: channel, message: message })); - } + }; public publishInternalEvent = (type: K, value?: InternalStreamTypes[K]): void => { this.publish('internal', type, typeof value === 'undefined' ? null : value); - } + }; public publishUserEvent = (userId: User['id'], type: K, value?: UserStreamTypes[K]): void => { this.publish(`user:${userId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishBroadcastStream = (type: K, value?: BroadcastTypes[K]): void => { this.publish('broadcast', type, typeof value === 'undefined' ? null : value); - } + }; public publishMainStream = (userId: User['id'], type: K, value?: MainStreamTypes[K]): void => { this.publish(`mainStream:${userId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishDriveStream = (userId: User['id'], type: K, value?: DriveStreamTypes[K]): void => { this.publish(`driveStream:${userId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishNoteStream = (noteId: Note['id'], type: K, value?: NoteStreamTypes[K]): void => { this.publish(`noteStream:${noteId}`, type, { id: noteId, body: value }); - } + }; public publishChannelStream = (channelId: Channel['id'], type: K, value?: ChannelStreamTypes[K]): void => { this.publish(`channelStream:${channelId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishUserListStream = (listId: UserList['id'], type: K, value?: UserListStreamTypes[K]): void => { this.publish(`userListStream:${listId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishAntennaStream = (antennaId: Antenna['id'], type: K, value?: AntennaStreamTypes[K]): void => { this.publish(`antennaStream:${antennaId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishMessagingStream = (userId: User['id'], otherpartyId: User['id'], type: K, value?: MessagingStreamTypes[K]): void => { this.publish(`messagingStream:${userId}-${otherpartyId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishGroupMessagingStream = (groupId: UserGroup['id'], type: K, value?: GroupMessagingStreamTypes[K]): void => { this.publish(`messagingStream:${groupId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishMessagingIndexStream = (userId: User['id'], type: K, value?: MessagingIndexStreamTypes[K]): void => { this.publish(`messagingIndexStream:${userId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishReversiStream = (userId: User['id'], type: K, value?: ReversiStreamTypes[K]): void => { this.publish(`reversiStream:${userId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishReversiGameStream = (gameId: ReversiGame['id'], type: K, value?: ReversiGameStreamTypes[K]): void => { this.publish(`reversiGameStream:${gameId}`, type, typeof value === 'undefined' ? null : value); - } + }; public publishNotesStream = (note: Packed<'Note'>): void => { this.publish('notesStream', null, note); - } + }; public publishAdminStream = (userId: User['id'], type: K, value?: AdminStreamTypes[K]): void => { this.publish(`adminStream:${userId}`, type, typeof value === 'undefined' ? null : value); - } + }; } const publisher = new Publisher(); -- cgit v1.3.1-freya