diff options
Diffstat (limited to 'packages/backend/src/services/chart')
33 files changed, 2516 insertions, 0 deletions
diff --git a/packages/backend/src/services/chart/charts/classes/active-users.ts b/packages/backend/src/services/chart/charts/classes/active-users.ts new file mode 100644 index 0000000000..f80d8a3322 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/active-users.ts @@ -0,0 +1,47 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { User } from '@/models/entities/user'; +import { SchemaType } from '@/misc/schema'; +import { Users } from '@/models/index'; +import { name, schema } from '../schemas/active-users'; + +type ActiveUsersLog = SchemaType<typeof schema>; + +export default class ActiveUsersChart extends Chart<ActiveUsersLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: ActiveUsersLog): DeepPartial<ActiveUsersLog> { + return {}; + } + + @autobind + protected aggregate(logs: ActiveUsersLog[]): ActiveUsersLog { + return { + local: { + users: logs.reduce((a, b) => a.concat(b.local.users), [] as ActiveUsersLog['local']['users']), + }, + remote: { + users: logs.reduce((a, b) => a.concat(b.remote.users), [] as ActiveUsersLog['remote']['users']), + }, + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<ActiveUsersLog>> { + return {}; + } + + @autobind + public async update(user: { id: User['id'], host: User['host'] }) { + const update: Obj = { + users: [user.id] + }; + + await this.inc({ + [Users.isLocalUser(user) ? 'local' : 'remote']: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/drive.ts b/packages/backend/src/services/chart/charts/classes/drive.ts new file mode 100644 index 0000000000..93eabf3096 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/drive.ts @@ -0,0 +1,91 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { DriveFiles } from '@/models/index'; +import { Not, IsNull } from 'typeorm'; +import { DriveFile } from '@/models/entities/drive-file'; +import { name, schema } from '../schemas/drive'; + +type DriveLog = SchemaType<typeof schema>; + +export default class DriveChart extends Chart<DriveLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: DriveLog): DeepPartial<DriveLog> { + return { + local: { + totalCount: latest.local.totalCount, + totalSize: latest.local.totalSize, + }, + remote: { + totalCount: latest.remote.totalCount, + totalSize: latest.remote.totalSize, + } + }; + } + + @autobind + protected aggregate(logs: DriveLog[]): DriveLog { + return { + local: { + totalCount: logs[0].local.totalCount, + totalSize: logs[0].local.totalSize, + incCount: logs.reduce((a, b) => a + b.local.incCount, 0), + incSize: logs.reduce((a, b) => a + b.local.incSize, 0), + decCount: logs.reduce((a, b) => a + b.local.decCount, 0), + decSize: logs.reduce((a, b) => a + b.local.decSize, 0), + }, + remote: { + totalCount: logs[0].remote.totalCount, + totalSize: logs[0].remote.totalSize, + incCount: logs.reduce((a, b) => a + b.remote.incCount, 0), + incSize: logs.reduce((a, b) => a + b.remote.incSize, 0), + decCount: logs.reduce((a, b) => a + b.remote.decCount, 0), + decSize: logs.reduce((a, b) => a + b.remote.decSize, 0), + }, + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<DriveLog>> { + const [localCount, remoteCount, localSize, remoteSize] = await Promise.all([ + DriveFiles.count({ userHost: null }), + DriveFiles.count({ userHost: Not(IsNull()) }), + DriveFiles.calcDriveUsageOfLocal(), + DriveFiles.calcDriveUsageOfRemote() + ]); + + return { + local: { + totalCount: localCount, + totalSize: localSize, + }, + remote: { + totalCount: remoteCount, + totalSize: remoteSize, + } + }; + } + + @autobind + public async update(file: DriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalCount = isAdditional ? 1 : -1; + update.totalSize = isAdditional ? file.size : -file.size; + if (isAdditional) { + update.incCount = 1; + update.incSize = file.size; + } else { + update.decCount = 1; + update.decSize = file.size; + } + + await this.inc({ + [file.userHost === null ? 'local' : 'remote']: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/federation.ts b/packages/backend/src/services/chart/charts/classes/federation.ts new file mode 100644 index 0000000000..5f918b294f --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/federation.ts @@ -0,0 +1,62 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { Instances } from '@/models/index'; +import { name, schema } from '../schemas/federation'; + +type FederationLog = SchemaType<typeof schema>; + +export default class FederationChart extends Chart<FederationLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: FederationLog): DeepPartial<FederationLog> { + return { + instance: { + total: latest.instance.total, + } + }; + } + + @autobind + protected aggregate(logs: FederationLog[]): FederationLog { + return { + instance: { + total: logs[0].instance.total, + inc: logs.reduce((a, b) => a + b.instance.inc, 0), + dec: logs.reduce((a, b) => a + b.instance.dec, 0), + }, + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<FederationLog>> { + const [total] = await Promise.all([ + Instances.count({}) + ]); + + return { + instance: { + total: total, + } + }; + } + + @autobind + public async update(isAdditional: boolean) { + const update: Obj = {}; + + update.total = isAdditional ? 1 : -1; + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + await this.inc({ + instance: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/hashtag.ts b/packages/backend/src/services/chart/charts/classes/hashtag.ts new file mode 100644 index 0000000000..f7f5e17dec --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/hashtag.ts @@ -0,0 +1,47 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { User } from '@/models/entities/user'; +import { SchemaType } from '@/misc/schema'; +import { Users } from '@/models/index'; +import { name, schema } from '../schemas/hashtag'; + +type HashtagLog = SchemaType<typeof schema>; + +export default class HashtagChart extends Chart<HashtagLog> { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: HashtagLog): DeepPartial<HashtagLog> { + return {}; + } + + @autobind + protected aggregate(logs: HashtagLog[]): HashtagLog { + return { + local: { + users: logs.reduce((a, b) => a.concat(b.local.users), [] as HashtagLog['local']['users']), + }, + remote: { + users: logs.reduce((a, b) => a.concat(b.remote.users), [] as HashtagLog['remote']['users']), + }, + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<HashtagLog>> { + return {}; + } + + @autobind + public async update(hashtag: string, user: { id: User['id'], host: User['host'] }) { + const update: Obj = { + users: [user.id] + }; + + await this.inc({ + [Users.isLocalUser(user) ? 'local' : 'remote']: update + }, hashtag); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/instance.ts b/packages/backend/src/services/chart/charts/classes/instance.ts new file mode 100644 index 0000000000..1032de7bc0 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/instance.ts @@ -0,0 +1,217 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { DriveFiles, Followings, Users, Notes } from '@/models/index'; +import { DriveFile } from '@/models/entities/drive-file'; +import { name, schema } from '../schemas/instance'; +import { Note } from '@/models/entities/note'; +import { toPuny } from '@/misc/convert-host'; + +type InstanceLog = SchemaType<typeof schema>; + +export default class InstanceChart extends Chart<InstanceLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: InstanceLog): DeepPartial<InstanceLog> { + return { + notes: { + total: latest.notes.total, + }, + users: { + total: latest.users.total, + }, + following: { + total: latest.following.total, + }, + followers: { + total: latest.followers.total, + }, + drive: { + totalFiles: latest.drive.totalFiles, + totalUsage: latest.drive.totalUsage, + } + }; + } + + @autobind + protected aggregate(logs: InstanceLog[]): InstanceLog { + return { + requests: { + failed: logs.reduce((a, b) => a + b.requests.failed, 0), + succeeded: logs.reduce((a, b) => a + b.requests.succeeded, 0), + received: logs.reduce((a, b) => a + b.requests.received, 0), + }, + notes: { + total: logs[0].notes.total, + inc: logs.reduce((a, b) => a + b.notes.inc, 0), + dec: logs.reduce((a, b) => a + b.notes.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.notes.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.notes.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.notes.diffs.normal, 0), + }, + }, + users: { + total: logs[0].users.total, + inc: logs.reduce((a, b) => a + b.users.inc, 0), + dec: logs.reduce((a, b) => a + b.users.dec, 0), + }, + following: { + total: logs[0].following.total, + inc: logs.reduce((a, b) => a + b.following.inc, 0), + dec: logs.reduce((a, b) => a + b.following.dec, 0), + }, + followers: { + total: logs[0].followers.total, + inc: logs.reduce((a, b) => a + b.followers.inc, 0), + dec: logs.reduce((a, b) => a + b.followers.dec, 0), + }, + drive: { + totalFiles: logs[0].drive.totalFiles, + totalUsage: logs[0].drive.totalUsage, + incFiles: logs.reduce((a, b) => a + b.drive.incFiles, 0), + incUsage: logs.reduce((a, b) => a + b.drive.incUsage, 0), + decFiles: logs.reduce((a, b) => a + b.drive.decFiles, 0), + decUsage: logs.reduce((a, b) => a + b.drive.decUsage, 0), + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<InstanceLog>> { + const [ + notesCount, + usersCount, + followingCount, + followersCount, + driveFiles, + driveUsage, + ] = await Promise.all([ + Notes.count({ userHost: group }), + Users.count({ host: group }), + Followings.count({ followerHost: group }), + Followings.count({ followeeHost: group }), + DriveFiles.count({ userHost: group }), + DriveFiles.calcDriveUsageOfHost(group), + ]); + + return { + notes: { + total: notesCount, + }, + users: { + total: usersCount, + }, + following: { + total: followingCount, + }, + followers: { + total: followersCount, + }, + drive: { + totalFiles: driveFiles, + totalUsage: driveUsage, + } + }; + } + + @autobind + public async requestReceived(host: string) { + await this.inc({ + requests: { + received: 1 + } + }, toPuny(host)); + } + + @autobind + public async requestSent(host: string, isSucceeded: boolean) { + const update: Obj = {}; + + if (isSucceeded) { + update.succeeded = 1; + } else { + update.failed = 1; + } + + await this.inc({ + requests: update + }, toPuny(host)); + } + + @autobind + public async newUser(host: string) { + await this.inc({ + users: { + total: 1, + inc: 1 + } + }, toPuny(host)); + } + + @autobind + public async updateNote(host: string, note: Note, isAdditional: boolean) { + const diffs = {} as any; + + if (note.replyId != null) { + diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + diffs.renote = isAdditional ? 1 : -1; + } else { + diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc({ + notes: { + total: isAdditional ? 1 : -1, + inc: isAdditional ? 1 : 0, + dec: isAdditional ? 0 : 1, + diffs: diffs + } + }, toPuny(host)); + } + + @autobind + public async updateFollowing(host: string, isAdditional: boolean) { + await this.inc({ + following: { + total: isAdditional ? 1 : -1, + inc: isAdditional ? 1 : 0, + dec: isAdditional ? 0 : 1, + } + }, toPuny(host)); + } + + @autobind + public async updateFollowers(host: string, isAdditional: boolean) { + await this.inc({ + followers: { + total: isAdditional ? 1 : -1, + inc: isAdditional ? 1 : 0, + dec: isAdditional ? 0 : 1, + } + }, toPuny(host)); + } + + @autobind + public async updateDrive(file: DriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalFiles = isAdditional ? 1 : -1; + update.totalUsage = isAdditional ? file.size : -file.size; + if (isAdditional) { + update.incFiles = 1; + update.incUsage = file.size; + } else { + update.decFiles = 1; + update.decUsage = file.size; + } + + await this.inc({ + drive: update + }, file.userHost); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/network.ts b/packages/backend/src/services/chart/charts/classes/network.ts new file mode 100644 index 0000000000..2ce75e0b34 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/network.ts @@ -0,0 +1,45 @@ +import autobind from 'autobind-decorator'; +import Chart, { DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { name, schema } from '../schemas/network'; + +type NetworkLog = SchemaType<typeof schema>; + +export default class NetworkChart extends Chart<NetworkLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: NetworkLog): DeepPartial<NetworkLog> { + return {}; + } + + @autobind + protected aggregate(logs: NetworkLog[]): NetworkLog { + return { + incomingRequests: logs.reduce((a, b) => a + b.incomingRequests, 0), + outgoingRequests: logs.reduce((a, b) => a + b.outgoingRequests, 0), + totalTime: logs.reduce((a, b) => a + b.totalTime, 0), + incomingBytes: logs.reduce((a, b) => a + b.incomingBytes, 0), + outgoingBytes: logs.reduce((a, b) => a + b.outgoingBytes, 0), + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<NetworkLog>> { + return {}; + } + + @autobind + public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number) { + const inc: DeepPartial<NetworkLog> = { + incomingRequests: incomingRequests, + totalTime: time, + incomingBytes: incomingBytes, + outgoingBytes: outgoingBytes + }; + + await this.inc(inc); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/notes.ts b/packages/backend/src/services/chart/charts/classes/notes.ts new file mode 100644 index 0000000000..0675d346d1 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/notes.ts @@ -0,0 +1,97 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { Notes } from '@/models/index'; +import { Not, IsNull } from 'typeorm'; +import { Note } from '@/models/entities/note'; +import { name, schema } from '../schemas/notes'; + +type NotesLog = SchemaType<typeof schema>; + +export default class NotesChart extends Chart<NotesLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: NotesLog): DeepPartial<NotesLog> { + return { + local: { + total: latest.local.total, + }, + remote: { + total: latest.remote.total, + } + }; + } + + @autobind + protected aggregate(logs: NotesLog[]): NotesLog { + return { + local: { + total: logs[0].local.total, + inc: logs.reduce((a, b) => a + b.local.inc, 0), + dec: logs.reduce((a, b) => a + b.local.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.local.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.local.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.local.diffs.normal, 0), + }, + }, + remote: { + total: logs[0].remote.total, + inc: logs.reduce((a, b) => a + b.remote.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.remote.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.remote.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.remote.diffs.normal, 0), + }, + }, + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<NotesLog>> { + const [localCount, remoteCount] = await Promise.all([ + Notes.count({ userHost: null }), + Notes.count({ userHost: Not(IsNull()) }) + ]); + + return { + local: { + total: localCount, + }, + remote: { + total: remoteCount, + } + }; + } + + @autobind + public async update(note: Note, isAdditional: boolean) { + const update: Obj = { + diffs: {} + }; + + update.total = isAdditional ? 1 : -1; + + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + if (note.replyId != null) { + update.diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + update.diffs.renote = isAdditional ? 1 : -1; + } else { + update.diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc({ + [note.userHost === null ? 'local' : 'remote']: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/per-user-drive.ts b/packages/backend/src/services/chart/charts/classes/per-user-drive.ts new file mode 100644 index 0000000000..f28987191b --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/per-user-drive.ts @@ -0,0 +1,64 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { DriveFiles } from '@/models/index'; +import { DriveFile } from '@/models/entities/drive-file'; +import { name, schema } from '../schemas/per-user-drive'; + +type PerUserDriveLog = SchemaType<typeof schema>; + +export default class PerUserDriveChart extends Chart<PerUserDriveLog> { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserDriveLog): DeepPartial<PerUserDriveLog> { + return { + totalCount: latest.totalCount, + totalSize: latest.totalSize, + }; + } + + @autobind + protected aggregate(logs: PerUserDriveLog[]): PerUserDriveLog { + return { + totalCount: logs[0].totalCount, + totalSize: logs[0].totalSize, + incCount: logs.reduce((a, b) => a + b.incCount, 0), + incSize: logs.reduce((a, b) => a + b.incSize, 0), + decCount: logs.reduce((a, b) => a + b.decCount, 0), + decSize: logs.reduce((a, b) => a + b.decSize, 0), + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<PerUserDriveLog>> { + const [count, size] = await Promise.all([ + DriveFiles.count({ userId: group }), + DriveFiles.calcDriveUsageOf(group) + ]); + + return { + totalCount: count, + totalSize: size, + }; + } + + @autobind + public async update(file: DriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalCount = isAdditional ? 1 : -1; + update.totalSize = isAdditional ? file.size : -file.size; + if (isAdditional) { + update.incCount = 1; + update.incSize = file.size; + } else { + update.decCount = 1; + update.decSize = file.size; + } + + await this.inc(update, file.userId); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/per-user-following.ts b/packages/backend/src/services/chart/charts/classes/per-user-following.ts new file mode 100644 index 0000000000..08a9ad1d2b --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/per-user-following.ts @@ -0,0 +1,121 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { Followings, Users } from '@/models/index'; +import { Not, IsNull } from 'typeorm'; +import { User } from '@/models/entities/user'; +import { name, schema } from '../schemas/per-user-following'; + +type PerUserFollowingLog = SchemaType<typeof schema>; + +export default class PerUserFollowingChart extends Chart<PerUserFollowingLog> { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserFollowingLog): DeepPartial<PerUserFollowingLog> { + return { + local: { + followings: { + total: latest.local.followings.total, + }, + followers: { + total: latest.local.followers.total, + } + }, + remote: { + followings: { + total: latest.remote.followings.total, + }, + followers: { + total: latest.remote.followers.total, + } + } + }; + } + + @autobind + protected aggregate(logs: PerUserFollowingLog[]): PerUserFollowingLog { + return { + local: { + followings: { + total: logs[0].local.followings.total, + inc: logs.reduce((a, b) => a + b.local.followings.inc, 0), + dec: logs.reduce((a, b) => a + b.local.followings.dec, 0), + }, + followers: { + total: logs[0].local.followers.total, + inc: logs.reduce((a, b) => a + b.local.followers.inc, 0), + dec: logs.reduce((a, b) => a + b.local.followers.dec, 0), + }, + }, + remote: { + followings: { + total: logs[0].remote.followings.total, + inc: logs.reduce((a, b) => a + b.remote.followings.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.followings.dec, 0), + }, + followers: { + total: logs[0].remote.followers.total, + inc: logs.reduce((a, b) => a + b.remote.followers.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.followers.dec, 0), + }, + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<PerUserFollowingLog>> { + const [ + localFollowingsCount, + localFollowersCount, + remoteFollowingsCount, + remoteFollowersCount + ] = await Promise.all([ + Followings.count({ followerId: group, followeeHost: null }), + Followings.count({ followeeId: group, followerHost: null }), + Followings.count({ followerId: group, followeeHost: Not(IsNull()) }), + Followings.count({ followeeId: group, followerHost: Not(IsNull()) }) + ]); + + return { + local: { + followings: { + total: localFollowingsCount, + }, + followers: { + total: localFollowersCount, + } + }, + remote: { + followings: { + total: remoteFollowingsCount, + }, + followers: { + total: remoteFollowersCount, + } + } + }; + } + + @autobind + public async update(follower: { id: User['id']; host: User['host']; }, followee: { id: User['id']; host: User['host']; }, isFollow: boolean) { + const update: Obj = {}; + + update.total = isFollow ? 1 : -1; + + if (isFollow) { + update.inc = 1; + } else { + update.dec = 1; + } + + this.inc({ + [Users.isLocalUser(follower) ? 'local' : 'remote']: { followings: update } + }, follower.id); + this.inc({ + [Users.isLocalUser(followee) ? 'local' : 'remote']: { followers: update } + }, followee.id); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/per-user-notes.ts b/packages/backend/src/services/chart/charts/classes/per-user-notes.ts new file mode 100644 index 0000000000..0e808766f5 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/per-user-notes.ts @@ -0,0 +1,72 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { User } from '@/models/entities/user'; +import { SchemaType } from '@/misc/schema'; +import { Notes } from '@/models/index'; +import { Note } from '@/models/entities/note'; +import { name, schema } from '../schemas/per-user-notes'; + +type PerUserNotesLog = SchemaType<typeof schema>; + +export default class PerUserNotesChart extends Chart<PerUserNotesLog> { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserNotesLog): DeepPartial<PerUserNotesLog> { + return { + total: latest.total, + }; + } + + @autobind + protected aggregate(logs: PerUserNotesLog[]): PerUserNotesLog { + return { + total: logs[0].total, + inc: logs.reduce((a, b) => a + b.inc, 0), + dec: logs.reduce((a, b) => a + b.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.diffs.normal, 0), + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<PerUserNotesLog>> { + const [count] = await Promise.all([ + Notes.count({ userId: group }), + ]); + + return { + total: count, + }; + } + + @autobind + public async update(user: { id: User['id'] }, note: Note, isAdditional: boolean) { + const update: Obj = { + diffs: {} + }; + + update.total = isAdditional ? 1 : -1; + + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + if (note.replyId != null) { + update.diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + update.diffs.renote = isAdditional ? 1 : -1; + } else { + update.diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc(update, user.id); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/per-user-reactions.ts b/packages/backend/src/services/chart/charts/classes/per-user-reactions.ts new file mode 100644 index 0000000000..e71bcb71c4 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/per-user-reactions.ts @@ -0,0 +1,44 @@ +import autobind from 'autobind-decorator'; +import Chart, { DeepPartial } from '../../core'; +import { User } from '@/models/entities/user'; +import { Note } from '@/models/entities/note'; +import { SchemaType } from '@/misc/schema'; +import { Users } from '@/models/index'; +import { name, schema } from '../schemas/per-user-reactions'; + +type PerUserReactionsLog = SchemaType<typeof schema>; + +export default class PerUserReactionsChart extends Chart<PerUserReactionsLog> { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserReactionsLog): DeepPartial<PerUserReactionsLog> { + return {}; + } + + @autobind + protected aggregate(logs: PerUserReactionsLog[]): PerUserReactionsLog { + return { + local: { + count: logs.reduce((a, b) => a + b.local.count, 0), + }, + remote: { + count: logs.reduce((a, b) => a + b.remote.count, 0), + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<PerUserReactionsLog>> { + return {}; + } + + @autobind + public async update(user: { id: User['id'], host: User['host'] }, note: Note) { + this.inc({ + [Users.isLocalUser(user) ? 'local' : 'remote']: { count: 1 } + }, note.userId); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/test-grouped.ts b/packages/backend/src/services/chart/charts/classes/test-grouped.ts new file mode 100644 index 0000000000..84e6d5e33f --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/test-grouped.ts @@ -0,0 +1,58 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { name, schema } from '../schemas/test-grouped'; + +type TestGroupedLog = SchemaType<typeof schema>; + +export default class TestGroupedChart extends Chart<TestGroupedLog> { + private total = {} as Record<string, number>; + + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: TestGroupedLog): DeepPartial<TestGroupedLog> { + return { + foo: { + total: latest.foo.total, + }, + }; + } + + @autobind + protected aggregate(logs: TestGroupedLog[]): TestGroupedLog { + return { + foo: { + total: logs[0].foo.total, + inc: logs.reduce((a, b) => a + b.foo.inc, 0), + dec: logs.reduce((a, b) => a + b.foo.dec, 0), + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<TestGroupedLog>> { + return { + foo: { + total: this.total[group], + }, + }; + } + + @autobind + public async increment(group: string) { + if (this.total[group] == null) this.total[group] = 0; + + const update: Obj = {}; + + update.total = 1; + update.inc = 1; + this.total[group]++; + + await this.inc({ + foo: update + }, group); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/test-unique.ts b/packages/backend/src/services/chart/charts/classes/test-unique.ts new file mode 100644 index 0000000000..559fda13c9 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/test-unique.ts @@ -0,0 +1,36 @@ +import autobind from 'autobind-decorator'; +import Chart, { DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { name, schema } from '../schemas/test-unique'; + +type TestUniqueLog = SchemaType<typeof schema>; + +export default class TestUniqueChart extends Chart<TestUniqueLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: TestUniqueLog): DeepPartial<TestUniqueLog> { + return {}; + } + + @autobind + protected aggregate(logs: TestUniqueLog[]): TestUniqueLog { + return { + foo: logs.reduce((a, b) => a.concat(b.foo), [] as TestUniqueLog['foo']), + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<TestUniqueLog>> { + return {}; + } + + @autobind + public async uniqueIncrement(key: string) { + await this.inc({ + foo: [key] + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/test.ts b/packages/backend/src/services/chart/charts/classes/test.ts new file mode 100644 index 0000000000..a91d5e1895 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/test.ts @@ -0,0 +1,69 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { name, schema } from '../schemas/test'; + +type TestLog = SchemaType<typeof schema>; + +export default class TestChart extends Chart<TestLog> { + public total = 0; // publicにするのはテストのため + + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: TestLog): DeepPartial<TestLog> { + return { + foo: { + total: latest.foo.total, + }, + }; + } + + @autobind + protected aggregate(logs: TestLog[]): TestLog { + return { + foo: { + total: logs[0].foo.total, + inc: logs.reduce((a, b) => a + b.foo.inc, 0), + dec: logs.reduce((a, b) => a + b.foo.dec, 0), + }, + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<TestLog>> { + return { + foo: { + total: this.total, + }, + }; + } + + @autobind + public async increment() { + const update: Obj = {}; + + update.total = 1; + update.inc = 1; + this.total++; + + await this.inc({ + foo: update + }); + } + + @autobind + public async decrement() { + const update: Obj = {}; + + update.total = -1; + update.dec = 1; + this.total--; + + await this.inc({ + foo: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/classes/users.ts b/packages/backend/src/services/chart/charts/classes/users.ts new file mode 100644 index 0000000000..89b480ef77 --- /dev/null +++ b/packages/backend/src/services/chart/charts/classes/users.ts @@ -0,0 +1,76 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '@/misc/schema'; +import { Users } from '@/models/index'; +import { Not, IsNull } from 'typeorm'; +import { User } from '@/models/entities/user'; +import { name, schema } from '../schemas/users'; + +type UsersLog = SchemaType<typeof schema>; + +export default class UsersChart extends Chart<UsersLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: UsersLog): DeepPartial<UsersLog> { + return { + local: { + total: latest.local.total, + }, + remote: { + total: latest.remote.total, + } + }; + } + + @autobind + protected aggregate(logs: UsersLog[]): UsersLog { + return { + local: { + total: logs[0].local.total, + inc: logs.reduce((a, b) => a + b.local.inc, 0), + dec: logs.reduce((a, b) => a + b.local.dec, 0), + }, + remote: { + total: logs[0].remote.total, + inc: logs.reduce((a, b) => a + b.remote.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.dec, 0), + }, + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<UsersLog>> { + const [localCount, remoteCount] = await Promise.all([ + Users.count({ host: null }), + Users.count({ host: Not(IsNull()) }) + ]); + + return { + local: { + total: localCount, + }, + remote: { + total: remoteCount, + } + }; + } + + @autobind + public async update(user: { id: User['id'], host: User['host'] }, isAdditional: boolean) { + const update: Obj = {}; + + update.total = isAdditional ? 1 : -1; + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + await this.inc({ + [Users.isLocalUser(user) ? 'local' : 'remote']: update + }); + } +} diff --git a/packages/backend/src/services/chart/charts/schemas/active-users.ts b/packages/backend/src/services/chart/charts/schemas/active-users.ts new file mode 100644 index 0000000000..1d65f280b0 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/active-users.ts @@ -0,0 +1,35 @@ +export const logSchema = { + /** + * アクティブユーザー + */ + users: { + type: 'array' as const, + optional: false as const, nullable: false as const, + items: { + type: 'string' as const, + optional: false as const, nullable: false as const, + } + }, +}; + +/** + * アクティブユーザーに関するチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'activeUsers'; diff --git a/packages/backend/src/services/chart/charts/schemas/drive.ts b/packages/backend/src/services/chart/charts/schemas/drive.ts new file mode 100644 index 0000000000..133b47846a --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/drive.ts @@ -0,0 +1,68 @@ +const logSchema = { + /** + * 集計期間時点での、全ドライブファイル数 + */ + totalCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 集計期間時点での、全ドライブファイルの合計サイズ + */ + totalSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 増加したドライブファイル数 + */ + incCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 増加したドライブ使用量 + */ + incSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 減少したドライブファイル数 + */ + decCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 減少したドライブ使用量 + */ + decSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, +}; + +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'drive'; diff --git a/packages/backend/src/services/chart/charts/schemas/federation.ts b/packages/backend/src/services/chart/charts/schemas/federation.ts new file mode 100644 index 0000000000..dca4587cac --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/federation.ts @@ -0,0 +1,29 @@ +/** + * フェデレーションに関するチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + instance: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + } + } +}; + +export const name = 'federation'; diff --git a/packages/backend/src/services/chart/charts/schemas/hashtag.ts b/packages/backend/src/services/chart/charts/schemas/hashtag.ts new file mode 100644 index 0000000000..4e7c542bbc --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/hashtag.ts @@ -0,0 +1,35 @@ +export const logSchema = { + /** + * 投稿したユーザー + */ + users: { + type: 'array' as const, + optional: false as const, nullable: false as const, + items: { + type: 'string' as const, + optional: false as const, nullable: false as const, + } + }, +}; + +/** + * ハッシュタグに関するチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'hashtag'; diff --git a/packages/backend/src/services/chart/charts/schemas/instance.ts b/packages/backend/src/services/chart/charts/schemas/instance.ts new file mode 100644 index 0000000000..785d6ae7ce --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/instance.ts @@ -0,0 +1,157 @@ +/** + * インスタンスごとのチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + requests: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + failed: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + succeeded: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + received: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + + notes: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + diffs: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + normal: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + reply: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + renote: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + } + }, + + users: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + + following: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + + followers: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + + drive: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + totalFiles: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + totalUsage: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + incFiles: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + incUsage: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + decFiles: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + decUsage: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + } +}; + +export const name = 'instance'; diff --git a/packages/backend/src/services/chart/charts/schemas/network.ts b/packages/backend/src/services/chart/charts/schemas/network.ts new file mode 100644 index 0000000000..49a364debc --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/network.ts @@ -0,0 +1,31 @@ +/** + * ネットワークに関するチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + incomingRequests: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + outgoingRequests: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + totalTime: { // TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + incomingBytes: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + outgoingBytes: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } +}; + +export const name = 'network'; diff --git a/packages/backend/src/services/chart/charts/schemas/notes.ts b/packages/backend/src/services/chart/charts/schemas/notes.ts new file mode 100644 index 0000000000..2b5105348c --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/notes.ts @@ -0,0 +1,56 @@ +const logSchema = { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + diffs: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + normal: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + reply: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + renote: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, +}; + +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'notes'; diff --git a/packages/backend/src/services/chart/charts/schemas/per-user-drive.ts b/packages/backend/src/services/chart/charts/schemas/per-user-drive.ts new file mode 100644 index 0000000000..856f1e0439 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/per-user-drive.ts @@ -0,0 +1,55 @@ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + /** + * 集計期間時点での、全ドライブファイル数 + */ + totalCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 集計期間時点での、全ドライブファイルの合計サイズ + */ + totalSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 増加したドライブファイル数 + */ + incCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 増加したドライブ使用量 + */ + incSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 減少したドライブファイル数 + */ + decCount: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 減少したドライブ使用量 + */ + decSize: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } +}; + +export const name = 'perUserDrive'; diff --git a/packages/backend/src/services/chart/charts/schemas/per-user-following.ts b/packages/backend/src/services/chart/charts/schemas/per-user-following.ts new file mode 100644 index 0000000000..eaf74aaf77 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/per-user-following.ts @@ -0,0 +1,86 @@ +export const logSchema = { + /** + * フォローしている + */ + followings: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + /** + * フォローしている合計 + */ + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * フォローした数 + */ + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * フォロー解除した数 + */ + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + + /** + * フォローされている + */ + followers: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + /** + * フォローされている合計 + */ + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * フォローされた数 + */ + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * フォロー解除された数 + */ + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, +}; + +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'perUserFollowing'; diff --git a/packages/backend/src/services/chart/charts/schemas/per-user-notes.ts b/packages/backend/src/services/chart/charts/schemas/per-user-notes.ts new file mode 100644 index 0000000000..72b3ff0210 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/per-user-notes.ts @@ -0,0 +1,43 @@ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + diffs: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + normal: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + reply: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + renote: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + }, + } +}; + +export const name = 'perUserNotes'; diff --git a/packages/backend/src/services/chart/charts/schemas/per-user-reactions.ts b/packages/backend/src/services/chart/charts/schemas/per-user-reactions.ts new file mode 100644 index 0000000000..2a8520db37 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/per-user-reactions.ts @@ -0,0 +1,31 @@ +export const logSchema = { + /** + * フォローしている合計 + */ + count: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, +}; + +/** + * ユーザーごとのリアクションに関するチャート + */ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'perUserReaction'; diff --git a/packages/backend/src/services/chart/charts/schemas/test-grouped.ts b/packages/backend/src/services/chart/charts/schemas/test-grouped.ts new file mode 100644 index 0000000000..f8c8250e79 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/test-grouped.ts @@ -0,0 +1,28 @@ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + foo: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + } + } +}; + +export const name = 'testGrouped'; diff --git a/packages/backend/src/services/chart/charts/schemas/test-unique.ts b/packages/backend/src/services/chart/charts/schemas/test-unique.ts new file mode 100644 index 0000000000..51280400ac --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/test-unique.ts @@ -0,0 +1,16 @@ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + foo: { + type: 'array' as const, + optional: false as const, nullable: false as const, + items: { + type: 'string' as const, + optional: false as const, nullable: false as const, + } + }, + } +}; + +export const name = 'testUnique'; diff --git a/packages/backend/src/services/chart/charts/schemas/test.ts b/packages/backend/src/services/chart/charts/schemas/test.ts new file mode 100644 index 0000000000..4b48d4d417 --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/test.ts @@ -0,0 +1,28 @@ +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + foo: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + } + } + } +}; + +export const name = 'test'; diff --git a/packages/backend/src/services/chart/charts/schemas/users.ts b/packages/backend/src/services/chart/charts/schemas/users.ts new file mode 100644 index 0000000000..2bf9d3c50f --- /dev/null +++ b/packages/backend/src/services/chart/charts/schemas/users.ts @@ -0,0 +1,44 @@ +const logSchema = { + /** + * 集計期間時点での、全ユーザー数 + */ + total: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 増加したユーザー数 + */ + inc: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, + + /** + * 減少したユーザー数 + */ + dec: { + type: 'number' as const, + optional: false as const, nullable: false as const, + }, +}; + +export const schema = { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: { + local: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + remote: { + type: 'object' as const, + optional: false as const, nullable: false as const, + properties: logSchema + }, + } +}; + +export const name = 'users'; diff --git a/packages/backend/src/services/chart/core.ts b/packages/backend/src/services/chart/core.ts new file mode 100644 index 0000000000..78b7dd1359 --- /dev/null +++ b/packages/backend/src/services/chart/core.ts @@ -0,0 +1,563 @@ +/** + * チャートエンジン + * + * Tests located in test/chart + */ + +import * as nestedProperty from 'nested-property'; +import autobind from 'autobind-decorator'; +import Logger from '../logger'; +import { SimpleSchema } from '@/misc/simple-schema'; +import { EntitySchema, getRepository, Repository, LessThan, Between } from 'typeorm'; +import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from '@/prelude/time'; +import { getChartInsertLock } from '@/misc/app-lock'; + +const logger = new Logger('chart', 'white', process.env.NODE_ENV !== 'test'); + +export type Obj = { [key: string]: any }; + +export type DeepPartial<T> = { + [P in keyof T]?: DeepPartial<T[P]>; +}; + +type ArrayValue<T> = { + [P in keyof T]: T[P] extends number ? T[P][] : ArrayValue<T[P]>; +}; + +type Log = { + id: number; + + /** + * 集計のグループ + */ + group: string | null; + + /** + * 集計日時のUnixタイムスタンプ(秒) + */ + date: number; +}; + +const camelToSnake = (str: string) => { + return str.replace(/([A-Z])/g, s => '_' + s.charAt(0).toLowerCase()); +}; + +const removeDuplicates = (array: any[]) => Array.from(new Set(array)); + +/** + * 様々なチャートの管理を司るクラス + */ +export default abstract class Chart<T extends Record<string, any>> { + private static readonly columnPrefix = '___'; + private static readonly columnDot = '_'; + + private name: string; + private buffer: { + diff: DeepPartial<T>; + group: string | null; + }[] = []; + public schema: SimpleSchema; + protected repository: Repository<Log>; + + protected abstract genNewLog(latest: T): DeepPartial<T>; + + /** + * @param logs 日時が新しい方が先頭 + */ + protected abstract aggregate(logs: T[]): T; + + protected abstract fetchActual(group: string | null): Promise<DeepPartial<T>>; + + @autobind + private static convertSchemaToFlatColumnDefinitions(schema: SimpleSchema) { + const columns = {} as Record<string, unknown>; + const flatColumns = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}${this.columnDot}${k}` : k; + if (v.type === 'object') { + flatColumns(v.properties, p); + } else if (v.type === 'number') { + columns[this.columnPrefix + p] = { + type: 'bigint', + }; + } else if (v.type === 'array' && v.items.type === 'string') { + columns[this.columnPrefix + p] = { + type: 'varchar', + array: true, + }; + } + } + }; + flatColumns(schema.properties!); + return columns; + } + + @autobind + private static convertFlattenColumnsToObject(x: Record<string, unknown>): Record<string, unknown> { + const obj = {} as Record<string, unknown>; + for (const k of Object.keys(x).filter(k => k.startsWith(Chart.columnPrefix))) { + // now k is ___x_y_z + const path = k.substr(Chart.columnPrefix.length).split(Chart.columnDot).join('.'); + nestedProperty.set(obj, path, x[k]); + } + return obj; + } + + @autobind + private static convertObjectToFlattenColumns(x: Record<string, unknown>) { + const columns = {} as Record<string, number | unknown[]>; + const flatten = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}${this.columnDot}${k}` : k; + if (typeof v === 'object' && !Array.isArray(v)) { + flatten(v, p); + } else { + columns[this.columnPrefix + p] = v; + } + } + }; + flatten(x); + return columns; + } + + @autobind + private static countUniqueFields(x: Record<string, unknown>) { + const exec = (x: Obj) => { + const res = {} as Record<string, unknown>; + for (const [k, v] of Object.entries(x)) { + if (typeof v === 'object' && !Array.isArray(v)) { + res[k] = exec(v); + } else if (Array.isArray(v)) { + res[k] = Array.from(new Set(v)).length; + } else { + res[k] = v; + } + } + return res; + }; + return exec(x); + } + + @autobind + private static convertQuery(diff: Record<string, number | unknown[]>) { + const query: Record<string, () => string> = {}; + + for (const [k, v] of Object.entries(diff)) { + if (typeof v === 'number') { + if (v > 0) query[k] = () => `"${k}" + ${v}`; + if (v < 0) query[k] = () => `"${k}" - ${Math.abs(v)}`; + } else if (Array.isArray(v)) { + // TODO: item が文字列以外の場合も対応 + // TODO: item をSQLエスケープ + const items = v.map(item => `"${item}"`).join(','); + query[k] = () => `array_cat("${k}", '{${items}}'::varchar[])`; + } + } + + return query; + } + + @autobind + private static dateToTimestamp(x: Date): Log['date'] { + return Math.floor(x.getTime() / 1000); + } + + @autobind + private static parseDate(date: Date): [number, number, number, number, number, number, number] { + const y = date.getUTCFullYear(); + const m = date.getUTCMonth(); + const d = date.getUTCDate(); + const h = date.getUTCHours(); + const _m = date.getUTCMinutes(); + const _s = date.getUTCSeconds(); + const _ms = date.getUTCMilliseconds(); + + return [y, m, d, h, _m, _s, _ms]; + } + + @autobind + private static getCurrentDate() { + return Chart.parseDate(new Date()); + } + + @autobind + public static schemaToEntity(name: string, schema: SimpleSchema): EntitySchema { + return new EntitySchema({ + name: `__chart__${camelToSnake(name)}`, + columns: { + id: { + type: 'integer', + primary: true, + generated: true + }, + date: { + type: 'integer', + }, + group: { + type: 'varchar', + length: 128, + nullable: true + }, + ...Chart.convertSchemaToFlatColumnDefinitions(schema) + }, + indices: [{ + columns: ['date', 'group'], + unique: true, + }, { // groupにnullが含まれると↑のuniqueは機能しないので↓の部分インデックスでカバー + columns: ['date'], + unique: true, + where: '"group" IS NULL' + }] + }); + } + + constructor(name: string, schema: SimpleSchema, grouped = false) { + this.name = name; + this.schema = schema; + const entity = Chart.schemaToEntity(name, schema); + + const keys = ['date']; + if (grouped) keys.push('group'); + + entity.options.uniques = [{ + columns: keys + }]; + + this.repository = getRepository<Log>(entity); + } + + @autobind + private getNewLog(latest: T | null): T { + const log = latest ? this.genNewLog(latest) : {}; + const flatColumns = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}.${k}` : k; + if (v.type === 'object') { + flatColumns(v.properties, p); + } else { + if (nestedProperty.get(log, p) == null) { + const emptyValue = v.type === 'number' ? 0 : []; + nestedProperty.set(log, p, emptyValue); + } + } + } + }; + flatColumns(this.schema.properties!); + return log as T; + } + + @autobind + private getLatestLog(group: string | null = null): Promise<Log | null> { + return this.repository.findOne({ + group: group, + }, { + order: { + date: -1 + } + }).then(x => x || null); + } + + @autobind + private async getCurrentLog(group: string | null = null): Promise<Log> { + const [y, m, d, h] = Chart.getCurrentDate(); + + const current = dateUTC([y, m, d, h]); + + // 現在(=今のHour)のログ + const currentLog = await this.repository.findOne({ + date: Chart.dateToTimestamp(current), + ...(group ? { group: group } : {}) + }); + + // ログがあればそれを返して終了 + if (currentLog != null) { + return currentLog; + } + + let log: Log; + let data: T; + + // 集計期間が変わってから、初めてのチャート更新なら + // 最も最近のログを持ってくる + // * 例えば集計期間が「日」である場合で考えると、 + // * 昨日何もチャートを更新するような出来事がなかった場合は、 + // * ログがそもそも作られずドキュメントが存在しないということがあり得るため、 + // * 「昨日の」と決め打ちせずに「もっとも最近の」とします + const latest = await this.getLatestLog(group); + + if (latest != null) { + const obj = Chart.convertFlattenColumnsToObject(latest) as T; + + // 空ログデータを作成 + data = this.getNewLog(obj); + } else { + // ログが存在しなかったら + // (Misskeyインスタンスを建てて初めてのチャート更新時など) + + // 初期ログデータを作成 + data = this.getNewLog(null); + + logger.info(`${this.name + (group ? `:${group}` : '')}: Initial commit created`); + } + + const date = Chart.dateToTimestamp(current); + const lockKey = `${this.name}:${date}:${group}`; + + const unlock = await getChartInsertLock(lockKey); + try { + // ロック内でもう1回チェックする + const currentLog = await this.repository.findOne({ + date: date, + ...(group ? { group: group } : {}) + }); + + // ログがあればそれを返して終了 + if (currentLog != null) return currentLog; + + // 新規ログ挿入 + log = await this.repository.insert({ + group: group, + date: date, + ...Chart.convertObjectToFlattenColumns(data) + }).then(x => this.repository.findOneOrFail(x.identifiers[0])); + + logger.info(`${this.name + (group ? `:${group}` : '')}: New commit created`); + + return log; + } finally { + unlock(); + } + } + + @autobind + protected commit(diff: DeepPartial<T>, group: string | null = null): void { + this.buffer.push({ + diff, group, + }); + } + + @autobind + public async save(): Promise<void> { + if (this.buffer.length === 0) { + logger.info(`${this.name}: Write skipped`); + return; + } + + // TODO: 前の時間のログがbufferにあった場合のハンドリング + // 例えば、save が20分ごとに行われるとして、前回行われたのは 01:50 だったとする。 + // 次に save が行われるのは 02:10 ということになるが、もし 01:55 に新規ログが buffer に追加されたとすると、 + // そのログは本来は 01:00~ のログとしてDBに保存されて欲しいのに、02:00~ のログ扱いになってしまう。 + // これを回避するための実装は複雑になりそうなため、一旦保留。 + + const update = async (log: Log) => { + const finalDiffs = {} as Record<string, number | unknown[]>; + + for (const diff of this.buffer.filter(q => q.group === log.group).map(q => q.diff)) { + const columns = Chart.convertObjectToFlattenColumns(diff); + + for (const [k, v] of Object.entries(columns)) { + if (finalDiffs[k] == null) { + finalDiffs[k] = v; + } else { + if (typeof finalDiffs[k] === 'number') { + (finalDiffs[k] as number) += v as number; + } else { + (finalDiffs[k] as unknown[]) = (finalDiffs[k] as unknown[]).concat(v); + } + } + } + } + + const query = Chart.convertQuery(finalDiffs); + + // ログ更新 + await this.repository.createQueryBuilder() + .update() + .set(query) + .where('id = :id', { id: log.id }) + .execute(); + + logger.info(`${this.name + (log.group ? `:${log.group}` : '')}: Updated`); + + // TODO: この一連の処理が始まった後に新たにbufferに入ったものは消さないようにする + this.buffer = this.buffer.filter(q => q.group !== log.group); + }; + + const groups = removeDuplicates(this.buffer.map(log => log.group)); + + await Promise.all(groups.map(group => this.getCurrentLog(group).then(log => update(log)))); + } + + @autobind + public async resync(group: string | null = null): Promise<any> { + const data = await this.fetchActual(group); + + const update = async (log: Log) => { + await this.repository.createQueryBuilder() + .update() + .set(Chart.convertObjectToFlattenColumns(data)) + .where('id = :id', { id: log.id }) + .execute(); + }; + + return this.getCurrentLog(group).then(log => update(log)); + } + + @autobind + protected async inc(inc: DeepPartial<T>, group: string | null = null): Promise<void> { + await this.commit(inc, group); + } + + @autobind + public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<ArrayValue<T>> { + const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate(); + const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [] as never; + + const lt = dateUTC([y, m, d, h, _m, _s, _ms]); + + const gt = + span === 'day' ? subtractTime(cursor ? dateUTC([y2, m2, d2, 0]) : dateUTC([y, m, d, 0]), amount - 1, 'day') : + span === 'hour' ? subtractTime(cursor ? dateUTC([y2, m2, d2, h2]) : dateUTC([y, m, d, h]), amount - 1, 'hour') : + null as never; + + // ログ取得 + let logs = await this.repository.find({ + where: { + group: group, + date: Between(Chart.dateToTimestamp(gt), Chart.dateToTimestamp(lt)) + }, + order: { + date: -1 + }, + }); + + // 要求された範囲にログがひとつもなかったら + if (logs.length === 0) { + // もっとも新しいログを持ってくる + // (すくなくともひとつログが無いと隙間埋めできないため) + const recentLog = await this.repository.findOne({ + group: group, + }, { + order: { + date: -1 + }, + }); + + if (recentLog) { + logs = [recentLog]; + } + + // 要求された範囲の最も古い箇所に位置するログが存在しなかったら + } else if (!isTimeSame(new Date(logs[logs.length - 1].date * 1000), gt)) { + // 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する + // (隙間埋めできないため) + const outdatedLog = await this.repository.findOne({ + group: group, + date: LessThan(Chart.dateToTimestamp(gt)) + }, { + order: { + date: -1 + }, + }); + + if (outdatedLog) { + logs.push(outdatedLog); + } + } + + const chart: T[] = []; + + if (span === 'hour') { + for (let i = (amount - 1); i >= 0; i--) { + const current = subtractTime(dateUTC([y, m, d, h]), i, 'hour'); + + const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current)); + + if (log) { + const data = Chart.convertFlattenColumnsToObject(log); + chart.unshift(Chart.countUniqueFields(data) as T); + } else { + // 隙間埋め + const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current)); + const data = latest ? Chart.convertFlattenColumnsToObject(latest) as T : null; + chart.unshift(Chart.countUniqueFields(this.getNewLog(data)) as T); + } + } + } else if (span === 'day') { + const logsForEachDays: T[][] = []; + let currentDay = -1; + let currentDayIndex = -1; + for (let i = ((amount - 1) * 24) + h; i >= 0; i--) { + const current = subtractTime(dateUTC([y, m, d, h]), i, 'hour'); + const _currentDay = Chart.parseDate(current)[2]; + if (currentDay != _currentDay) currentDayIndex++; + currentDay = _currentDay; + + const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current)); + + if (log) { + if (logsForEachDays[currentDayIndex]) { + logsForEachDays[currentDayIndex].unshift(Chart.convertFlattenColumnsToObject(log) as T); + } else { + logsForEachDays[currentDayIndex] = [Chart.convertFlattenColumnsToObject(log) as T]; + } + } else { + // 隙間埋め + const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current)); + const data = latest ? Chart.convertFlattenColumnsToObject(latest) as T : null; + const newLog = this.getNewLog(data); + if (logsForEachDays[currentDayIndex]) { + logsForEachDays[currentDayIndex].unshift(newLog); + } else { + logsForEachDays[currentDayIndex] = [newLog]; + } + } + } + + for (const logs of logsForEachDays) { + const log = this.aggregate(logs); + chart.unshift(Chart.countUniqueFields(log) as T); + } + } + + const res: ArrayValue<T> = {} as any; + + /** + * [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }] + * を + * { foo: [1, 2, 3], bar: [5, 6, 7] } + * にする + */ + const compact = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}.${k}` : k; + if (typeof v === 'object' && !Array.isArray(v)) { + compact(v, p); + } else { + const values = chart.map(s => nestedProperty.get(s, p)); + nestedProperty.set(res, p, values); + } + } + }; + + compact(chart[0]); + + return res; + } +} + +export function convertLog(logSchema: SimpleSchema): SimpleSchema { + const v: SimpleSchema = JSON.parse(JSON.stringify(logSchema)); // copy + if (v.type === 'number') { + v.type = 'array'; + v.items = { + type: 'number' as const, + optional: false as const, nullable: false as const, + }; + } else if (v.type === 'object') { + for (const k of Object.keys(v.properties!)) { + v.properties![k] = convertLog(v.properties![k]); + } + } + return v; +} diff --git a/packages/backend/src/services/chart/entities.ts b/packages/backend/src/services/chart/entities.ts new file mode 100644 index 0000000000..23a97607eb --- /dev/null +++ b/packages/backend/src/services/chart/entities.ts @@ -0,0 +1,15 @@ +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; +import Chart from './core'; + +//const _filename = fileURLToPath(import.meta.url); +const _filename = __filename; +const _dirname = dirname(_filename); + +export const entities = Object.values(require('require-all')({ + dirname: _dirname + '/charts/schemas', + filter: /^.+\.[jt]s$/, + resolve: (x: any) => { + return Chart.schemaToEntity(x.name, x.schema); + } +})); diff --git a/packages/backend/src/services/chart/index.ts b/packages/backend/src/services/chart/index.ts new file mode 100644 index 0000000000..61eb431ea3 --- /dev/null +++ b/packages/backend/src/services/chart/index.ts @@ -0,0 +1,50 @@ +import FederationChart from './charts/classes/federation'; +import NotesChart from './charts/classes/notes'; +import UsersChart from './charts/classes/users'; +import NetworkChart from './charts/classes/network'; +import ActiveUsersChart from './charts/classes/active-users'; +import InstanceChart from './charts/classes/instance'; +import PerUserNotesChart from './charts/classes/per-user-notes'; +import DriveChart from './charts/classes/drive'; +import PerUserReactionsChart from './charts/classes/per-user-reactions'; +import HashtagChart from './charts/classes/hashtag'; +import PerUserFollowingChart from './charts/classes/per-user-following'; +import PerUserDriveChart from './charts/classes/per-user-drive'; +import { beforeShutdown } from '@/misc/before-shutdown'; + +export const federationChart = new FederationChart(); +export const notesChart = new NotesChart(); +export const usersChart = new UsersChart(); +export const networkChart = new NetworkChart(); +export const activeUsersChart = new ActiveUsersChart(); +export const instanceChart = new InstanceChart(); +export const perUserNotesChart = new PerUserNotesChart(); +export const driveChart = new DriveChart(); +export const perUserReactionsChart = new PerUserReactionsChart(); +export const hashtagChart = new HashtagChart(); +export const perUserFollowingChart = new PerUserFollowingChart(); +export const perUserDriveChart = new PerUserDriveChart(); + +const charts = [ + federationChart, + notesChart, + usersChart, + networkChart, + activeUsersChart, + instanceChart, + perUserNotesChart, + driveChart, + perUserReactionsChart, + hashtagChart, + perUserFollowingChart, + perUserDriveChart, +]; + +// 20分おきにメモリ情報をDBに書き込み +setInterval(() => { + for (const chart of charts) { + chart.save(); + } +}, 1000 * 60 * 20); + +beforeShutdown(() => Promise.all(charts.map(chart => chart.save()))); |