diff options
| author | syuilo <syuilotan@yahoo.co.jp> | 2019-04-14 20:38:55 +0900 |
|---|---|---|
| committer | syuilo <syuilotan@yahoo.co.jp> | 2019-04-14 20:38:55 +0900 |
| commit | d66e4b7ff97d512e2a2523815e2eef170456b37f (patch) | |
| tree | 59ae1a102d88b5c2c2236b734ea4a584b4f9ba46 /src/services/chart | |
| parent | 10.100.0 (diff) | |
| parent | 11.0.0 (diff) | |
| download | misskey-d66e4b7ff97d512e2a2523815e2eef170456b37f.tar.gz misskey-d66e4b7ff97d512e2a2523815e2eef170456b37f.tar.bz2 misskey-d66e4b7ff97d512e2a2523815e2eef170456b37f.zip | |
Merge branch 'develop'
Diffstat (limited to 'src/services/chart')
45 files changed, 2061 insertions, 1699 deletions
diff --git a/src/services/chart/active-users.ts b/src/services/chart/active-users.ts deleted file mode 100644 index 2a4e1a97ac..0000000000 --- a/src/services/chart/active-users.ts +++ /dev/null @@ -1,48 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Obj } from '.'; -import { IUser, isLocalUser } from '../../models/user'; - -/** - * アクティブユーザーに関するチャート - */ -type ActiveUsersLog = { - local: { - /** - * アクティブユーザー数 - */ - count: number; - }; - - remote: ActiveUsersLog['local']; -}; - -class ActiveUsersChart extends Chart<ActiveUsersLog> { - constructor() { - super('activeUsers'); - } - - @autobind - protected async getTemplate(init: boolean, latest?: ActiveUsersLog): Promise<ActiveUsersLog> { - return { - local: { - count: 0 - }, - remote: { - count: 0 - } - }; - } - - @autobind - public async update(user: IUser) { - const update: Obj = { - count: 1 - }; - - await this.incIfUnique({ - [isLocalUser(user) ? 'local' : 'remote']: update - }, 'users', user._id.toHexString()); - } -} - -export default new ActiveUsersChart(); diff --git a/src/services/chart/charts/classes/active-users.ts b/src/services/chart/charts/classes/active-users.ts new file mode 100644 index 0000000000..5128150de6 --- /dev/null +++ b/src/services/chart/charts/classes/active-users.ts @@ -0,0 +1,35 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { User } from '../../../../models/entities/user'; +import { SchemaType } from '../../../../misc/schema'; +import { Users } from '../../../../models'; +import { name, schema } from '../schemas/active-users'; + +type ActiveUsersLog = SchemaType<typeof schema>; + +export default class ActiveUsersChart extends Chart<ActiveUsersLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: ActiveUsersLog): DeepPartial<ActiveUsersLog> { + return {}; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<ActiveUsersLog>> { + return {}; + } + + @autobind + public async update(user: User) { + const update: Obj = { + count: 1 + }; + + await this.incIfUnique({ + [Users.isLocalUser(user) ? 'local' : 'remote']: update + }, 'users', user.id); + } +} diff --git a/src/services/chart/charts/classes/drive.ts b/src/services/chart/charts/classes/drive.ts new file mode 100644 index 0000000000..ae52df19ac --- /dev/null +++ b/src/services/chart/charts/classes/drive.ts @@ -0,0 +1,69 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { DriveFiles } from '../../../../models'; +import { Not } from 'typeorm'; +import { DriveFile } from '../../../../models/entities/drive-file'; +import { name, schema } from '../schemas/drive'; + +type DriveLog = SchemaType<typeof schema>; + +export default class DriveChart extends Chart<DriveLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: DriveLog): DeepPartial<DriveLog> { + return { + local: { + totalCount: latest.local.totalCount, + totalSize: latest.local.totalSize, + }, + remote: { + totalCount: latest.remote.totalCount, + totalSize: latest.remote.totalSize, + } + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<DriveLog>> { + const [localCount, remoteCount, localSize, remoteSize] = await Promise.all([ + DriveFiles.count({ userHost: null }), + DriveFiles.count({ userHost: Not(null) }), + DriveFiles.clacDriveUsageOfLocal(), + DriveFiles.clacDriveUsageOfRemote() + ]); + + return { + local: { + totalCount: localCount, + totalSize: localSize, + }, + remote: { + totalCount: remoteCount, + totalSize: remoteSize, + } + }; + } + + @autobind + public async update(file: DriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalCount = isAdditional ? 1 : -1; + update.totalSize = isAdditional ? file.size : -file.size; + if (isAdditional) { + update.incCount = 1; + update.incSize = file.size; + } else { + update.decCount = 1; + update.decSize = file.size; + } + + await this.inc({ + [file.userHost === null ? 'local' : 'remote']: update + }); + } +} diff --git a/src/services/chart/charts/classes/federation.ts b/src/services/chart/charts/classes/federation.ts new file mode 100644 index 0000000000..bd2c497e7b --- /dev/null +++ b/src/services/chart/charts/classes/federation.ts @@ -0,0 +1,51 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { Instances } from '../../../../models'; +import { name, schema } from '../schemas/federation'; + +type FederationLog = SchemaType<typeof schema>; + +export default class FederationChart extends Chart<FederationLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: FederationLog): DeepPartial<FederationLog> { + return { + instance: { + total: latest.instance.total, + } + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<FederationLog>> { + const [total] = await Promise.all([ + Instances.count({}) + ]); + + return { + instance: { + total: total, + } + }; + } + + @autobind + public async update(isAdditional: boolean) { + const update: Obj = {}; + + update.total = isAdditional ? 1 : -1; + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + await this.inc({ + instance: update + }); + } +} diff --git a/src/services/chart/charts/classes/hashtag.ts b/src/services/chart/charts/classes/hashtag.ts new file mode 100644 index 0000000000..38c3a94f0c --- /dev/null +++ b/src/services/chart/charts/classes/hashtag.ts @@ -0,0 +1,35 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { User } from '../../../../models/entities/user'; +import { SchemaType } from '../../../../misc/schema'; +import { Users } from '../../../../models'; +import { name, schema } from '../schemas/hashtag'; + +type HashtagLog = SchemaType<typeof schema>; + +export default class HashtagChart extends Chart<HashtagLog> { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: HashtagLog): DeepPartial<HashtagLog> { + return {}; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<HashtagLog>> { + return {}; + } + + @autobind + public async update(hashtag: string, user: User) { + const update: Obj = { + count: 1 + }; + + await this.incIfUnique({ + [Users.isLocalUser(user) ? 'local' : 'remote']: update + }, 'users', user.id, hashtag); + } +} diff --git a/src/services/chart/charts/classes/instance.ts b/src/services/chart/charts/classes/instance.ts new file mode 100644 index 0000000000..f3d341f383 --- /dev/null +++ b/src/services/chart/charts/classes/instance.ts @@ -0,0 +1,173 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { DriveFiles, Followings, Users, Notes } from '../../../../models'; +import { DriveFile } from '../../../../models/entities/drive-file'; +import { name, schema } from '../schemas/instance'; +import { Note } from '../../../../models/entities/note'; +import { toPuny } from '../../../../misc/convert-host'; + +type InstanceLog = SchemaType<typeof schema>; + +export default class InstanceChart extends Chart<InstanceLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: InstanceLog): DeepPartial<InstanceLog> { + return { + notes: { + total: latest.notes.total, + }, + users: { + total: latest.users.total, + }, + following: { + total: latest.following.total, + }, + followers: { + total: latest.followers.total, + }, + drive: { + totalFiles: latest.drive.totalFiles, + totalUsage: latest.drive.totalUsage, + } + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<InstanceLog>> { + const [ + notesCount, + usersCount, + followingCount, + followersCount, + driveFiles, + driveUsage, + ] = await Promise.all([ + Notes.count({ userHost: group }), + Users.count({ host: group }), + Followings.count({ followerHost: group }), + Followings.count({ followeeHost: group }), + DriveFiles.count({ userHost: group }), + DriveFiles.clacDriveUsageOfHost(group), + ]); + + return { + notes: { + total: notesCount, + }, + users: { + total: usersCount, + }, + following: { + total: followingCount, + }, + followers: { + total: followersCount, + }, + drive: { + totalFiles: driveFiles, + totalUsage: driveUsage, + } + }; + } + + @autobind + public async requestReceived(host: string) { + await this.inc({ + requests: { + received: 1 + } + }, toPuny(host)); + } + + @autobind + public async requestSent(host: string, isSucceeded: boolean) { + const update: Obj = {}; + + if (isSucceeded) { + update.succeeded = 1; + } else { + update.failed = 1; + } + + await this.inc({ + requests: update + }, toPuny(host)); + } + + @autobind + public async newUser(host: string) { + await this.inc({ + users: { + total: 1, + inc: 1 + } + }, toPuny(host)); + } + + @autobind + public async updateNote(host: string, note: Note, isAdditional: boolean) { + const diffs = {} as any; + + if (note.replyId != null) { + diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + diffs.renote = isAdditional ? 1 : -1; + } else { + diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc({ + notes: { + total: isAdditional ? 1 : -1, + inc: isAdditional ? 1 : 0, + dec: isAdditional ? 0 : 1, + diffs: diffs + } + }, toPuny(host)); + } + + @autobind + public async updateFollowing(host: string, isAdditional: boolean) { + await this.inc({ + following: { + total: isAdditional ? 1 : -1, + inc: isAdditional ? 1 : 0, + dec: isAdditional ? 0 : 1, + } + }, toPuny(host)); + } + + @autobind + public async updateFollowers(host: string, isAdditional: boolean) { + await this.inc({ + followers: { + total: isAdditional ? 1 : -1, + inc: isAdditional ? 1 : 0, + dec: isAdditional ? 0 : 1, + } + }, toPuny(host)); + } + + @autobind + public async updateDrive(file: DriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalFiles = isAdditional ? 1 : -1; + update.totalUsage = isAdditional ? file.size : -file.size; + if (isAdditional) { + update.incFiles = 1; + update.incUsage = file.size; + } else { + update.decFiles = 1; + update.decUsage = file.size; + } + + await this.inc({ + drive: update + }, file.userHost); + } +} diff --git a/src/services/chart/charts/classes/network.ts b/src/services/chart/charts/classes/network.ts new file mode 100644 index 0000000000..8b26e5c4c2 --- /dev/null +++ b/src/services/chart/charts/classes/network.ts @@ -0,0 +1,34 @@ +import autobind from 'autobind-decorator'; +import Chart, { DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { name, schema } from '../schemas/network'; + +type NetworkLog = SchemaType<typeof schema>; + +export default class NetworkChart extends Chart<NetworkLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: NetworkLog): DeepPartial<NetworkLog> { + return {}; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<NetworkLog>> { + return {}; + } + + @autobind + public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number) { + const inc: DeepPartial<NetworkLog> = { + incomingRequests: incomingRequests, + totalTime: time, + incomingBytes: incomingBytes, + outgoingBytes: outgoingBytes + }; + + await this.inc(inc); + } +} diff --git a/src/services/chart/charts/classes/notes.ts b/src/services/chart/charts/classes/notes.ts new file mode 100644 index 0000000000..85ccf000d8 --- /dev/null +++ b/src/services/chart/charts/classes/notes.ts @@ -0,0 +1,71 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { Notes } from '../../../../models'; +import { Not } from 'typeorm'; +import { Note } from '../../../../models/entities/note'; +import { name, schema } from '../schemas/notes'; + +type NotesLog = SchemaType<typeof schema>; + +export default class NotesChart extends Chart<NotesLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: NotesLog): DeepPartial<NotesLog> { + return { + local: { + total: latest.local.total, + }, + remote: { + total: latest.remote.total, + } + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<NotesLog>> { + const [localCount, remoteCount] = await Promise.all([ + Notes.count({ userHost: null }), + Notes.count({ userHost: Not(null) }) + ]); + + return { + local: { + total: localCount, + }, + remote: { + total: remoteCount, + } + }; + } + + @autobind + public async update(note: Note, isAdditional: boolean) { + const update: Obj = { + diffs: {} + }; + + update.total = isAdditional ? 1 : -1; + + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + if (note.replyId != null) { + update.diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + update.diffs.renote = isAdditional ? 1 : -1; + } else { + update.diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc({ + [note.userHost === null ? 'local' : 'remote']: update + }); + } +} diff --git a/src/services/chart/charts/classes/per-user-drive.ts b/src/services/chart/charts/classes/per-user-drive.ts new file mode 100644 index 0000000000..822f4eda0f --- /dev/null +++ b/src/services/chart/charts/classes/per-user-drive.ts @@ -0,0 +1,52 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { DriveFiles } from '../../../../models'; +import { DriveFile } from '../../../../models/entities/drive-file'; +import { name, schema } from '../schemas/per-user-drive'; + +type PerUserDriveLog = SchemaType<typeof schema>; + +export default class PerUserDriveChart extends Chart<PerUserDriveLog> { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserDriveLog): DeepPartial<PerUserDriveLog> { + return { + totalCount: latest.totalCount, + totalSize: latest.totalSize, + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<PerUserDriveLog>> { + const [count, size] = await Promise.all([ + DriveFiles.count({ userId: group }), + DriveFiles.clacDriveUsageOf(group) + ]); + + return { + totalCount: count, + totalSize: size, + }; + } + + @autobind + public async update(file: DriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalCount = isAdditional ? 1 : -1; + update.totalSize = isAdditional ? file.size : -file.size; + if (isAdditional) { + update.incCount = 1; + update.incSize = file.size; + } else { + update.decCount = 1; + update.decSize = file.size; + } + + await this.inc(update, file.userId); + } +} diff --git a/src/services/chart/charts/classes/per-user-following.ts b/src/services/chart/charts/classes/per-user-following.ts new file mode 100644 index 0000000000..f3809a7c94 --- /dev/null +++ b/src/services/chart/charts/classes/per-user-following.ts @@ -0,0 +1,91 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { Followings, Users } from '../../../../models'; +import { Not } from 'typeorm'; +import { User } from '../../../../models/entities/user'; +import { name, schema } from '../schemas/per-user-following'; + +type PerUserFollowingLog = SchemaType<typeof schema>; + +export default class PerUserFollowingChart extends Chart<PerUserFollowingLog> { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserFollowingLog): DeepPartial<PerUserFollowingLog> { + return { + local: { + followings: { + total: latest.local.followings.total, + }, + followers: { + total: latest.local.followers.total, + } + }, + remote: { + followings: { + total: latest.remote.followings.total, + }, + followers: { + total: latest.remote.followers.total, + } + } + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<PerUserFollowingLog>> { + const [ + localFollowingsCount, + localFollowersCount, + remoteFollowingsCount, + remoteFollowersCount + ] = await Promise.all([ + Followings.count({ followerId: group, followeeHost: null }), + Followings.count({ followeeId: group, followerHost: null }), + Followings.count({ followerId: group, followeeHost: Not(null) }), + Followings.count({ followeeId: group, followerHost: Not(null) }) + ]); + + return { + local: { + followings: { + total: localFollowingsCount, + }, + followers: { + total: localFollowersCount, + } + }, + remote: { + followings: { + total: remoteFollowingsCount, + }, + followers: { + total: remoteFollowersCount, + } + } + }; + } + + @autobind + public async update(follower: User, followee: User, isFollow: boolean) { + const update: Obj = {}; + + update.total = isFollow ? 1 : -1; + + if (isFollow) { + update.inc = 1; + } else { + update.dec = 1; + } + + this.inc({ + [Users.isLocalUser(follower) ? 'local' : 'remote']: { followings: update } + }, follower.id); + this.inc({ + [Users.isLocalUser(followee) ? 'local' : 'remote']: { followers: update } + }, followee.id); + } +} diff --git a/src/services/chart/charts/classes/per-user-notes.ts b/src/services/chart/charts/classes/per-user-notes.ts new file mode 100644 index 0000000000..cccd495604 --- /dev/null +++ b/src/services/chart/charts/classes/per-user-notes.ts @@ -0,0 +1,58 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { User } from '../../../../models/entities/user'; +import { SchemaType } from '../../../../misc/schema'; +import { Notes } from '../../../../models'; +import { Note } from '../../../../models/entities/note'; +import { name, schema } from '../schemas/per-user-notes'; + +type PerUserNotesLog = SchemaType<typeof schema>; + +export default class PerUserNotesChart extends Chart<PerUserNotesLog> { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserNotesLog): DeepPartial<PerUserNotesLog> { + return { + total: latest.total, + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<PerUserNotesLog>> { + const [count] = await Promise.all([ + Notes.count({ userId: group }), + ]); + + return { + total: count, + }; + } + + @autobind + public async update(user: User, note: Note, isAdditional: boolean) { + const update: Obj = { + diffs: {} + }; + + update.total = isAdditional ? 1 : -1; + + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + if (note.replyId != null) { + update.diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + update.diffs.renote = isAdditional ? 1 : -1; + } else { + update.diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc(update, user.id); + } +} diff --git a/src/services/chart/charts/classes/per-user-reactions.ts b/src/services/chart/charts/classes/per-user-reactions.ts new file mode 100644 index 0000000000..124fb4153c --- /dev/null +++ b/src/services/chart/charts/classes/per-user-reactions.ts @@ -0,0 +1,32 @@ +import autobind from 'autobind-decorator'; +import Chart, { DeepPartial } from '../../core'; +import { User } from '../../../../models/entities/user'; +import { Note } from '../../../../models/entities/note'; +import { SchemaType } from '../../../../misc/schema'; +import { Users } from '../../../../models'; +import { name, schema } from '../schemas/per-user-reactions'; + +type PerUserReactionsLog = SchemaType<typeof schema>; + +export default class PerUserReactionsChart extends Chart<PerUserReactionsLog> { + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: PerUserReactionsLog): DeepPartial<PerUserReactionsLog> { + return {}; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<PerUserReactionsLog>> { + return {}; + } + + @autobind + public async update(user: User, note: Note) { + this.inc({ + [Users.isLocalUser(user) ? 'local' : 'remote']: { count: 1 } + }, note.userId); + } +} diff --git a/src/services/chart/charts/classes/test-grouped.ts b/src/services/chart/charts/classes/test-grouped.ts new file mode 100644 index 0000000000..e32cbcf416 --- /dev/null +++ b/src/services/chart/charts/classes/test-grouped.ts @@ -0,0 +1,47 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { name, schema } from '../schemas/test-grouped'; + +type TestGroupedLog = SchemaType<typeof schema>; + +export default class TestGroupedChart extends Chart<TestGroupedLog> { + private total = {} as Record<string, number>; + + constructor() { + super(name, schema, true); + } + + @autobind + protected genNewLog(latest: TestGroupedLog): DeepPartial<TestGroupedLog> { + return { + foo: { + total: latest.foo.total, + }, + }; + } + + @autobind + protected async fetchActual(group: string): Promise<DeepPartial<TestGroupedLog>> { + return { + foo: { + total: this.total[group], + }, + }; + } + + @autobind + public async increment(group: string) { + if (this.total[group] == null) this.total[group] = 0; + + const update: Obj = {}; + + update.total = 1; + update.inc = 1; + this.total[group]++; + + await this.inc({ + foo: update + }, group); + } +} diff --git a/src/services/chart/charts/classes/test-unique.ts b/src/services/chart/charts/classes/test-unique.ts new file mode 100644 index 0000000000..1eb396c293 --- /dev/null +++ b/src/services/chart/charts/classes/test-unique.ts @@ -0,0 +1,29 @@ +import autobind from 'autobind-decorator'; +import Chart, { DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { name, schema } from '../schemas/test-unique'; + +type TestUniqueLog = SchemaType<typeof schema>; + +export default class TestUniqueChart extends Chart<TestUniqueLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: TestUniqueLog): DeepPartial<TestUniqueLog> { + return {}; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<TestUniqueLog>> { + return {}; + } + + @autobind + public async uniqueIncrement(key: string) { + await this.incIfUnique({ + foo: 1 + }, 'foos', key); + } +} diff --git a/src/services/chart/charts/classes/test.ts b/src/services/chart/charts/classes/test.ts new file mode 100644 index 0000000000..57c22822f2 --- /dev/null +++ b/src/services/chart/charts/classes/test.ts @@ -0,0 +1,45 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { name, schema } from '../schemas/test'; + +type TestLog = SchemaType<typeof schema>; + +export default class TestChart extends Chart<TestLog> { + private total = 0; + + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: TestLog): DeepPartial<TestLog> { + return { + foo: { + total: latest.foo.total, + }, + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<TestLog>> { + return { + foo: { + total: this.total, + }, + }; + } + + @autobind + public async increment() { + const update: Obj = {}; + + update.total = 1; + update.inc = 1; + this.total++; + + await this.inc({ + foo: update + }); + } +} diff --git a/src/services/chart/charts/classes/users.ts b/src/services/chart/charts/classes/users.ts new file mode 100644 index 0000000000..eec30de8dc --- /dev/null +++ b/src/services/chart/charts/classes/users.ts @@ -0,0 +1,60 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj, DeepPartial } from '../../core'; +import { SchemaType } from '../../../../misc/schema'; +import { Users } from '../../../../models'; +import { Not } from 'typeorm'; +import { User } from '../../../../models/entities/user'; +import { name, schema } from '../schemas/users'; + +type UsersLog = SchemaType<typeof schema>; + +export default class UsersChart extends Chart<UsersLog> { + constructor() { + super(name, schema); + } + + @autobind + protected genNewLog(latest: UsersLog): DeepPartial<UsersLog> { + return { + local: { + total: latest.local.total, + }, + remote: { + total: latest.remote.total, + } + }; + } + + @autobind + protected async fetchActual(): Promise<DeepPartial<UsersLog>> { + const [localCount, remoteCount] = await Promise.all([ + Users.count({ host: null }), + Users.count({ host: Not(null) }) + ]); + + return { + local: { + total: localCount, + }, + remote: { + total: remoteCount, + } + }; + } + + @autobind + public async update(user: User, isAdditional: boolean) { + const update: Obj = {}; + + update.total = isAdditional ? 1 : -1; + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + await this.inc({ + [Users.isLocalUser(user) ? 'local' : 'remote']: update + }); + } +} diff --git a/src/services/chart/charts/schemas/active-users.ts b/src/services/chart/charts/schemas/active-users.ts new file mode 100644 index 0000000000..da8c63389c --- /dev/null +++ b/src/services/chart/charts/schemas/active-users.ts @@ -0,0 +1,28 @@ +export const logSchema = { + /** + * アクティブユーザー数 + */ + count: { + type: 'number' as 'number', + description: 'アクティブユーザー数', + }, +}; + +/** + * アクティブユーザーに関するチャート + */ +export const schema = { + type: 'object' as 'object', + properties: { + local: { + type: 'object' as 'object', + properties: logSchema + }, + remote: { + type: 'object' as 'object', + properties: logSchema + }, + } +}; + +export const name = 'activeUsers'; diff --git a/src/services/chart/charts/schemas/drive.ts b/src/services/chart/charts/schemas/drive.ts new file mode 100644 index 0000000000..47530e8417 --- /dev/null +++ b/src/services/chart/charts/schemas/drive.ts @@ -0,0 +1,65 @@ +const logSchema = { + /** + * 集計期間時点での、全ドライブファイル数 + */ + totalCount: { + type: 'number' as 'number', + description: '集計期間時点での、全ドライブファイル数' + }, + + /** + * 集計期間時点での、全ドライブファイルの合計サイズ + */ + totalSize: { + type: 'number' as 'number', + description: '集計期間時点での、全ドライブファイルの合計サイズ' + }, + + /** + * 増加したドライブファイル数 + */ + incCount: { + type: 'number' as 'number', + description: '増加したドライブファイル数' + }, + + /** + * 増加したドライブ使用量 + */ + incSize: { + type: 'number' as 'number', + description: '増加したドライブ使用量' + }, + + /** + * 減少したドライブファイル数 + */ + decCount: { + type: 'number' as 'number', + description: '減少したドライブファイル数' + }, + + /** + * 減少したドライブ使用量 + */ + decSize: { + type: 'number' as 'number', + description: '減少したドライブ使用量' + }, +}; + +export const schema = { + type: 'object' as 'object', + properties: { + local: { + type: 'object' as 'object', + properties: logSchema + }, + remote: { + type: 'object' as 'object', + properties: logSchema + }, + } +}; + +export const name = 'drive'; diff --git a/src/services/chart/charts/schemas/federation.ts b/src/services/chart/charts/schemas/federation.ts new file mode 100644 index 0000000000..d1d275fc95 --- /dev/null +++ b/src/services/chart/charts/schemas/federation.ts @@ -0,0 +1,27 @@ +/** + * フェデレーションに関するチャート + */ +export const schema = { + type: 'object' as 'object', + properties: { + instance: { + type: 'object' as 'object', + properties: { + total: { + type: 'number' as 'number', + description: 'インスタンス数の合計' + }, + inc: { + type: 'number' as 'number', + description: '増加インスタンス数' + }, + dec: { + type: 'number' as 'number', + description: '減少インスタンス数' + }, + } + } + } +}; + +export const name = 'federation'; diff --git a/src/services/chart/charts/schemas/hashtag.ts b/src/services/chart/charts/schemas/hashtag.ts new file mode 100644 index 0000000000..c1904b6701 --- /dev/null +++ b/src/services/chart/charts/schemas/hashtag.ts @@ -0,0 +1,28 @@ +export const logSchema = { + /** + * 投稿された数 + */ + count: { + type: 'number' as 'number', + description: '投稿された数', + }, +}; + +/** + * ハッシュタグに関するチャート + */ +export const schema = { + type: 'object' as 'object', + properties: { + local: { + type: 'object' as 'object', + properties: logSchema + }, + remote: { + type: 'object' as 'object', + properties: logSchema + }, + } +}; + +export const name = 'hashtag'; diff --git a/src/services/chart/charts/schemas/instance.ts b/src/services/chart/charts/schemas/instance.ts new file mode 100644 index 0000000000..001f2428b5 --- /dev/null +++ b/src/services/chart/charts/schemas/instance.ts @@ -0,0 +1,149 @@ +/** + * インスタンスごとのチャート + */ +export const schema = { + type: 'object' as 'object', + properties: { + requests: { + type: 'object' as 'object', + properties: { + failed: { + type: 'number' as 'number', + description: '失敗したリクエスト数' + }, + succeeded: { + type: 'number' as 'number', + description: '成功したリクエスト数' + }, + received: { + type: 'number' as 'number', + description: '受信したリクエスト数' + }, + } + }, + + notes: { + type: 'object' as 'object', + properties: { + total: { + type: 'number' as 'number', + description: '集計期間時点での、全投稿数' + }, + inc: { + type: 'number' as 'number', + description: '増加した投稿数' + }, + dec: { + type: 'number' as 'number', + description: '減少した投稿数' + }, + + diffs: { + type: 'object' as 'object', + properties: { + normal: { + type: 'number' as 'number', + description: '通常の投稿数の差分' + }, + + reply: { + type: 'number' as 'number', + description: 'リプライの投稿数の差分' + }, + + renote: { + type: 'number' as 'number', + description: 'Renoteの投稿数の差分' + }, + } + }, + } + }, + + users: { + type: 'object' as 'object', + properties: { + total: { + type: 'number' as 'number', + description: '集計期間時点での、全ユーザー数' + }, + inc: { + type: 'number' as 'number', + description: '増加したユーザー数' + }, + dec: { + type: 'number' as 'number', + description: '減少したユーザー数' + }, + } + }, + + following: { + type: 'object' as 'object', + properties: { + total: { + type: 'number' as 'number', + description: '集計期間時点での、全フォロー数' + }, + inc: { + type: 'number' as 'number', + description: '増加したフォロー数' + }, + dec: { + type: 'number' as 'number', + description: '減少したフォロー数' + }, + } + }, + + followers: { + type: 'object' as 'object', + properties: { + total: { + type: 'number' as 'number', + description: '集計期間時点での、全フォロワー数' + }, + inc: { + type: 'number' as 'number', + description: '増加したフォロワー数' + }, + dec: { + type: 'number' as 'number', + description: '減少したフォロワー数' + }, + } + }, + + drive: { + type: 'object' as 'object', + properties: { + totalFiles: { + type: 'number' as 'number', + description: '集計期間時点での、全ドライブファイル数' + }, + totalUsage: { + type: 'number' as 'number', + description: '集計期間時点での、全ドライブファイルの合計サイズ' + }, + incFiles: { + type: 'number' as 'number', + description: '増加したドライブファイル数' + }, + incUsage: { + type: 'number' as 'number', + description: '増加したドライブ使用量' + }, + decFiles: { + type: 'number' as 'number', + description: '減少したドライブファイル数' + }, + decUsage: { + type: 'number' as 'number', + description: '減少したドライブ使用量' + }, + } + }, + } +}; + +export const name = 'instance'; diff --git a/src/services/chart/charts/schemas/network.ts b/src/services/chart/charts/schemas/network.ts new file mode 100644 index 0000000000..4ef530c07c --- /dev/null +++ b/src/services/chart/charts/schemas/network.ts @@ -0,0 +1,30 @@ +/** + * ネットワークに関するチャート + */ +export const schema = { + type: 'object' as 'object', + properties: { + incomingRequests: { + type: 'number' as 'number', + description: '受信したリクエスト数' + }, + outgoingRequests: { + type: 'number' as 'number', + description: '送信したリクエスト数' + }, + totalTime: { + type: 'number' as 'number', + description: '応答時間の合計' // TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる + }, + incomingBytes: { + type: 'number' as 'number', + description: '合計受信データ量' + }, + outgoingBytes: { + type: 'number' as 'number', + description: '合計送信データ量' + }, + } +}; + +export const name = 'network'; diff --git a/src/services/chart/charts/schemas/notes.ts b/src/services/chart/charts/schemas/notes.ts new file mode 100644 index 0000000000..133d1e3730 --- /dev/null +++ b/src/services/chart/charts/schemas/notes.ts @@ -0,0 +1,52 @@ +const logSchema = { + total: { + type: 'number' as 'number', + description: '集計期間時点での、全投稿数' + }, + + inc: { + type: 'number' as 'number', + description: '増加した投稿数' + }, + + dec: { + type: 'number' as 'number', + description: '減少した投稿数' + }, + + diffs: { + type: 'object' as 'object', + properties: { + normal: { + type: 'number' as 'number', + description: '通常の投稿数の差分' + }, + + reply: { + type: 'number' as 'number', + description: 'リプライの投稿数の差分' + }, + + renote: { + type: 'number' as 'number', + description: 'Renoteの投稿数の差分' + }, + } + }, +}; + +export const schema = { + type: 'object' as 'object', + properties: { + local: { + type: 'object' as 'object', + properties: logSchema + }, + remote: { + type: 'object' as 'object', + properties: logSchema + }, + } +}; + +export const name = 'notes'; diff --git a/src/services/chart/charts/schemas/per-user-drive.ts b/src/services/chart/charts/schemas/per-user-drive.ts new file mode 100644 index 0000000000..713bd7ed84 --- /dev/null +++ b/src/services/chart/charts/schemas/per-user-drive.ts @@ -0,0 +1,54 @@ +export const schema = { + type: 'object' as 'object', + properties: { + /** + * 集計期間時点での、全ドライブファイル数 + */ + totalCount: { + type: 'number' as 'number', + description: '集計期間時点での、全ドライブファイル数' + }, + + /** + * 集計期間時点での、全ドライブファイルの合計サイズ + */ + totalSize: { + type: 'number' as 'number', + description: '集計期間時点での、全ドライブファイルの合計サイズ' + }, + + /** + * 増加したドライブファイル数 + */ + incCount: { + type: 'number' as 'number', + description: '増加したドライブファイル数' + }, + + /** + * 増加したドライブ使用量 + */ + incSize: { + type: 'number' as 'number', + description: '増加したドライブ使用量' + }, + + /** + * 減少したドライブファイル数 + */ + decCount: { + type: 'number' as 'number', + description: '減少したドライブファイル数' + }, + + /** + * 減少したドライブ使用量 + */ + decSize: { + type: 'number' as 'number', + description: '減少したドライブ使用量' + }, + } +}; + +export const name = 'perUserDrive'; diff --git a/src/services/chart/charts/schemas/per-user-following.ts b/src/services/chart/charts/schemas/per-user-following.ts new file mode 100644 index 0000000000..d6ca1130e0 --- /dev/null +++ b/src/services/chart/charts/schemas/per-user-following.ts @@ -0,0 +1,81 @@ +export const logSchema = { + /** + * フォローしている + */ + followings: { + type: 'object' as 'object', + properties: { + /** + * フォローしている合計 + */ + total: { + type: 'number' as 'number', + description: 'フォローしている合計', + }, + + /** + * フォローした数 + */ + inc: { + type: 'number' as 'number', + description: 'フォローした数', + }, + + /** + * フォロー解除した数 + */ + dec: { + type: 'number' as 'number', + description: 'フォロー解除した数', + }, + } + }, + + /** + * フォローされている + */ + followers: { + type: 'object' as 'object', + properties: { + /** + * フォローされている合計 + */ + total: { + type: 'number' as 'number', + description: 'フォローされている合計', + }, + + /** + * フォローされた数 + */ + inc: { + type: 'number' as 'number', + description: 'フォローされた数', + }, + + /** + * フォロー解除された数 + */ + dec: { + type: 'number' as 'number', + description: 'フォロー解除された数', + }, + } + }, +}; + +export const schema = { + type: 'object' as 'object', + properties: { + local: { + type: 'object' as 'object', + properties: logSchema + }, + remote: { + type: 'object' as 'object', + properties: logSchema + }, + } +}; + +export const name = 'perUserFollowing'; diff --git a/src/services/chart/charts/schemas/per-user-notes.ts b/src/services/chart/charts/schemas/per-user-notes.ts new file mode 100644 index 0000000000..3c448c4cee --- /dev/null +++ b/src/services/chart/charts/schemas/per-user-notes.ts @@ -0,0 +1,41 @@ +export const schema = { + type: 'object' as 'object', + properties: { + total: { + type: 'number' as 'number', + description: '集計期間時点での、全投稿数' + }, + + inc: { + type: 'number' as 'number', + description: '増加した投稿数' + }, + + dec: { + type: 'number' as 'number', + description: '減少した投稿数' + }, + + diffs: { + type: 'object' as 'object', + properties: { + normal: { + type: 'number' as 'number', + description: '通常の投稿数の差分' + }, + + reply: { + type: 'number' as 'number', + description: 'リプライの投稿数の差分' + }, + + renote: { + type: 'number' as 'number', + description: 'Renoteの投稿数の差分' + }, + } + }, + } +}; + +export const name = 'perUserNotes'; diff --git a/src/services/chart/charts/schemas/per-user-reactions.ts b/src/services/chart/charts/schemas/per-user-reactions.ts new file mode 100644 index 0000000000..1278184da6 --- /dev/null +++ b/src/services/chart/charts/schemas/per-user-reactions.ts @@ -0,0 +1,28 @@ +export const logSchema = { + /** + * フォローしている合計 + */ + count: { + type: 'number' as 'number', + description: 'リアクションされた数', + }, +}; + +/** + * ユーザーごとのリアクションに関するチャート + */ +export const schema = { + type: 'object' as 'object', + properties: { + local: { + type: 'object' as 'object', + properties: logSchema + }, + remote: { + type: 'object' as 'object', + properties: logSchema + }, + } +}; + +export const name = 'perUserReaction'; diff --git a/src/services/chart/charts/schemas/test-grouped.ts b/src/services/chart/charts/schemas/test-grouped.ts new file mode 100644 index 0000000000..acf3fddb31 --- /dev/null +++ b/src/services/chart/charts/schemas/test-grouped.ts @@ -0,0 +1,26 @@ +export const schema = { + type: 'object' as 'object', + properties: { + foo: { + type: 'object' as 'object', + properties: { + total: { + type: 'number' as 'number', + description: '' + }, + + inc: { + type: 'number' as 'number', + description: '' + }, + + dec: { + type: 'number' as 'number', + description: '' + }, + } + } + } +}; + +export const name = 'testGrouped'; diff --git a/src/services/chart/charts/schemas/test-unique.ts b/src/services/chart/charts/schemas/test-unique.ts new file mode 100644 index 0000000000..8fcfbf3c72 --- /dev/null +++ b/src/services/chart/charts/schemas/test-unique.ts @@ -0,0 +1,11 @@ +export const schema = { + type: 'object' as 'object', + properties: { + foo: { + type: 'number' as 'number', + description: '' + }, + } +}; + +export const name = 'testUnique'; diff --git a/src/services/chart/charts/schemas/test.ts b/src/services/chart/charts/schemas/test.ts new file mode 100644 index 0000000000..b1344500bf --- /dev/null +++ b/src/services/chart/charts/schemas/test.ts @@ -0,0 +1,26 @@ +export const schema = { + type: 'object' as 'object', + properties: { + foo: { + type: 'object' as 'object', + properties: { + total: { + type: 'number' as 'number', + description: '' + }, + + inc: { + type: 'number' as 'number', + description: '' + }, + + dec: { + type: 'number' as 'number', + description: '' + }, + } + } + } +}; + +export const name = 'test'; diff --git a/src/services/chart/charts/schemas/users.ts b/src/services/chart/charts/schemas/users.ts new file mode 100644 index 0000000000..db7e2dd057 --- /dev/null +++ b/src/services/chart/charts/schemas/users.ts @@ -0,0 +1,41 @@ +const logSchema = { + /** + * 集計期間時点での、全ユーザー数 + */ + total: { + type: 'number' as 'number', + description: '集計期間時点での、全ユーザー数' + }, + + /** + * 増加したユーザー数 + */ + inc: { + type: 'number' as 'number', + description: '増加したユーザー数' + }, + + /** + * 減少したユーザー数 + */ + dec: { + type: 'number' as 'number', + description: '減少したユーザー数' + }, +}; + +export const schema = { + type: 'object' as 'object', + properties: { + local: { + type: 'object' as 'object', + properties: logSchema + }, + remote: { + type: 'object' as 'object', + properties: logSchema + }, + } +}; + +export const name = 'users'; diff --git a/src/services/chart/core.ts b/src/services/chart/core.ts new file mode 100644 index 0000000000..0a9ec8dae0 --- /dev/null +++ b/src/services/chart/core.ts @@ -0,0 +1,460 @@ +/** + * チャートエンジン + * + * Tests located in test/chart + */ + +import * as moment from 'moment'; +import * as nestedProperty from 'nested-property'; +import autobind from 'autobind-decorator'; +import Logger from '../logger'; +import { Schema } from '../../misc/schema'; +import { EntitySchema, getRepository, Repository, LessThan, MoreThanOrEqual } from 'typeorm'; +import { isDuplicateKeyValueError } from '../../misc/is-duplicate-key-value-error'; + +const logger = new Logger('chart', 'white', process.env.NODE_ENV !== 'test'); + +const utc = moment.utc; + +export type Obj = { [key: string]: any }; + +export type DeepPartial<T> = { + [P in keyof T]?: DeepPartial<T[P]>; +}; + +type ArrayValue<T> = { + [P in keyof T]: T[P] extends number ? T[P][] : ArrayValue<T[P]>; +}; + +type Span = 'day' | 'hour'; + +type Log = { + id: number; + + /** + * 集計のグループ + */ + group: string | null; + + /** + * 集計日時のUnixタイムスタンプ(秒) + */ + date: number; + + /** + * 集計期間 + */ + span: Span; + + /** + * ユニークインクリメント用 + */ + unique?: Record<string, any>; +}; + +const camelToSnake = (str: string) => { + return str.replace(/([A-Z])/g, s => '_' + s.charAt(0).toLowerCase()); +}; + +/** + * 様々なチャートの管理を司るクラス + */ +export default abstract class Chart<T extends Record<string, any>> { + private static readonly columnPrefix = '___'; + private static readonly columnDot = '_'; + + private name: string; + public schema: Schema; + protected repository: Repository<Log>; + protected abstract genNewLog(latest: T): DeepPartial<T>; + protected abstract async fetchActual(group?: string): Promise<DeepPartial<T>>; + + @autobind + private static convertSchemaToFlatColumnDefinitions(schema: Schema) { + const columns = {} as any; + const flatColumns = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}${this.columnDot}${k}` : k; + if (v.type === 'object') { + flatColumns(v.properties, p); + } else { + columns[this.columnPrefix + p] = { + type: 'integer', + }; + } + } + }; + flatColumns(schema.properties!); + return columns; + } + + @autobind + private static convertFlattenColumnsToObject(x: Record<string, number>) { + const obj = {} as any; + for (const k of Object.keys(x).filter(k => k.startsWith(Chart.columnPrefix))) { + // now k is ___x_y_z + const path = k.substr(Chart.columnPrefix.length).split(Chart.columnDot).join('.'); + nestedProperty.set(obj, path, x[k]); + } + return obj; + } + + @autobind + private static convertObjectToFlattenColumns(x: Record<string, any>) { + const columns = {} as Record<string, number>; + const flatten = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}${this.columnDot}${k}` : k; + if (typeof v === 'object') { + flatten(v, p); + } else { + columns[this.columnPrefix + p] = v; + } + } + }; + flatten(x); + return columns; + } + + @autobind + private static convertQuery(x: Record<string, any>) { + const query: Record<string, Function> = {}; + + const columns = Chart.convertObjectToFlattenColumns(x); + + for (const [k, v] of Object.entries(columns)) { + if (v > 0) query[k] = () => `"${k}" + ${v}`; + if (v < 0) query[k] = () => `"${k}" - ${v}`; + } + + return query; + } + + @autobind + private static momentToTimestamp(x: moment.Moment): Log['date'] { + return x.unix(); + } + + @autobind + public static schemaToEntity(name: string, schema: Schema): EntitySchema { + return new EntitySchema({ + name: `__chart__${camelToSnake(name)}`, + columns: { + id: { + type: 'integer', + primary: true, + generated: true + }, + date: { + type: 'integer', + }, + group: { + type: 'varchar', + length: 128, + nullable: true + }, + span: { + type: 'enum', + enum: ['hour', 'day'] + }, + unique: { + type: 'jsonb', + default: {} + }, + ...Chart.convertSchemaToFlatColumnDefinitions(schema) + }, + }); + } + + constructor(name: string, schema: Schema, grouped = false) { + this.name = name; + this.schema = schema; + const entity = Chart.schemaToEntity(name, schema); + + const keys = ['span', 'date']; + if (grouped) keys.push('group'); + + entity.options.uniques = [{ + columns: keys + }]; + + this.repository = getRepository<Log>(entity); + } + + @autobind + private getNewLog(latest: T | null): T { + const log = latest ? this.genNewLog(latest) : {}; + const flatColumns = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}.${k}` : k; + if (v.type === 'object') { + flatColumns(v.properties, p); + } else { + if (nestedProperty.get(log, p) == null) { + nestedProperty.set(log, p, 0); + } + } + } + }; + flatColumns(this.schema.properties!); + return log as T; + } + + @autobind + private getCurrentDate(): [number, number, number, number] { + const now = moment().utc(); + + const y = now.year(); + const m = now.month(); + const d = now.date(); + const h = now.hour(); + + return [y, m, d, h]; + } + + @autobind + private getLatestLog(span: Span, group: string | null = null): Promise<Log | null> { + return this.repository.findOne({ + group: group, + span: span + }, { + order: { + date: -1 + } + }).then(x => x || null); + } + + @autobind + private async getCurrentLog(span: Span, group: string | null = null): Promise<Log> { + const [y, m, d, h] = this.getCurrentDate(); + + const current = + span == 'day' ? utc([y, m, d]) : + span == 'hour' ? utc([y, m, d, h]) : + null as never; + + // 現在(今日または今のHour)のログ + const currentLog = await this.repository.findOne({ + span: span, + date: Chart.momentToTimestamp(current), + ...(group ? { group: group } : {}) + }); + + // ログがあればそれを返して終了 + if (currentLog != null) { + return currentLog; + } + + let log: Log; + let data: T; + + // 集計期間が変わってから、初めてのチャート更新なら + // 最も最近のログを持ってくる + // * 例えば集計期間が「日」である場合で考えると、 + // * 昨日何もチャートを更新するような出来事がなかった場合は、 + // * ログがそもそも作られずドキュメントが存在しないということがあり得るため、 + // * 「昨日の」と決め打ちせずに「もっとも最近の」とします + const latest = await this.getLatestLog(span, group); + + if (latest != null) { + const obj = Chart.convertFlattenColumnsToObject( + latest as Record<string, any>); + + // 空ログデータを作成 + data = await this.getNewLog(obj); + } else { + // ログが存在しなかったら + // (Misskeyインスタンスを建てて初めてのチャート更新時) + + // 初期ログデータを作成 + data = await this.getNewLog(null); + + logger.info(`${this.name}: Initial commit created`); + } + + try { + // 新規ログ挿入 + log = await this.repository.save({ + group: group, + span: span, + date: Chart.momentToTimestamp(current), + ...Chart.convertObjectToFlattenColumns(data) + }); + } catch (e) { + // duplicate key error + // 並列動作している他のチャートエンジンプロセスと処理が重なる場合がある + // その場合は再度最も新しいログを持ってくる + if (isDuplicateKeyValueError(e)) { + log = await this.getLatestLog(span, group) as Log; + } else { + logger.error(e); + throw e; + } + } + + return log; + } + + @autobind + protected commit(query: Record<string, Function>, group: string | null = null, uniqueKey?: string, uniqueValue?: string): Promise<any> { + const update = async (log: Log) => { + // ユニークインクリメントの場合、指定のキーに指定の値が既に存在していたら弾く + if ( + uniqueKey && log.unique && + log.unique[uniqueKey] && + log.unique[uniqueKey].includes(uniqueValue) + ) return; + + // ユニークインクリメントの指定のキーに値を追加 + if (uniqueKey && log.unique) { + if (log.unique[uniqueKey]) { + const sql = `jsonb_set("unique", '{${uniqueKey}}', ("unique"->>'${uniqueKey}')::jsonb || '["${uniqueValue}"]'::jsonb)`; + query['unique'] = () => sql; + } else { + const sql = `jsonb_set("unique", '{${uniqueKey}}', '["${uniqueValue}"]')`; + query['unique'] = () => sql; + } + } + + // ログ更新 + await this.repository.createQueryBuilder() + .update() + .set(query) + .where('id = :id', { id: log.id }) + .execute(); + }; + + return Promise.all([ + this.getCurrentLog('day', group).then(log => update(log)), + this.getCurrentLog('hour', group).then(log => update(log)), + ]); + } + + @autobind + protected async inc(inc: DeepPartial<T>, group: string | null = null): Promise<void> { + await this.commit(Chart.convertQuery(inc as any), group); + } + + @autobind + protected async incIfUnique(inc: DeepPartial<T>, key: string, value: string, group: string | null = null): Promise<void> { + await this.commit(Chart.convertQuery(inc as any), group, key, value); + } + + @autobind + public async getChart(span: Span, range: number, group: string | null = null): Promise<ArrayValue<T>> { + const [y, m, d, h] = this.getCurrentDate(); + + const gt = + span == 'day' ? utc([y, m, d]).subtract(range, 'days') : + span == 'hour' ? utc([y, m, d, h]).subtract(range, 'hours') : + null as never; + + // ログ取得 + let logs = await this.repository.find({ + where: { + group: group, + span: span, + date: MoreThanOrEqual(Chart.momentToTimestamp(gt)) + }, + order: { + date: -1 + }, + }); + + // 要求された範囲にログがひとつもなかったら + if (logs.length === 0) { + // もっとも新しいログを持ってくる + // (すくなくともひとつログが無いと隙間埋めできないため) + const recentLog = await this.repository.findOne({ + group: group, + span: span + }, { + order: { + date: -1 + }, + }); + + if (recentLog) { + logs = [recentLog]; + } + + // 要求された範囲の最も古い箇所に位置するログが存在しなかったら + } else if (!utc(logs[logs.length - 1].date * 1000).isSame(gt)) { + // 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する + // (隙間埋めできないため) + const outdatedLog = await this.repository.findOne({ + group: group, + span: span, + date: LessThan(Chart.momentToTimestamp(gt)) + }, { + order: { + date: -1 + }, + }); + + if (outdatedLog) { + logs.push(outdatedLog); + } + } + + const chart: T[] = []; + + // 整形 + for (let i = (range - 1); i >= 0; i--) { + const current = + span == 'day' ? utc([y, m, d]).subtract(i, 'days') : + span == 'hour' ? utc([y, m, d, h]).subtract(i, 'hours') : + null as never; + + const log = logs.find(l => utc(l.date * 1000).isSame(current)); + + if (log) { + const data = Chart.convertFlattenColumnsToObject(log as Record<string, any>); + chart.unshift(data); + } else { + // 隙間埋め + const latest = logs.find(l => utc(l.date * 1000).isBefore(current)); + const data = latest ? Chart.convertFlattenColumnsToObject(latest as Record<string, any>) : null; + chart.unshift(this.getNewLog(data)); + } + } + + const res: ArrayValue<T> = {} as any; + + /** + * [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }] + * を + * { foo: [1, 2, 3], bar: [5, 6, 7] } + * にする + */ + const dive = (x: Obj, path?: string) => { + for (const [k, v] of Object.entries(x)) { + const p = path ? `${path}.${k}` : k; + if (typeof v == 'object') { + dive(v, p); + } else { + nestedProperty.set(res, p, chart.map(s => nestedProperty.get(s, p))); + } + } + }; + + dive(chart[0]); + + return res; + } +} + +export function convertLog(logSchema: Schema): Schema { + const v: Schema = JSON.parse(JSON.stringify(logSchema)); // copy + if (v.type === 'number') { + v.type = 'array'; + v.items = { + type: 'number' + }; + } else if (v.type === 'object') { + for (const k of Object.keys(v.properties!)) { + v.properties![k] = convertLog(v.properties![k]); + } + } + return v; +} diff --git a/src/services/chart/drive.ts b/src/services/chart/drive.ts deleted file mode 100644 index dd23412c7d..0000000000 --- a/src/services/chart/drive.ts +++ /dev/null @@ -1,150 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Obj } from './'; -import DriveFile, { IDriveFile } from '../../models/drive-file'; -import { isLocalUser } from '../../models/user'; -import { SchemaType } from '../../misc/schema'; - -const logSchema = { - /** - * 集計期間時点での、全ドライブファイル数 - */ - totalCount: { - type: 'number' as 'number', - description: '集計期間時点での、全ドライブファイル数' - }, - - /** - * 集計期間時点での、全ドライブファイルの合計サイズ - */ - totalSize: { - type: 'number' as 'number', - description: '集計期間時点での、全ドライブファイルの合計サイズ' - }, - - /** - * 増加したドライブファイル数 - */ - incCount: { - type: 'number' as 'number', - description: '増加したドライブファイル数' - }, - - /** - * 増加したドライブ使用量 - */ - incSize: { - type: 'number' as 'number', - description: '増加したドライブ使用量' - }, - - /** - * 減少したドライブファイル数 - */ - decCount: { - type: 'number' as 'number', - description: '減少したドライブファイル数' - }, - - /** - * 減少したドライブ使用量 - */ - decSize: { - type: 'number' as 'number', - description: '減少したドライブ使用量' - }, -}; - -export const driveLogSchema = { - type: 'object' as 'object', - properties: { - local: { - type: 'object' as 'object', - properties: logSchema - }, - remote: { - type: 'object' as 'object', - properties: logSchema - }, - } -}; - -type DriveLog = SchemaType<typeof driveLogSchema>; - -class DriveChart extends Chart<DriveLog> { - constructor() { - super('drive'); - } - - @autobind - protected async getTemplate(init: boolean, latest?: DriveLog): Promise<DriveLog> { - const calcSize = (local: boolean) => DriveFile - .aggregate([{ - $match: { - 'metadata._user.host': local ? null : { $ne: null }, - 'metadata.deletedAt': { $exists: false } - } - }, { - $project: { - length: true - } - }, { - $group: { - _id: null, - usage: { $sum: '$length' } - } - }]) - .then(res => res.length > 0 ? res[0].usage : 0); - - const [localCount, remoteCount, localSize, remoteSize] = init ? await Promise.all([ - DriveFile.count({ 'metadata._user.host': null }), - DriveFile.count({ 'metadata._user.host': { $ne: null } }), - calcSize(true), - calcSize(false) - ]) : [ - latest ? latest.local.totalCount : 0, - latest ? latest.remote.totalCount : 0, - latest ? latest.local.totalSize : 0, - latest ? latest.remote.totalSize : 0 - ]; - - return { - local: { - totalCount: localCount, - totalSize: localSize, - incCount: 0, - incSize: 0, - decCount: 0, - decSize: 0 - }, - remote: { - totalCount: remoteCount, - totalSize: remoteSize, - incCount: 0, - incSize: 0, - decCount: 0, - decSize: 0 - } - }; - } - - @autobind - public async update(file: IDriveFile, isAdditional: boolean) { - const update: Obj = {}; - - update.totalCount = isAdditional ? 1 : -1; - update.totalSize = isAdditional ? file.length : -file.length; - if (isAdditional) { - update.incCount = 1; - update.incSize = file.length; - } else { - update.decCount = 1; - update.decSize = file.length; - } - - await this.inc({ - [isLocalUser(file.metadata._user) ? 'local' : 'remote']: update - }); - } -} - -export default new DriveChart(); diff --git a/src/services/chart/entities.ts b/src/services/chart/entities.ts new file mode 100644 index 0000000000..14fd3adba0 --- /dev/null +++ b/src/services/chart/entities.ts @@ -0,0 +1,8 @@ +import Chart from './core'; + +export const entities = Object.values(require('require-all')({ + dirname: __dirname + '/charts/schemas', + resolve: (x: any) => { + return Chart.schemaToEntity(x.name, x.schema); + } +})); diff --git a/src/services/chart/federation.ts b/src/services/chart/federation.ts deleted file mode 100644 index 20da7a7421..0000000000 --- a/src/services/chart/federation.ts +++ /dev/null @@ -1,66 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Obj } from '.'; -import Instance from '../../models/instance'; - -/** - * フェデレーションに関するチャート - */ -type FederationLog = { - instance: { - /** - * インスタンス数の合計 - */ - total: number; - - /** - * 増加インスタンス数 - */ - inc: number; - - /** - * 減少インスタンス数 - */ - dec: number; - }; -}; - -class FederationChart extends Chart<FederationLog> { - constructor() { - super('federation'); - } - - @autobind - protected async getTemplate(init: boolean, latest?: FederationLog): Promise<FederationLog> { - const [total] = init ? await Promise.all([ - Instance.count({}) - ]) : [ - latest ? latest.instance.total : 0 - ]; - - return { - instance: { - total: total, - inc: 0, - dec: 0 - } - }; - } - - @autobind - public async update(isAdditional: boolean) { - const update: Obj = {}; - - update.total = isAdditional ? 1 : -1; - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } - - await this.inc({ - instance: update - }); - } -} - -export default new FederationChart(); diff --git a/src/services/chart/hashtag.ts b/src/services/chart/hashtag.ts deleted file mode 100644 index 7a31e9cced..0000000000 --- a/src/services/chart/hashtag.ts +++ /dev/null @@ -1,56 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Obj } from './'; -import { IUser, isLocalUser } from '../../models/user'; -import db from '../../db/mongodb'; - -/** - * ハッシュタグに関するチャート - */ -type HashtagLog = { - local: { - /** - * 投稿された数 - */ - count: number; - }; - - remote: HashtagLog['local']; -}; - -class HashtagChart extends Chart<HashtagLog> { - constructor() { - super('hashtag', true); - - // 後方互換性のため - db.get('chart.hashtag').findOne().then(doc => { - if (doc != null && doc.data.local == null) { - db.get('chart.hashtag').drop(); - } - }); - } - - @autobind - protected async getTemplate(init: boolean, latest?: HashtagLog): Promise<HashtagLog> { - return { - local: { - count: 0 - }, - remote: { - count: 0 - } - }; - } - - @autobind - public async update(hashtag: string, user: IUser) { - const update: Obj = { - count: 1 - }; - - await this.incIfUnique({ - [isLocalUser(user) ? 'local' : 'remote']: update - }, 'users', user._id.toHexString(), hashtag); - } -} - -export default new HashtagChart(); diff --git a/src/services/chart/index.ts b/src/services/chart/index.ts index 7a6470f4d8..9626e3d6b3 100644 --- a/src/services/chart/index.ts +++ b/src/services/chart/index.ts @@ -1,364 +1,25 @@ -/** - * チャートエンジン - */ +import FederationChart from './charts/classes/federation'; +import NotesChart from './charts/classes/notes'; +import UsersChart from './charts/classes/users'; +import NetworkChart from './charts/classes/network'; +import ActiveUsersChart from './charts/classes/active-users'; +import InstanceChart from './charts/classes/instance'; +import PerUserNotesChart from './charts/classes/per-user-notes'; +import DriveChart from './charts/classes/drive'; +import PerUserReactionsChart from './charts/classes/per-user-reactions'; +import HashtagChart from './charts/classes/hashtag'; +import PerUserFollowingChart from './charts/classes/per-user-following'; +import PerUserDriveChart from './charts/classes/per-user-drive'; -import * as moment from 'moment'; -import * as nestedProperty from 'nested-property'; -import autobind from 'autobind-decorator'; -import * as mongo from 'mongodb'; -import db from '../../db/mongodb'; -import { ICollection } from 'monk'; -import Logger from '../logger'; -import { Schema } from '../../misc/schema'; - -const logger = new Logger('chart'); - -const utc = moment.utc; - -export type Obj = { [key: string]: any }; - -export type Partial<T> = { - [P in keyof T]?: Partial<T[P]>; -}; - -type ArrayValue<T> = { - [P in keyof T]: T[P] extends number ? T[P][] : ArrayValue<T[P]>; -}; - -type Span = 'day' | 'hour'; - -type Log<T extends Obj> = { - _id: mongo.ObjectID; - - /** - * 集計のグループ - */ - group?: any; - - /** - * 集計日時 - */ - date: Date; - - /** - * 集計期間 - */ - span: Span; - - /** - * データ - */ - data: T; - - /** - * ユニークインクリメント用 - */ - unique?: Obj; -}; - -/** - * 様々なチャートの管理を司るクラス - */ -export default abstract class Chart<T extends Obj> { - protected collection: ICollection<Log<T>>; - protected abstract async getTemplate(init: boolean, latest?: T, group?: any): Promise<T>; - private name: string; - - constructor(name: string, grouped = false) { - this.name = name; - this.collection = db.get<Log<T>>(`chart.${name}`); - - const keys = { - span: -1, - date: -1 - } as { [key: string]: 1 | -1; }; - if (grouped) keys.group = -1; - - this.collection.createIndex(keys, { unique: true }); - } - - @autobind - private convertQuery(x: Obj, path: string): Obj { - const query: Obj = {}; - - const dive = (x: Obj, path: string) => { - for (const [k, v] of Object.entries(x)) { - const p = path ? `${path}.${k}` : k; - if (typeof v === 'number') { - query[p] = v; - } else { - dive(v, p); - } - } - }; - - dive(x, path); - - return query; - } - - @autobind - private getCurrentDate(): [number, number, number, number] { - const now = moment().utc(); - - const y = now.year(); - const m = now.month(); - const d = now.date(); - const h = now.hour(); - - return [y, m, d, h]; - } - - @autobind - private getLatestLog(span: Span, group?: any): Promise<Log<T>> { - return this.collection.findOne({ - group: group, - span: span - }, { - sort: { - date: -1 - } - }); - } - - @autobind - private async getCurrentLog(span: Span, group?: any): Promise<Log<T>> { - const [y, m, d, h] = this.getCurrentDate(); - - const current = - span == 'day' ? utc([y, m, d]) : - span == 'hour' ? utc([y, m, d, h]) : - null; - - // 現在(今日または今のHour)のログ - const currentLog = await this.collection.findOne({ - group: group, - span: span, - date: current.toDate() - }); - - // ログがあればそれを返して終了 - if (currentLog != null) { - return currentLog; - } - - let log: Log<T>; - let data: T; - - // 集計期間が変わってから、初めてのチャート更新なら - // 最も最近のログを持ってくる - // * 例えば集計期間が「日」である場合で考えると、 - // * 昨日何もチャートを更新するような出来事がなかった場合は、 - // * ログがそもそも作られずドキュメントが存在しないということがあり得るため、 - // * 「昨日の」と決め打ちせずに「もっとも最近の」とします - const latest = await this.getLatestLog(span, group); - - if (latest != null) { - // 空ログデータを作成 - data = await this.getTemplate(false, latest.data); - } else { - // ログが存在しなかったら - // (Misskeyインスタンスを建てて初めてのチャート更新時など - // または何らかの理由でチャートコレクションを抹消した場合) - - // 初期ログデータを作成 - data = await this.getTemplate(true, null, group); - - logger.info(`${this.name}: Initial commit created`); - } - - try { - // 新規ログ挿入 - log = await this.collection.insert({ - group: group, - span: span, - date: current.toDate(), - data: data - }); - } catch (e) { - // 11000 is duplicate key error - // 並列動作している他のチャートエンジンプロセスと処理が重なる場合がある - // その場合は再度最も新しいログを持ってくる - if (e.code === 11000) { - log = await this.getLatestLog(span, group); - } else { - logger.error(e); - throw e; - } - } - - return log; - } - - @autobind - protected commit(query: Obj, group?: any, uniqueKey?: string, uniqueValue?: string): void { - const update = (log: Log<T>) => { - // ユニークインクリメントの場合、指定のキーに指定の値が既に存在していたら弾く - if ( - uniqueKey && - log.unique && - log.unique[uniqueKey] && - log.unique[uniqueKey].includes(uniqueValue) - ) return; - - // ユニークインクリメントの指定のキーに値を追加 - if (uniqueKey) { - query['$push'] = { - [`unique.${uniqueKey}`]: uniqueValue - }; - } - - // ログ更新 - this.collection.update({ - _id: log._id - }, query); - }; - - this.getCurrentLog('day', group).then(log => update(log)); - this.getCurrentLog('hour', group).then(log => update(log)); - } - - @autobind - protected inc(inc: Partial<T>, group?: any): void { - this.commit({ - $inc: this.convertQuery(inc, 'data') - }, group); - } - - @autobind - protected incIfUnique(inc: Partial<T>, key: string, value: string, group?: any): void { - this.commit({ - $inc: this.convertQuery(inc, 'data') - }, group, key, value); - } - - @autobind - public async getChart(span: Span, range: number, group?: any): Promise<ArrayValue<T>> { - const promisedChart: Promise<T>[] = []; - - const [y, m, d, h] = this.getCurrentDate(); - - const gt = - span == 'day' ? utc([y, m, d]).subtract(range, 'days') : - span == 'hour' ? utc([y, m, d, h]).subtract(range, 'hours') : - null; - - // ログ取得 - let logs = await this.collection.find({ - group: group, - span: span, - date: { - $gte: gt.toDate() - } - }, { - sort: { - date: -1 - }, - fields: { - _id: 0 - } - }); - - // 要求された範囲にログがひとつもなかったら - if (logs.length == 0) { - // もっとも新しいログを持ってくる - // (すくなくともひとつログが無いと隙間埋めできないため) - const recentLog = await this.collection.findOne({ - group: group, - span: span - }, { - sort: { - date: -1 - }, - fields: { - _id: 0 - } - }); - - if (recentLog) { - logs = [recentLog]; - } - - // 要求された範囲の最も古い箇所に位置するログが存在しなかったら - } else if (!utc(logs[logs.length - 1].date).isSame(gt)) { - // 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する - // (隙間埋めできないため) - const outdatedLog = await this.collection.findOne({ - group: group, - span: span, - date: { - $lt: gt.toDate() - } - }, { - sort: { - date: -1 - }, - fields: { - _id: 0 - } - }); - - if (outdatedLog) { - logs.push(outdatedLog); - } - } - - // 整形 - for (let i = (range - 1); i >= 0; i--) { - const current = - span == 'day' ? utc([y, m, d]).subtract(i, 'days') : - span == 'hour' ? utc([y, m, d, h]).subtract(i, 'hours') : - null; - - const log = logs.find(l => utc(l.date).isSame(current)); - - if (log) { - promisedChart.unshift(Promise.resolve(log.data)); - } else { - // 隙間埋め - const latest = logs.find(l => utc(l.date).isBefore(current)); - promisedChart.unshift(this.getTemplate(false, latest ? latest.data : null)); - } - } - - const chart = await Promise.all(promisedChart); - - const res: ArrayValue<T> = {} as any; - - /** - * [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }] - * を - * { foo: [1, 2, 3], bar: [5, 6, 7] } - * にする - */ - const dive = (x: Obj, path?: string) => { - for (const [k, v] of Object.entries(x)) { - const p = path ? `${path}.${k}` : k; - if (typeof v == 'object') { - dive(v, p); - } else { - nestedProperty.set(res, p, chart.map(s => nestedProperty.get(s, p))); - } - } - }; - - dive(chart[0]); - - return res; - } -} - -export function convertLog(logSchema: Schema): Schema { - const v: Schema = JSON.parse(JSON.stringify(logSchema)); // copy - if (v.type === 'number') { - v.type = 'array'; - v.items = { - type: 'number' - }; - } else if (v.type === 'object') { - for (const k of Object.keys(v.properties)) { - v.properties[k] = convertLog(v.properties[k]); - } - } - return v; -} +export const federationChart = new FederationChart(); +export const notesChart = new NotesChart(); +export const usersChart = new UsersChart(); +export const networkChart = new NetworkChart(); +export const activeUsersChart = new ActiveUsersChart(); +export const instanceChart = new InstanceChart(); +export const perUserNotesChart = new PerUserNotesChart(); +export const driveChart = new DriveChart(); +export const perUserReactionsChart = new PerUserReactionsChart(); +export const hashtagChart = new HashtagChart(); +export const perUserFollowingChart = new PerUserFollowingChart(); +export const perUserDriveChart = new PerUserDriveChart(); diff --git a/src/services/chart/instance.ts b/src/services/chart/instance.ts deleted file mode 100644 index 5af398b902..0000000000 --- a/src/services/chart/instance.ts +++ /dev/null @@ -1,302 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Obj } from '.'; -import User from '../../models/user'; -import Note from '../../models/note'; -import Following from '../../models/following'; -import DriveFile, { IDriveFile } from '../../models/drive-file'; - -/** - * インスタンスごとのチャート - */ -type InstanceLog = { - requests: { - /** - * 失敗したリクエスト数 - */ - failed: number; - - /** - * 成功したリクエスト数 - */ - succeeded: number; - - /** - * 受信したリクエスト数 - */ - received: number; - }; - - notes: { - /** - * 集計期間時点での、全投稿数 - */ - total: number; - - /** - * 増加した投稿数 - */ - inc: number; - - /** - * 減少した投稿数 - */ - dec: number; - }; - - users: { - /** - * 集計期間時点での、全ユーザー数 - */ - total: number; - - /** - * 増加したユーザー数 - */ - inc: number; - - /** - * 減少したユーザー数 - */ - dec: number; - }; - - following: { - /** - * 集計期間時点での、全フォロー数 - */ - total: number; - - /** - * 増加したフォロー数 - */ - inc: number; - - /** - * 減少したフォロー数 - */ - dec: number; - }; - - followers: { - /** - * 集計期間時点での、全フォロワー数 - */ - total: number; - - /** - * 増加したフォロワー数 - */ - inc: number; - - /** - * 減少したフォロワー数 - */ - dec: number; - }; - - drive: { - /** - * 集計期間時点での、全ドライブファイル数 - */ - totalFiles: number; - - /** - * 集計期間時点での、全ドライブファイルの合計サイズ - */ - totalUsage: number; - - /** - * 増加したドライブファイル数 - */ - incFiles: number; - - /** - * 増加したドライブ使用量 - */ - incUsage: number; - - /** - * 減少したドライブファイル数 - */ - decFiles: number; - - /** - * 減少したドライブ使用量 - */ - decUsage: number; - }; -}; - -class InstanceChart extends Chart<InstanceLog> { - constructor() { - super('instance', true); - } - - @autobind - protected async getTemplate(init: boolean, latest?: InstanceLog, group?: any): Promise<InstanceLog> { - const calcUsage = () => DriveFile - .aggregate([{ - $match: { - 'metadata._user.host': group, - 'metadata.deletedAt': { $exists: false } - } - }, { - $project: { - length: true - } - }, { - $group: { - _id: null, - usage: { $sum: '$length' } - } - }]) - .then(res => res.length > 0 ? res[0].usage : 0); - - const [ - notesCount, - usersCount, - followingCount, - followersCount, - driveFiles, - driveUsage, - ] = init ? await Promise.all([ - Note.count({ '_user.host': group }), - User.count({ host: group }), - Following.count({ '_follower.host': group }), - Following.count({ '_followee.host': group }), - DriveFile.count({ 'metadata._user.host': group }), - calcUsage(), - ]) : [ - latest ? latest.notes.total : 0, - latest ? latest.users.total : 0, - latest ? latest.following.total : 0, - latest ? latest.followers.total : 0, - latest ? latest.drive.totalFiles : 0, - latest ? latest.drive.totalUsage : 0, - ]; - - return { - requests: { - failed: 0, - succeeded: 0, - received: 0 - }, - notes: { - total: notesCount, - inc: 0, - dec: 0 - }, - users: { - total: usersCount, - inc: 0, - dec: 0 - }, - following: { - total: followingCount, - inc: 0, - dec: 0 - }, - followers: { - total: followersCount, - inc: 0, - dec: 0 - }, - drive: { - totalFiles: driveFiles, - totalUsage: driveUsage, - incFiles: 0, - incUsage: 0, - decFiles: 0, - decUsage: 0 - } - }; - } - - @autobind - public async requestReceived(host: string) { - await this.inc({ - requests: { - received: 1 - } - }, host); - } - - @autobind - public async requestSent(host: string, isSucceeded: boolean) { - const update: Obj = {}; - - if (isSucceeded) { - update.succeeded = 1; - } else { - update.failed = 1; - } - - await this.inc({ - requests: update - }, host); - } - - @autobind - public async newUser(host: string) { - await this.inc({ - users: { - total: 1, - inc: 1 - } - }, host); - } - - @autobind - public async updateNote(host: string, isAdditional: boolean) { - await this.inc({ - notes: { - total: isAdditional ? 1 : -1, - inc: isAdditional ? 1 : 0, - dec: isAdditional ? 0 : 1, - } - }, host); - } - - @autobind - public async updateFollowing(host: string, isAdditional: boolean) { - await this.inc({ - following: { - total: isAdditional ? 1 : -1, - inc: isAdditional ? 1 : 0, - dec: isAdditional ? 0 : 1, - } - }, host); - } - - @autobind - public async updateFollowers(host: string, isAdditional: boolean) { - await this.inc({ - followers: { - total: isAdditional ? 1 : -1, - inc: isAdditional ? 1 : 0, - dec: isAdditional ? 0 : 1, - } - }, host); - } - - @autobind - public async updateDrive(file: IDriveFile, isAdditional: boolean) { - const update: Obj = {}; - - update.totalFiles = isAdditional ? 1 : -1; - update.totalUsage = isAdditional ? file.length : -file.length; - if (isAdditional) { - update.incFiles = 1; - update.incUsage = file.length; - } else { - update.decFiles = 1; - update.decUsage = file.length; - } - - await this.inc({ - drive: update - }, file.metadata._user.host); - } -} - -export default new InstanceChart(); diff --git a/src/services/chart/network.ts b/src/services/chart/network.ts deleted file mode 100644 index fce47099d1..0000000000 --- a/src/services/chart/network.ts +++ /dev/null @@ -1,64 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Partial } from './'; - -/** - * ネットワークに関するチャート - */ -type NetworkLog = { - /** - * 受信したリクエスト数 - */ - incomingRequests: number; - - /** - * 送信したリクエスト数 - */ - outgoingRequests: number; - - /** - * 応答時間の合計 - * TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる - */ - totalTime: number; - - /** - * 合計受信データ量 - */ - incomingBytes: number; - - /** - * 合計送信データ量 - */ - outgoingBytes: number; -}; - -class NetworkChart extends Chart<NetworkLog> { - constructor() { - super('network'); - } - - @autobind - protected async getTemplate(init: boolean, latest?: NetworkLog): Promise<NetworkLog> { - return { - incomingRequests: 0, - outgoingRequests: 0, - totalTime: 0, - incomingBytes: 0, - outgoingBytes: 0 - }; - } - - @autobind - public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number) { - const inc: Partial<NetworkLog> = { - incomingRequests: incomingRequests, - totalTime: time, - incomingBytes: incomingBytes, - outgoingBytes: outgoingBytes - }; - - await this.inc(inc); - } -} - -export default new NetworkChart(); diff --git a/src/services/chart/notes.ts b/src/services/chart/notes.ts deleted file mode 100644 index b047ec273f..0000000000 --- a/src/services/chart/notes.ts +++ /dev/null @@ -1,127 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Obj } from '.'; -import Note, { INote } from '../../models/note'; -import { isLocalUser } from '../../models/user'; -import { SchemaType } from '../../misc/schema'; - -const logSchema = { - total: { - type: 'number' as 'number', - description: '集計期間時点での、全投稿数' - }, - - inc: { - type: 'number' as 'number', - description: '増加した投稿数' - }, - - dec: { - type: 'number' as 'number', - description: '減少した投稿数' - }, - - diffs: { - type: 'object' as 'object', - properties: { - normal: { - type: 'number' as 'number', - description: '通常の投稿数の差分' - }, - - reply: { - type: 'number' as 'number', - description: 'リプライの投稿数の差分' - }, - - renote: { - type: 'number' as 'number', - description: 'Renoteの投稿数の差分' - }, - } - }, -}; - -export const notesLogSchema = { - type: 'object' as 'object', - properties: { - local: { - type: 'object' as 'object', - properties: logSchema - }, - remote: { - type: 'object' as 'object', - properties: logSchema - }, - } -}; - -type NotesLog = SchemaType<typeof notesLogSchema>; - -class NotesChart extends Chart<NotesLog> { - constructor() { - super('notes'); - } - - @autobind - protected async getTemplate(init: boolean, latest?: NotesLog): Promise<NotesLog> { - const [localCount, remoteCount] = init ? await Promise.all([ - Note.count({ '_user.host': null }), - Note.count({ '_user.host': { $ne: null } }) - ]) : [ - latest ? latest.local.total : 0, - latest ? latest.remote.total : 0 - ]; - - return { - local: { - total: localCount, - inc: 0, - dec: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - }, - remote: { - total: remoteCount, - inc: 0, - dec: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - } - }; - } - - @autobind - public async update(note: INote, isAdditional: boolean) { - const update: Obj = { - diffs: {} - }; - - update.total = isAdditional ? 1 : -1; - - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } - - if (note.replyId != null) { - update.diffs.reply = isAdditional ? 1 : -1; - } else if (note.renoteId != null) { - update.diffs.renote = isAdditional ? 1 : -1; - } else { - update.diffs.normal = isAdditional ? 1 : -1; - } - - await this.inc({ - [isLocalUser(note._user) ? 'local' : 'remote']: update - }); - } -} - -export default new NotesChart(); diff --git a/src/services/chart/per-user-drive.ts b/src/services/chart/per-user-drive.ts deleted file mode 100644 index 4f335f1688..0000000000 --- a/src/services/chart/per-user-drive.ts +++ /dev/null @@ -1,122 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Obj } from './'; -import DriveFile, { IDriveFile } from '../../models/drive-file'; -import { SchemaType } from '../../misc/schema'; - -export const perUserDriveLogSchema = { - type: 'object' as 'object', - properties: { - /** - * 集計期間時点での、全ドライブファイル数 - */ - totalCount: { - type: 'number' as 'number', - description: '集計期間時点での、全ドライブファイル数' - }, - - /** - * 集計期間時点での、全ドライブファイルの合計サイズ - */ - totalSize: { - type: 'number' as 'number', - description: '集計期間時点での、全ドライブファイルの合計サイズ' - }, - - /** - * 増加したドライブファイル数 - */ - incCount: { - type: 'number' as 'number', - description: '増加したドライブファイル数' - }, - - /** - * 増加したドライブ使用量 - */ - incSize: { - type: 'number' as 'number', - description: '増加したドライブ使用量' - }, - - /** - * 減少したドライブファイル数 - */ - decCount: { - type: 'number' as 'number', - description: '減少したドライブファイル数' - }, - - /** - * 減少したドライブ使用量 - */ - decSize: { - type: 'number' as 'number', - description: '減少したドライブ使用量' - }, - } -}; - -type PerUserDriveLog = SchemaType<typeof perUserDriveLogSchema>; - -class PerUserDriveChart extends Chart<PerUserDriveLog> { - constructor() { - super('perUserDrive', true); - } - - @autobind - protected async getTemplate(init: boolean, latest?: PerUserDriveLog, group?: any): Promise<PerUserDriveLog> { - const calcSize = () => DriveFile - .aggregate([{ - $match: { - 'metadata.userId': group, - 'metadata.deletedAt': { $exists: false } - } - }, { - $project: { - length: true - } - }, { - $group: { - _id: null, - usage: { $sum: '$length' } - } - }]) - .then(res => res.length > 0 ? res[0].usage : 0); - - const [count, size] = init ? await Promise.all([ - DriveFile.count({ 'metadata.userId': group }), - calcSize() - ]) : [ - latest ? latest.totalCount : 0, - latest ? latest.totalSize : 0 - ]; - - return { - totalCount: count, - totalSize: size, - incCount: 0, - incSize: 0, - decCount: 0, - decSize: 0 - }; - } - - @autobind - public async update(file: IDriveFile, isAdditional: boolean) { - const update: Obj = {}; - - update.totalCount = isAdditional ? 1 : -1; - update.totalSize = isAdditional ? file.length : -file.length; - if (isAdditional) { - update.incCount = 1; - update.incSize = file.length; - } else { - update.decCount = 1; - update.decSize = file.length; - } - - await this.inc(update, file.metadata.userId); - } -} - -export default new PerUserDriveChart(); diff --git a/src/services/chart/per-user-following.ts b/src/services/chart/per-user-following.ts deleted file mode 100644 index 8a94a4f155..0000000000 --- a/src/services/chart/per-user-following.ts +++ /dev/null @@ -1,162 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Obj } from './'; -import Following from '../../models/following'; -import { IUser, isLocalUser } from '../../models/user'; -import { SchemaType } from '../../misc/schema'; - -export const logSchema = { - /** - * フォローしている - */ - followings: { - type: 'object' as 'object', - properties: { - /** - * フォローしている合計 - */ - total: { - type: 'number', - description: 'フォローしている合計', - }, - - /** - * フォローした数 - */ - inc: { - type: 'number', - description: 'フォローした数', - }, - - /** - * フォロー解除した数 - */ - dec: { - type: 'number', - description: 'フォロー解除した数', - }, - } - }, - - /** - * フォローされている - */ - followers: { - type: 'object' as 'object', - properties: { - /** - * フォローされている合計 - */ - total: { - type: 'number', - description: 'フォローされている合計', - }, - - /** - * フォローされた数 - */ - inc: { - type: 'number', - description: 'フォローされた数', - }, - - /** - * フォロー解除された数 - */ - dec: { - type: 'number', - description: 'フォロー解除された数', - }, - } - }, -}; - -export const perUserFollowingLogSchema = { - type: 'object' as 'object', - properties: { - local: { - type: 'object' as 'object', - properties: logSchema - }, - remote: { - type: 'object' as 'object', - properties: logSchema - }, - } -}; - -type PerUserFollowingLog = SchemaType<typeof perUserFollowingLogSchema>; - -class PerUserFollowingChart extends Chart<PerUserFollowingLog> { - constructor() { - super('perUserFollowing', true); - } - - @autobind - protected async getTemplate(init: boolean, latest?: PerUserFollowingLog, group?: any): Promise<PerUserFollowingLog> { - const [ - localFollowingsCount, - localFollowersCount, - remoteFollowingsCount, - remoteFollowersCount - ] = init ? await Promise.all([ - Following.count({ followerId: group, '_followee.host': null }), - Following.count({ followeeId: group, '_follower.host': null }), - Following.count({ followerId: group, '_followee.host': { $ne: null } }), - Following.count({ followeeId: group, '_follower.host': { $ne: null } }) - ]) : [ - latest ? latest.local.followings.total : 0, - latest ? latest.local.followers.total : 0, - latest ? latest.remote.followings.total : 0, - latest ? latest.remote.followers.total : 0 - ]; - - return { - local: { - followings: { - total: localFollowingsCount, - inc: 0, - dec: 0 - }, - followers: { - total: localFollowersCount, - inc: 0, - dec: 0 - } - }, - remote: { - followings: { - total: remoteFollowingsCount, - inc: 0, - dec: 0 - }, - followers: { - total: remoteFollowersCount, - inc: 0, - dec: 0 - } - } - }; - } - - @autobind - public async update(follower: IUser, followee: IUser, isFollow: boolean) { - const update: Obj = {}; - - update.total = isFollow ? 1 : -1; - - if (isFollow) { - update.inc = 1; - } else { - update.dec = 1; - } - - this.inc({ - [isLocalUser(follower) ? 'local' : 'remote']: { followings: update } - }, follower._id); - this.inc({ - [isLocalUser(followee) ? 'local' : 'remote']: { followers: update } - }, followee._id); - } -} - -export default new PerUserFollowingChart(); diff --git a/src/services/chart/per-user-notes.ts b/src/services/chart/per-user-notes.ts deleted file mode 100644 index 2f4f882091..0000000000 --- a/src/services/chart/per-user-notes.ts +++ /dev/null @@ -1,100 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Obj } from './'; -import Note, { INote } from '../../models/note'; -import { IUser } from '../../models/user'; -import { SchemaType } from '../../misc/schema'; - -export const perUserNotesLogSchema = { - type: 'object' as 'object', - properties: { - total: { - type: 'number' as 'number', - description: '集計期間時点での、全投稿数' - }, - - inc: { - type: 'number' as 'number', - description: '増加した投稿数' - }, - - dec: { - type: 'number' as 'number', - description: '減少した投稿数' - }, - - diffs: { - type: 'object' as 'object', - properties: { - normal: { - type: 'number' as 'number', - description: '通常の投稿数の差分' - }, - - reply: { - type: 'number' as 'number', - description: 'リプライの投稿数の差分' - }, - - renote: { - type: 'number' as 'number', - description: 'Renoteの投稿数の差分' - }, - } - }, - } -}; - -type PerUserNotesLog = SchemaType<typeof perUserNotesLogSchema>; - -class PerUserNotesChart extends Chart<PerUserNotesLog> { - constructor() { - super('perUserNotes', true); - } - - @autobind - protected async getTemplate(init: boolean, latest?: PerUserNotesLog, group?: any): Promise<PerUserNotesLog> { - const [count] = init ? await Promise.all([ - Note.count({ userId: group, deletedAt: null }), - ]) : [ - latest ? latest.total : 0 - ]; - - return { - total: count, - inc: 0, - dec: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - }; - } - - @autobind - public async update(user: IUser, note: INote, isAdditional: boolean) { - const update: Obj = { - diffs: {} - }; - - update.total = isAdditional ? 1 : -1; - - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } - - if (note.replyId != null) { - update.diffs.reply = isAdditional ? 1 : -1; - } else if (note.renoteId != null) { - update.diffs.renote = isAdditional ? 1 : -1; - } else { - update.diffs.normal = isAdditional ? 1 : -1; - } - - await this.inc(update, user._id); - } -} - -export default new PerUserNotesChart(); diff --git a/src/services/chart/per-user-reactions.ts b/src/services/chart/per-user-reactions.ts deleted file mode 100644 index 60495aeb02..0000000000 --- a/src/services/chart/per-user-reactions.ts +++ /dev/null @@ -1,45 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart from './'; -import { IUser, isLocalUser } from '../../models/user'; -import { INote } from '../../models/note'; - -/** - * ユーザーごとのリアクションに関するチャート - */ -type PerUserReactionsLog = { - local: { - /** - * リアクションされた数 - */ - count: number; - }; - - remote: PerUserReactionsLog['local']; -}; - -class PerUserReactionsChart extends Chart<PerUserReactionsLog> { - constructor() { - super('perUserReaction', true); - } - - @autobind - protected async getTemplate(init: boolean, latest?: PerUserReactionsLog, group?: any): Promise<PerUserReactionsLog> { - return { - local: { - count: 0 - }, - remote: { - count: 0 - } - }; - } - - @autobind - public async update(user: IUser, note: INote) { - this.inc({ - [isLocalUser(user) ? 'local' : 'remote']: { count: 1 } - }, note.userId); - } -} - -export default new PerUserReactionsChart(); diff --git a/src/services/chart/users.ts b/src/services/chart/users.ts deleted file mode 100644 index cca9590842..0000000000 --- a/src/services/chart/users.ts +++ /dev/null @@ -1,94 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { Obj } from './'; -import User, { IUser, isLocalUser } from '../../models/user'; -import { SchemaType } from '../../misc/schema'; - -const logSchema = { - /** - * 集計期間時点での、全ユーザー数 - */ - total: { - type: 'number' as 'number', - description: '集計期間時点での、全ユーザー数' - }, - - /** - * 増加したユーザー数 - */ - inc: { - type: 'number' as 'number', - description: '増加したユーザー数' - }, - - /** - * 減少したユーザー数 - */ - dec: { - type: 'number' as 'number', - description: '減少したユーザー数' - }, -}; - -export const usersLogSchema = { - type: 'object' as 'object', - properties: { - local: { - type: 'object' as 'object', - properties: logSchema - }, - remote: { - type: 'object' as 'object', - properties: logSchema - }, - } -}; - -type UsersLog = SchemaType<typeof usersLogSchema>; - -class UsersChart extends Chart<UsersLog> { - constructor() { - super('users'); - } - - @autobind - protected async getTemplate(init: boolean, latest?: UsersLog): Promise<UsersLog> { - const [localCount, remoteCount] = init ? await Promise.all([ - User.count({ host: null }), - User.count({ host: { $ne: null } }) - ]) : [ - latest ? latest.local.total : 0, - latest ? latest.remote.total : 0 - ]; - - return { - local: { - total: localCount, - inc: 0, - dec: 0 - }, - remote: { - total: remoteCount, - inc: 0, - dec: 0 - } - }; - } - - @autobind - public async update(user: IUser, isAdditional: boolean) { - const update: Obj = {}; - - update.total = isAdditional ? 1 : -1; - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } - - await this.inc({ - [isLocalUser(user) ? 'local' : 'remote']: update - }); - } -} - -export default new UsersChart(); |