From 4f249159d310d4296753fcfe8e2bbd390fd9002b Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 18 Mar 2021 11:17:05 +0900 Subject: Improve chart performance (#7360) * wip * wip * wip * wip * wip * Update chart.ts * wip * Improve server performance * wip * wip --- src/services/chart/charts/classes/active-users.ts | 18 +++++++-- src/services/chart/charts/classes/drive.ts | 22 +++++++++++ src/services/chart/charts/classes/federation.ts | 11 ++++++ src/services/chart/charts/classes/hashtag.ts | 18 +++++++-- src/services/chart/charts/classes/instance.ts | 44 ++++++++++++++++++++++ src/services/chart/charts/classes/network.ts | 11 ++++++ src/services/chart/charts/classes/notes.ts | 26 +++++++++++++ .../chart/charts/classes/per-user-drive.ts | 12 ++++++ .../chart/charts/classes/per-user-following.ts | 30 +++++++++++++++ .../chart/charts/classes/per-user-notes.ts | 14 +++++++ .../chart/charts/classes/per-user-reactions.ts | 12 ++++++ src/services/chart/charts/classes/test-grouped.ts | 11 ++++++ src/services/chart/charts/classes/test-unique.ts | 13 +++++-- src/services/chart/charts/classes/test.ts | 11 ++++++ src/services/chart/charts/classes/users.ts | 16 ++++++++ 15 files changed, 260 insertions(+), 9 deletions(-) (limited to 'src/services/chart/charts/classes') diff --git a/src/services/chart/charts/classes/active-users.ts b/src/services/chart/charts/classes/active-users.ts index 5128150de6..4820f8281b 100644 --- a/src/services/chart/charts/classes/active-users.ts +++ b/src/services/chart/charts/classes/active-users.ts @@ -17,6 +17,18 @@ export default class ActiveUsersChart extends Chart { return {}; } + @autobind + protected aggregate(logs: ActiveUsersLog[]): ActiveUsersLog { + return { + local: { + users: logs.reduce((a, b) => a.concat(b.local.users), [] as ActiveUsersLog['local']['users']), + }, + remote: { + users: logs.reduce((a, b) => a.concat(b.remote.users), [] as ActiveUsersLog['remote']['users']), + }, + }; + } + @autobind protected async fetchActual(): Promise> { return {}; @@ -25,11 +37,11 @@ export default class ActiveUsersChart extends Chart { @autobind public async update(user: User) { const update: Obj = { - count: 1 + users: [user.id] }; - await this.incIfUnique({ + await this.inc({ [Users.isLocalUser(user) ? 'local' : 'remote']: update - }, 'users', user.id); + }); } } diff --git a/src/services/chart/charts/classes/drive.ts b/src/services/chart/charts/classes/drive.ts index 57bb120beb..46399a34d8 100644 --- a/src/services/chart/charts/classes/drive.ts +++ b/src/services/chart/charts/classes/drive.ts @@ -27,6 +27,28 @@ export default class DriveChart extends Chart { }; } + @autobind + protected aggregate(logs: DriveLog[]): DriveLog { + return { + local: { + totalCount: logs[0].local.totalCount, + totalSize: logs[0].local.totalSize, + incCount: logs.reduce((a, b) => a + b.local.incCount, 0), + incSize: logs.reduce((a, b) => a + b.local.incSize, 0), + decCount: logs.reduce((a, b) => a + b.local.decCount, 0), + decSize: logs.reduce((a, b) => a + b.local.decSize, 0), + }, + remote: { + totalCount: logs[0].remote.totalCount, + totalSize: logs[0].remote.totalSize, + incCount: logs.reduce((a, b) => a + b.remote.incCount, 0), + incSize: logs.reduce((a, b) => a + b.remote.incSize, 0), + decCount: logs.reduce((a, b) => a + b.remote.decCount, 0), + decSize: logs.reduce((a, b) => a + b.remote.decSize, 0), + }, + }; + } + @autobind protected async fetchActual(): Promise> { const [localCount, remoteCount, localSize, remoteSize] = await Promise.all([ diff --git a/src/services/chart/charts/classes/federation.ts b/src/services/chart/charts/classes/federation.ts index bd2c497e7b..ab6ec2d4dd 100644 --- a/src/services/chart/charts/classes/federation.ts +++ b/src/services/chart/charts/classes/federation.ts @@ -20,6 +20,17 @@ export default class FederationChart extends Chart { }; } + @autobind + protected aggregate(logs: FederationLog[]): FederationLog { + return { + instance: { + total: logs[0].instance.total, + inc: logs.reduce((a, b) => a + b.instance.inc, 0), + dec: logs.reduce((a, b) => a + b.instance.dec, 0), + }, + }; + } + @autobind protected async fetchActual(): Promise> { const [total] = await Promise.all([ diff --git a/src/services/chart/charts/classes/hashtag.ts b/src/services/chart/charts/classes/hashtag.ts index 38c3a94f0c..43db5b0a83 100644 --- a/src/services/chart/charts/classes/hashtag.ts +++ b/src/services/chart/charts/classes/hashtag.ts @@ -17,6 +17,18 @@ export default class HashtagChart extends Chart { return {}; } + @autobind + protected aggregate(logs: HashtagLog[]): HashtagLog { + return { + local: { + users: logs.reduce((a, b) => a.concat(b.local.users), [] as HashtagLog['local']['users']), + }, + remote: { + users: logs.reduce((a, b) => a.concat(b.remote.users), [] as HashtagLog['remote']['users']), + }, + }; + } + @autobind protected async fetchActual(): Promise> { return {}; @@ -25,11 +37,11 @@ export default class HashtagChart extends Chart { @autobind public async update(hashtag: string, user: User) { const update: Obj = { - count: 1 + users: [user.id] }; - await this.incIfUnique({ + await this.inc({ [Users.isLocalUser(user) ? 'local' : 'remote']: update - }, 'users', user.id, hashtag); + }, hashtag); } } diff --git a/src/services/chart/charts/classes/instance.ts b/src/services/chart/charts/classes/instance.ts index 7575abfb6f..c32b864d87 100644 --- a/src/services/chart/charts/classes/instance.ts +++ b/src/services/chart/charts/classes/instance.ts @@ -36,6 +36,50 @@ export default class InstanceChart extends Chart { }; } + @autobind + protected aggregate(logs: InstanceLog[]): InstanceLog { + return { + requests: { + failed: logs.reduce((a, b) => a + b.requests.failed, 0), + succeeded: logs.reduce((a, b) => a + b.requests.succeeded, 0), + received: logs.reduce((a, b) => a + b.requests.received, 0), + }, + notes: { + total: logs[0].notes.total, + inc: logs.reduce((a, b) => a + b.notes.inc, 0), + dec: logs.reduce((a, b) => a + b.notes.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.notes.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.notes.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.notes.diffs.normal, 0), + }, + }, + users: { + total: logs[0].users.total, + inc: logs.reduce((a, b) => a + b.users.inc, 0), + dec: logs.reduce((a, b) => a + b.users.dec, 0), + }, + following: { + total: logs[0].following.total, + inc: logs.reduce((a, b) => a + b.following.inc, 0), + dec: logs.reduce((a, b) => a + b.following.dec, 0), + }, + followers: { + total: logs[0].followers.total, + inc: logs.reduce((a, b) => a + b.followers.inc, 0), + dec: logs.reduce((a, b) => a + b.followers.dec, 0), + }, + drive: { + totalFiles: logs[0].drive.totalFiles, + totalUsage: logs[0].drive.totalUsage, + incFiles: logs.reduce((a, b) => a + b.drive.incFiles, 0), + incUsage: logs.reduce((a, b) => a + b.drive.incUsage, 0), + decFiles: logs.reduce((a, b) => a + b.drive.decFiles, 0), + decUsage: logs.reduce((a, b) => a + b.drive.decUsage, 0), + }, + }; + } + @autobind protected async fetchActual(group: string): Promise> { const [ diff --git a/src/services/chart/charts/classes/network.ts b/src/services/chart/charts/classes/network.ts index 8b26e5c4c2..693af48f73 100644 --- a/src/services/chart/charts/classes/network.ts +++ b/src/services/chart/charts/classes/network.ts @@ -15,6 +15,17 @@ export default class NetworkChart extends Chart { return {}; } + @autobind + protected aggregate(logs: NetworkLog[]): NetworkLog { + return { + incomingRequests: logs.reduce((a, b) => a + b.incomingRequests, 0), + outgoingRequests: logs.reduce((a, b) => a + b.outgoingRequests, 0), + totalTime: logs.reduce((a, b) => a + b.totalTime, 0), + incomingBytes: logs.reduce((a, b) => a + b.incomingBytes, 0), + outgoingBytes: logs.reduce((a, b) => a + b.outgoingBytes, 0), + }; + } + @autobind protected async fetchActual(): Promise> { return {}; diff --git a/src/services/chart/charts/classes/notes.ts b/src/services/chart/charts/classes/notes.ts index 815061c445..965087bc08 100644 --- a/src/services/chart/charts/classes/notes.ts +++ b/src/services/chart/charts/classes/notes.ts @@ -25,6 +25,32 @@ export default class NotesChart extends Chart { }; } + @autobind + protected aggregate(logs: NotesLog[]): NotesLog { + return { + local: { + total: logs[0].local.total, + inc: logs.reduce((a, b) => a + b.local.inc, 0), + dec: logs.reduce((a, b) => a + b.local.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.local.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.local.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.local.diffs.normal, 0), + }, + }, + remote: { + total: logs[0].remote.total, + inc: logs.reduce((a, b) => a + b.remote.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.remote.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.remote.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.remote.diffs.normal, 0), + }, + }, + }; + } + @autobind protected async fetchActual(): Promise> { const [localCount, remoteCount] = await Promise.all([ diff --git a/src/services/chart/charts/classes/per-user-drive.ts b/src/services/chart/charts/classes/per-user-drive.ts index aed9f6fce7..e778f7bf61 100644 --- a/src/services/chart/charts/classes/per-user-drive.ts +++ b/src/services/chart/charts/classes/per-user-drive.ts @@ -20,6 +20,18 @@ export default class PerUserDriveChart extends Chart { }; } + @autobind + protected aggregate(logs: PerUserDriveLog[]): PerUserDriveLog { + return { + totalCount: logs[0].totalCount, + totalSize: logs[0].totalSize, + incCount: logs.reduce((a, b) => a + b.incCount, 0), + incSize: logs.reduce((a, b) => a + b.incSize, 0), + decCount: logs.reduce((a, b) => a + b.decCount, 0), + decSize: logs.reduce((a, b) => a + b.decSize, 0), + }; + } + @autobind protected async fetchActual(group: string): Promise> { const [count, size] = await Promise.all([ diff --git a/src/services/chart/charts/classes/per-user-following.ts b/src/services/chart/charts/classes/per-user-following.ts index 8295c0cb0d..8b536009c8 100644 --- a/src/services/chart/charts/classes/per-user-following.ts +++ b/src/services/chart/charts/classes/per-user-following.ts @@ -35,6 +35,36 @@ export default class PerUserFollowingChart extends Chart { }; } + @autobind + protected aggregate(logs: PerUserFollowingLog[]): PerUserFollowingLog { + return { + local: { + followings: { + total: logs[0].local.followings.total, + inc: logs.reduce((a, b) => a + b.local.followings.inc, 0), + dec: logs.reduce((a, b) => a + b.local.followings.dec, 0), + }, + followers: { + total: logs[0].local.followers.total, + inc: logs.reduce((a, b) => a + b.local.followers.inc, 0), + dec: logs.reduce((a, b) => a + b.local.followers.dec, 0), + }, + }, + remote: { + followings: { + total: logs[0].remote.followings.total, + inc: logs.reduce((a, b) => a + b.remote.followings.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.followings.dec, 0), + }, + followers: { + total: logs[0].remote.followers.total, + inc: logs.reduce((a, b) => a + b.remote.followers.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.followers.dec, 0), + }, + }, + }; + } + @autobind protected async fetchActual(group: string): Promise> { const [ diff --git a/src/services/chart/charts/classes/per-user-notes.ts b/src/services/chart/charts/classes/per-user-notes.ts index cccd495604..8d1fb8d2b0 100644 --- a/src/services/chart/charts/classes/per-user-notes.ts +++ b/src/services/chart/charts/classes/per-user-notes.ts @@ -20,6 +20,20 @@ export default class PerUserNotesChart extends Chart { }; } + @autobind + protected aggregate(logs: PerUserNotesLog[]): PerUserNotesLog { + return { + total: logs[0].total, + inc: logs.reduce((a, b) => a + b.inc, 0), + dec: logs.reduce((a, b) => a + b.dec, 0), + diffs: { + reply: logs.reduce((a, b) => a + b.diffs.reply, 0), + renote: logs.reduce((a, b) => a + b.diffs.renote, 0), + normal: logs.reduce((a, b) => a + b.diffs.normal, 0), + }, + }; + } + @autobind protected async fetchActual(group: string): Promise> { const [count] = await Promise.all([ diff --git a/src/services/chart/charts/classes/per-user-reactions.ts b/src/services/chart/charts/classes/per-user-reactions.ts index 124fb4153c..b4cdced40c 100644 --- a/src/services/chart/charts/classes/per-user-reactions.ts +++ b/src/services/chart/charts/classes/per-user-reactions.ts @@ -18,6 +18,18 @@ export default class PerUserReactionsChart extends Chart { return {}; } + @autobind + protected aggregate(logs: PerUserReactionsLog[]): PerUserReactionsLog { + return { + local: { + count: logs.reduce((a, b) => a + b.local.count, 0), + }, + remote: { + count: logs.reduce((a, b) => a + b.remote.count, 0), + }, + }; + } + @autobind protected async fetchActual(group: string): Promise> { return {}; diff --git a/src/services/chart/charts/classes/test-grouped.ts b/src/services/chart/charts/classes/test-grouped.ts index e32cbcf416..92c8df636e 100644 --- a/src/services/chart/charts/classes/test-grouped.ts +++ b/src/services/chart/charts/classes/test-grouped.ts @@ -21,6 +21,17 @@ export default class TestGroupedChart extends Chart { }; } + @autobind + protected aggregate(logs: TestGroupedLog[]): TestGroupedLog { + return { + foo: { + total: logs[0].foo.total, + inc: logs.reduce((a, b) => a + b.foo.inc, 0), + dec: logs.reduce((a, b) => a + b.foo.dec, 0), + }, + }; + } + @autobind protected async fetchActual(group: string): Promise> { return { diff --git a/src/services/chart/charts/classes/test-unique.ts b/src/services/chart/charts/classes/test-unique.ts index 1eb396c293..5680d713ec 100644 --- a/src/services/chart/charts/classes/test-unique.ts +++ b/src/services/chart/charts/classes/test-unique.ts @@ -15,6 +15,13 @@ export default class TestUniqueChart extends Chart { return {}; } + @autobind + protected aggregate(logs: TestUniqueLog[]): TestUniqueLog { + return { + foo: logs.reduce((a, b) => a.concat(b.foo), [] as TestUniqueLog['foo']), + }; + } + @autobind protected async fetchActual(): Promise> { return {}; @@ -22,8 +29,8 @@ export default class TestUniqueChart extends Chart { @autobind public async uniqueIncrement(key: string) { - await this.incIfUnique({ - foo: 1 - }, 'foos', key); + await this.inc({ + foo: [key] + }); } } diff --git a/src/services/chart/charts/classes/test.ts b/src/services/chart/charts/classes/test.ts index ea64040f3e..d37d298de7 100644 --- a/src/services/chart/charts/classes/test.ts +++ b/src/services/chart/charts/classes/test.ts @@ -21,6 +21,17 @@ export default class TestChart extends Chart { }; } + @autobind + protected aggregate(logs: TestLog[]): TestLog { + return { + foo: { + total: logs[0].foo.total, + inc: logs.reduce((a, b) => a + b.foo.inc, 0), + dec: logs.reduce((a, b) => a + b.foo.dec, 0), + }, + }; + } + @autobind protected async fetchActual(): Promise> { return { diff --git a/src/services/chart/charts/classes/users.ts b/src/services/chart/charts/classes/users.ts index 47e4caa1b7..87b19d88f9 100644 --- a/src/services/chart/charts/classes/users.ts +++ b/src/services/chart/charts/classes/users.ts @@ -25,6 +25,22 @@ export default class UsersChart extends Chart { }; } + @autobind + protected aggregate(logs: UsersLog[]): UsersLog { + return { + local: { + total: logs[0].local.total, + inc: logs.reduce((a, b) => a + b.local.inc, 0), + dec: logs.reduce((a, b) => a + b.local.dec, 0), + }, + remote: { + total: logs[0].remote.total, + inc: logs.reduce((a, b) => a + b.remote.inc, 0), + dec: logs.reduce((a, b) => a + b.remote.dec, 0), + }, + }; + } + @autobind protected async fetchActual(): Promise> { const [localCount, remoteCount] = await Promise.all([ -- cgit v1.2.3-freya