From 0afebcfd9e10a1c5deb98fb739e00aa73c757624 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 10 Feb 2022 17:45:12 +0900 Subject: enhance: improve federation chart --- .../src/services/chart/charts/active-users.ts | 7 ++++- .../src/services/chart/charts/ap-request.ts | 7 ++++- .../backend/src/services/chart/charts/drive.ts | 7 ++++- .../services/chart/charts/entities/federation.ts | 5 ++-- .../src/services/chart/charts/federation.ts | 33 +++++++++++++--------- .../backend/src/services/chart/charts/hashtag.ts | 7 ++++- .../backend/src/services/chart/charts/instance.ts | 7 ++++- .../backend/src/services/chart/charts/notes.ts | 7 ++++- .../src/services/chart/charts/per-user-drive.ts | 7 ++++- .../services/chart/charts/per-user-following.ts | 7 ++++- .../src/services/chart/charts/per-user-notes.ts | 7 ++++- .../services/chart/charts/per-user-reactions.ts | 7 ++++- .../src/services/chart/charts/test-grouped.ts | 7 ++++- .../src/services/chart/charts/test-intersection.ts | 7 ++++- .../src/services/chart/charts/test-unique.ts | 7 ++++- packages/backend/src/services/chart/charts/test.ts | 7 ++++- .../backend/src/services/chart/charts/users.ts | 7 ++++- packages/backend/src/services/chart/core.ts | 19 +++++++++++-- 18 files changed, 128 insertions(+), 34 deletions(-) (limited to 'packages/backend/src/services/chart') diff --git a/packages/backend/src/services/chart/charts/active-users.ts b/packages/backend/src/services/chart/charts/active-users.ts index 87dd95f4dc..5baf46f772 100644 --- a/packages/backend/src/services/chart/charts/active-users.ts +++ b/packages/backend/src/services/chart/charts/active-users.ts @@ -18,7 +18,12 @@ export default class ActiveUsersChart extends Chart { } @autobind - protected async queryCurrentState(): Promise>> { + protected async tickMajor(): Promise>> { + return {}; + } + + @autobind + protected async tickMinor(): Promise>> { return {}; } diff --git a/packages/backend/src/services/chart/charts/ap-request.ts b/packages/backend/src/services/chart/charts/ap-request.ts index bac5e425c8..ca763c8847 100644 --- a/packages/backend/src/services/chart/charts/ap-request.ts +++ b/packages/backend/src/services/chart/charts/ap-request.ts @@ -12,7 +12,12 @@ export default class ApRequestChart extends Chart { } @autobind - protected async queryCurrentState(): Promise>> { + protected async tickMajor(): Promise>> { + return {}; + } + + @autobind + protected async tickMinor(): Promise>> { return {}; } diff --git a/packages/backend/src/services/chart/charts/drive.ts b/packages/backend/src/services/chart/charts/drive.ts index 2f00adae2b..288689784e 100644 --- a/packages/backend/src/services/chart/charts/drive.ts +++ b/packages/backend/src/services/chart/charts/drive.ts @@ -15,7 +15,12 @@ export default class DriveChart extends Chart { } @autobind - protected async queryCurrentState(): Promise>> { + protected async tickMajor(): Promise>> { + return {}; + } + + @autobind + protected async tickMinor(): Promise>> { return {}; } diff --git a/packages/backend/src/services/chart/charts/entities/federation.ts b/packages/backend/src/services/chart/charts/entities/federation.ts index 0c8c20991d..6b2089f0b4 100644 --- a/packages/backend/src/services/chart/charts/entities/federation.ts +++ b/packages/backend/src/services/chart/charts/entities/federation.ts @@ -3,12 +3,11 @@ import Chart from '../../core'; export const name = 'federation'; export const schema = { - 'instance.total': { accumulate: true }, - 'instance.inc': { range: 'small' }, - 'instance.dec': { range: 'small' }, 'deliveredInstances': { uniqueIncrement: true, range: 'small' }, 'inboxInstances': { uniqueIncrement: true, range: 'small' }, 'stalled': { uniqueIncrement: true, range: 'small' }, + 'sub': { accumulate: true, range: 'small' }, + 'pub': { accumulate: true, range: 'small' }, } as const; export const entity = Chart.schemaToEntity(name, schema); diff --git a/packages/backend/src/services/chart/charts/federation.ts b/packages/backend/src/services/chart/charts/federation.ts index 19c75c98ad..211ba1debc 100644 --- a/packages/backend/src/services/chart/charts/federation.ts +++ b/packages/backend/src/services/chart/charts/federation.ts @@ -1,6 +1,6 @@ import autobind from 'autobind-decorator'; import Chart, { KVs } from '../core'; -import { Instances } from '@/models/index'; +import { Followings } from '@/models/index'; import { name, schema } from './entities/federation'; /** @@ -13,23 +13,30 @@ export default class FederationChart extends Chart { } @autobind - protected async queryCurrentState(): Promise>> { - const [total] = await Promise.all([ - Instances.count({}), - ]); - + protected async tickMajor(): Promise>> { return { - 'instance.total': total, }; } @autobind - public async update(isAdditional: boolean): Promise { - await this.commit({ - 'instance.total': isAdditional ? 1 : -1, - 'instance.inc': isAdditional ? 1 : 0, - 'instance.dec': isAdditional ? 0 : 1, - }); + protected async tickMinor(): Promise>> { + const [sub, pub] = await Promise.all([ + Followings.createQueryBuilder('following') + .select('COUNT(DISTINCT following.followeeHost)') + .where('following.followeeHost IS NOT NULL') + .getRawOne() + .then(x => parseInt(x.count, 10)), + Followings.createQueryBuilder('following') + .select('COUNT(DISTINCT following.followerHost)') + .where('following.followerHost IS NOT NULL') + .getRawOne() + .then(x => parseInt(x.count, 10)), + ]); + + return { + 'sub': sub, + 'pub': pub, + }; } @autobind diff --git a/packages/backend/src/services/chart/charts/hashtag.ts b/packages/backend/src/services/chart/charts/hashtag.ts index 0b7bc467d2..cbae686833 100644 --- a/packages/backend/src/services/chart/charts/hashtag.ts +++ b/packages/backend/src/services/chart/charts/hashtag.ts @@ -14,7 +14,12 @@ export default class HashtagChart extends Chart { } @autobind - protected async queryCurrentState(): Promise>> { + protected async tickMajor(): Promise>> { + return {}; + } + + @autobind + protected async tickMinor(): Promise>> { return {}; } diff --git a/packages/backend/src/services/chart/charts/instance.ts b/packages/backend/src/services/chart/charts/instance.ts index 5ea4d567e1..930ac4729b 100644 --- a/packages/backend/src/services/chart/charts/instance.ts +++ b/packages/backend/src/services/chart/charts/instance.ts @@ -16,7 +16,7 @@ export default class InstanceChart extends Chart { } @autobind - protected async queryCurrentState(group: string): Promise>> { + protected async tickMajor(group: string): Promise>> { const [ notesCount, usersCount, @@ -42,6 +42,11 @@ export default class InstanceChart extends Chart { }; } + @autobind + protected async tickMinor(): Promise>> { + return {}; + } + @autobind public async requestReceived(host: string): Promise { await this.commit({ diff --git a/packages/backend/src/services/chart/charts/notes.ts b/packages/backend/src/services/chart/charts/notes.ts index 5c56a9a718..624ee5db28 100644 --- a/packages/backend/src/services/chart/charts/notes.ts +++ b/packages/backend/src/services/chart/charts/notes.ts @@ -15,7 +15,7 @@ export default class NotesChart extends Chart { } @autobind - protected async queryCurrentState(): Promise>> { + protected async tickMajor(): Promise>> { const [localCount, remoteCount] = await Promise.all([ Notes.count({ userHost: null }), Notes.count({ userHost: Not(IsNull()) }), @@ -27,6 +27,11 @@ export default class NotesChart extends Chart { }; } + @autobind + protected async tickMinor(): Promise>> { + return {}; + } + @autobind public async update(note: Note, isAdditional: boolean): Promise { const prefix = note.userHost === null ? 'local' : 'remote'; diff --git a/packages/backend/src/services/chart/charts/per-user-drive.ts b/packages/backend/src/services/chart/charts/per-user-drive.ts index 969ed018f9..ae9e8c5694 100644 --- a/packages/backend/src/services/chart/charts/per-user-drive.ts +++ b/packages/backend/src/services/chart/charts/per-user-drive.ts @@ -14,7 +14,7 @@ export default class PerUserDriveChart extends Chart { } @autobind - protected async queryCurrentState(group: string): Promise>> { + protected async tickMajor(group: string): Promise>> { const [count, size] = await Promise.all([ DriveFiles.count({ userId: group }), DriveFiles.calcDriveUsageOf(group), @@ -26,6 +26,11 @@ export default class PerUserDriveChart extends Chart { }; } + @autobind + protected async tickMinor(): Promise>> { + return {}; + } + @autobind public async update(file: DriveFile, isAdditional: boolean): Promise { const fileSizeKb = file.size / 1000; diff --git a/packages/backend/src/services/chart/charts/per-user-following.ts b/packages/backend/src/services/chart/charts/per-user-following.ts index cdd0aad947..0b39881c14 100644 --- a/packages/backend/src/services/chart/charts/per-user-following.ts +++ b/packages/backend/src/services/chart/charts/per-user-following.ts @@ -15,7 +15,7 @@ export default class PerUserFollowingChart extends Chart { } @autobind - protected async queryCurrentState(group: string): Promise>> { + protected async tickMajor(group: string): Promise>> { const [ localFollowingsCount, localFollowersCount, @@ -36,6 +36,11 @@ export default class PerUserFollowingChart extends Chart { }; } + @autobind + protected async tickMinor(): Promise>> { + return {}; + } + @autobind public async update(follower: { id: User['id']; host: User['host']; }, followee: { id: User['id']; host: User['host']; }, isFollow: boolean): Promise { const prefixFollower = Users.isLocalUser(follower) ? 'local' : 'remote'; diff --git a/packages/backend/src/services/chart/charts/per-user-notes.ts b/packages/backend/src/services/chart/charts/per-user-notes.ts index 6a4f0363b2..01a2785158 100644 --- a/packages/backend/src/services/chart/charts/per-user-notes.ts +++ b/packages/backend/src/services/chart/charts/per-user-notes.ts @@ -15,7 +15,7 @@ export default class PerUserNotesChart extends Chart { } @autobind - protected async queryCurrentState(group: string): Promise>> { + protected async tickMajor(group: string): Promise>> { const [count] = await Promise.all([ Notes.count({ userId: group }), ]); @@ -25,6 +25,11 @@ export default class PerUserNotesChart extends Chart { }; } + @autobind + protected async tickMinor(): Promise>> { + return {}; + } + @autobind public async update(user: { id: User['id'] }, note: Note, isAdditional: boolean): Promise { await this.commit({ diff --git a/packages/backend/src/services/chart/charts/per-user-reactions.ts b/packages/backend/src/services/chart/charts/per-user-reactions.ts index 2ec347f40a..59af0e86c0 100644 --- a/packages/backend/src/services/chart/charts/per-user-reactions.ts +++ b/packages/backend/src/services/chart/charts/per-user-reactions.ts @@ -15,7 +15,12 @@ export default class PerUserReactionsChart extends Chart { } @autobind - protected async queryCurrentState(group: string): Promise>> { + protected async tickMajor(group: string): Promise>> { + return {}; + } + + @autobind + protected async tickMinor(): Promise>> { return {}; } diff --git a/packages/backend/src/services/chart/charts/test-grouped.ts b/packages/backend/src/services/chart/charts/test-grouped.ts index 5f0b1aafdc..19b2135849 100644 --- a/packages/backend/src/services/chart/charts/test-grouped.ts +++ b/packages/backend/src/services/chart/charts/test-grouped.ts @@ -14,12 +14,17 @@ export default class TestGroupedChart extends Chart { } @autobind - protected async queryCurrentState(group: string): Promise>> { + protected async tickMajor(group: string): Promise>> { return { 'foo.total': this.total[group], }; } + @autobind + protected async tickMinor(): Promise>> { + return {}; + } + @autobind public async increment(group: string): Promise { if (this.total[group] == null) this.total[group] = 0; diff --git a/packages/backend/src/services/chart/charts/test-intersection.ts b/packages/backend/src/services/chart/charts/test-intersection.ts index c6ba71a956..6fd780f9b9 100644 --- a/packages/backend/src/services/chart/charts/test-intersection.ts +++ b/packages/backend/src/services/chart/charts/test-intersection.ts @@ -12,7 +12,12 @@ export default class TestIntersectionChart extends Chart { } @autobind - protected async queryCurrentState(): Promise>> { + protected async tickMajor(): Promise>> { + return {}; + } + + @autobind + protected async tickMinor(): Promise>> { return {}; } diff --git a/packages/backend/src/services/chart/charts/test-unique.ts b/packages/backend/src/services/chart/charts/test-unique.ts index e67036acef..2c9cc2fd6a 100644 --- a/packages/backend/src/services/chart/charts/test-unique.ts +++ b/packages/backend/src/services/chart/charts/test-unique.ts @@ -12,7 +12,12 @@ export default class TestUniqueChart extends Chart { } @autobind - protected async queryCurrentState(): Promise>> { + protected async tickMajor(): Promise>> { + return {}; + } + + @autobind + protected async tickMinor(): Promise>> { return {}; } diff --git a/packages/backend/src/services/chart/charts/test.ts b/packages/backend/src/services/chart/charts/test.ts index 878acd51be..b539625c10 100644 --- a/packages/backend/src/services/chart/charts/test.ts +++ b/packages/backend/src/services/chart/charts/test.ts @@ -14,12 +14,17 @@ export default class TestChart extends Chart { } @autobind - protected async queryCurrentState(): Promise>> { + protected async tickMajor(): Promise>> { return { 'foo.total': this.total, }; } + @autobind + protected async tickMinor(): Promise>> { + return {}; + } + @autobind public async increment(): Promise { this.total++; diff --git a/packages/backend/src/services/chart/charts/users.ts b/packages/backend/src/services/chart/charts/users.ts index 50fca3a8d6..70ef89f8cd 100644 --- a/packages/backend/src/services/chart/charts/users.ts +++ b/packages/backend/src/services/chart/charts/users.ts @@ -15,7 +15,7 @@ export default class UsersChart extends Chart { } @autobind - protected async queryCurrentState(): Promise>> { + protected async tickMajor(): Promise>> { const [localCount, remoteCount] = await Promise.all([ Users.count({ host: null }), Users.count({ host: Not(IsNull()) }), @@ -27,6 +27,11 @@ export default class UsersChart extends Chart { }; } + @autobind + protected async tickMinor(): Promise>> { + return {}; + } + @autobind public async update(user: { id: User['id'], host: User['host'] }, isAdditional: boolean): Promise { const prefix = Users.isLocalUser(user) ? 'local' : 'remote'; diff --git a/packages/backend/src/services/chart/core.ts b/packages/backend/src/services/chart/core.ts index 5888e1a144..61a7575706 100644 --- a/packages/backend/src/services/chart/core.ts +++ b/packages/backend/src/services/chart/core.ts @@ -81,7 +81,15 @@ export default abstract class Chart { protected repositoryForHour: Repository>; protected repositoryForDay: Repository>; - protected abstract queryCurrentState(group: string | null): Promise>>; + /** + * 1日に一回程度実行されれば良いような計算処理を入れる(主にCASCADE削除などアプリケーション側で感知できない変動によるズレの修正用) + */ + protected abstract tickMajor(group: string | null): Promise>>; + + /** + * 少なくとも最小スパン内に1回は実行されて欲しい計算処理を入れる + */ + protected abstract tickMinor(group: string | null): Promise>>; @autobind private static convertSchemaToColumnDefinitions(schema: Schema): Record { @@ -445,8 +453,8 @@ export default abstract class Chart { } @autobind - public async resync(group: string | null = null): Promise { - const data = await this.queryCurrentState(group); + public async tick(major: boolean, group: string | null = null): Promise { + const data = major ? await this.tickMajor(group) : await this.tickMinor(group); const columns = {} as Record; for (const [k, v] of Object.entries(data)) { @@ -480,6 +488,11 @@ export default abstract class Chart { update(logHour, logDay)); } + @autobind + public resync(group: string | null = null): Promise { + return this.tick(true, group); + } + @autobind public async clean(): Promise { const current = dateUTC(Chart.getCurrentDate()); -- cgit v1.2.3-freya