From c1b264e4e9686804e1b8ea17ba39753c41bd205b Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 6 Feb 2022 00:13:52 +0900 Subject: Improve chart engine (#8253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * wip * wip * wip * wip * wip * wip * Update core.ts * wip * wip * #7361 * delete network chart * federationChart強化 apRequestChart追加 * tweak --- .../src/services/chart/charts/active-users.ts | 35 +- .../src/services/chart/charts/ap-request.ts | 39 +++ .../backend/src/services/chart/charts/drive.ts | 88 +---- .../services/chart/charts/entities/active-users.ts | 32 +- .../services/chart/charts/entities/ap-request.ts | 11 + .../src/services/chart/charts/entities/drive.ts | 74 +---- .../services/chart/charts/entities/federation.ts | 29 +- .../src/services/chart/charts/entities/hashtag.ts | 32 +- .../src/services/chart/charts/entities/instance.ts | 175 ++-------- .../src/services/chart/charts/entities/network.ts | 32 -- .../src/services/chart/charts/entities/notes.ts | 66 +--- .../chart/charts/entities/per-user-drive.ts | 59 +--- .../chart/charts/entities/per-user-following.ts | 96 +----- .../chart/charts/entities/per-user-notes.ts | 47 +-- .../chart/charts/entities/per-user-reactions.ts | 28 +- .../services/chart/charts/entities/test-grouped.ts | 29 +- .../services/chart/charts/entities/test-unique.ts | 15 +- .../src/services/chart/charts/entities/test.ts | 29 +- .../src/services/chart/charts/entities/users.ts | 48 +-- .../src/services/chart/charts/federation.ts | 58 ++-- .../backend/src/services/chart/charts/hashtag.ts | 35 +- .../backend/src/services/chart/charts/instance.ts | 182 +++------- .../backend/src/services/chart/charts/network.ts | 49 --- .../backend/src/services/chart/charts/notes.ts | 86 +---- .../src/services/chart/charts/per-user-drive.ts | 55 +-- .../services/chart/charts/per-user-following.ts | 104 +----- .../src/services/chart/charts/per-user-notes.ts | 59 +--- .../services/chart/charts/per-user-reactions.ts | 31 +- .../src/services/chart/charts/test-grouped.ts | 42 +-- .../src/services/chart/charts/test-unique.ts | 23 +- packages/backend/src/services/chart/charts/test.ts | 51 +-- .../backend/src/services/chart/charts/users.ts | 60 +--- packages/backend/src/services/chart/core.ts | 368 +++++++++++---------- packages/backend/src/services/chart/entities.ts | 4 +- packages/backend/src/services/chart/index.ts | 6 +- 35 files changed, 526 insertions(+), 1651 deletions(-) create mode 100644 packages/backend/src/services/chart/charts/ap-request.ts create mode 100644 packages/backend/src/services/chart/charts/entities/ap-request.ts delete mode 100644 packages/backend/src/services/chart/charts/entities/network.ts delete mode 100644 packages/backend/src/services/chart/charts/network.ts (limited to 'packages/backend/src/services/chart') diff --git a/packages/backend/src/services/chart/charts/active-users.ts b/packages/backend/src/services/chart/charts/active-users.ts index 9490101e36..9e5c332d3b 100644 --- a/packages/backend/src/services/chart/charts/active-users.ts +++ b/packages/backend/src/services/chart/charts/active-users.ts @@ -1,51 +1,28 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; +import Chart, { KVs } from '../core'; import { User } from '@/models/entities/user'; -import { SchemaType } from '@/misc/schema'; import { Users } from '@/models/index'; import { name, schema } from './entities/active-users'; -type ActiveUsersLog = SchemaType; - /** * アクティブユーザーに関するチャート */ // eslint-disable-next-line import/no-default-export -export default class ActiveUsersChart extends Chart { +export default class ActiveUsersChart extends Chart { constructor() { super(name, schema); } @autobind - protected genNewLog(latest: ActiveUsersLog): DeepPartial { - return {}; - } - - @autobind - protected aggregate(logs: ActiveUsersLog[]): ActiveUsersLog { - return { - local: { - users: logs.reduce((a, b) => a.concat(b.local.users), [] as ActiveUsersLog['local']['users']), - }, - remote: { - users: logs.reduce((a, b) => a.concat(b.remote.users), [] as ActiveUsersLog['remote']['users']), - }, - }; - } - - @autobind - protected async fetchActual(): Promise> { + protected async queryCurrentState(): Promise>> { return {}; } @autobind public async update(user: { id: User['id'], host: User['host'] }): Promise { - const update: Obj = { - users: [user.id], - }; - - await this.inc({ - [Users.isLocalUser(user) ? 'local' : 'remote']: update, + await this.commit({ + 'local.users': Users.isLocalUser(user) ? [user.id] : [], + 'remote.users': Users.isLocalUser(user) ? [] : [user.id], }); } } diff --git a/packages/backend/src/services/chart/charts/ap-request.ts b/packages/backend/src/services/chart/charts/ap-request.ts new file mode 100644 index 0000000000..bac5e425c8 --- /dev/null +++ b/packages/backend/src/services/chart/charts/ap-request.ts @@ -0,0 +1,39 @@ +import autobind from 'autobind-decorator'; +import Chart, { KVs } from '../core'; +import { name, schema } from './entities/ap-request'; + +/** + * Chart about ActivityPub requests + */ +// eslint-disable-next-line import/no-default-export +export default class ApRequestChart extends Chart { + constructor() { + super(name, schema); + } + + @autobind + protected async queryCurrentState(): Promise>> { + return {}; + } + + @autobind + public async deliverSucc(): Promise { + await this.commit({ + 'deliverSucceeded': 1, + }); + } + + @autobind + public async deliverFail(): Promise { + await this.commit({ + 'deliverFailed': 1, + }); + } + + @autobind + public async inbox(): Promise { + await this.commit({ + 'inboxReceived': 1, + }); + } +} diff --git a/packages/backend/src/services/chart/charts/drive.ts b/packages/backend/src/services/chart/charts/drive.ts index 06cf7ebeeb..2f00adae2b 100644 --- a/packages/backend/src/services/chart/charts/drive.ts +++ b/packages/backend/src/services/chart/charts/drive.ts @@ -1,95 +1,37 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; +import Chart, { KVs } from '../core'; import { DriveFiles } from '@/models/index'; import { Not, IsNull } from 'typeorm'; import { DriveFile } from '@/models/entities/drive-file'; import { name, schema } from './entities/drive'; -type DriveLog = SchemaType; - /** * ドライブに関するチャート */ // eslint-disable-next-line import/no-default-export -export default class DriveChart extends Chart { +export default class DriveChart extends Chart { constructor() { super(name, schema); } @autobind - protected genNewLog(latest: DriveLog): DeepPartial { - return { - local: { - totalCount: latest.local.totalCount, - totalSize: latest.local.totalSize, - }, - remote: { - totalCount: latest.remote.totalCount, - totalSize: latest.remote.totalSize, - }, - }; - } - - @autobind - protected aggregate(logs: DriveLog[]): DriveLog { - return { - local: { - totalCount: logs[0].local.totalCount, - totalSize: logs[0].local.totalSize, - incCount: logs.reduce((a, b) => a + b.local.incCount, 0), - incSize: logs.reduce((a, b) => a + b.local.incSize, 0), - decCount: logs.reduce((a, b) => a + b.local.decCount, 0), - decSize: logs.reduce((a, b) => a + b.local.decSize, 0), - }, - remote: { - totalCount: logs[0].remote.totalCount, - totalSize: logs[0].remote.totalSize, - incCount: logs.reduce((a, b) => a + b.remote.incCount, 0), - incSize: logs.reduce((a, b) => a + b.remote.incSize, 0), - decCount: logs.reduce((a, b) => a + b.remote.decCount, 0), - decSize: logs.reduce((a, b) => a + b.remote.decSize, 0), - }, - }; - } - - @autobind - protected async fetchActual(): Promise> { - const [localCount, remoteCount, localSize, remoteSize] = await Promise.all([ - DriveFiles.count({ userHost: null }), - DriveFiles.count({ userHost: Not(IsNull()) }), - DriveFiles.calcDriveUsageOfLocal(), - DriveFiles.calcDriveUsageOfRemote(), - ]); - - return { - local: { - totalCount: localCount, - totalSize: localSize, - }, - remote: { - totalCount: remoteCount, - totalSize: remoteSize, - }, - }; + protected async queryCurrentState(): Promise>> { + return {}; } @autobind public async update(file: DriveFile, isAdditional: boolean): Promise { - const update: Obj = {}; - - update.totalCount = isAdditional ? 1 : -1; - update.totalSize = isAdditional ? file.size : -file.size; - if (isAdditional) { - update.incCount = 1; - update.incSize = file.size; - } else { - update.decCount = 1; - update.decSize = file.size; - } - - await this.inc({ - [file.userHost === null ? 'local' : 'remote']: update, + const fileSizeKb = file.size / 1000; + await this.commit(file.userHost === null ? { + 'local.incCount': isAdditional ? 1 : 0, + 'local.incSize': isAdditional ? fileSizeKb : 0, + 'local.decCount': isAdditional ? 0 : 1, + 'local.decSize': isAdditional ? 0 : fileSizeKb, + } : { + 'remote.incCount': isAdditional ? 1 : 0, + 'remote.incSize': isAdditional ? fileSizeKb : 0, + 'remote.decCount': isAdditional ? 0 : 1, + 'remote.decSize': isAdditional ? 0 : fileSizeKb, }); } } diff --git a/packages/backend/src/services/chart/charts/entities/active-users.ts b/packages/backend/src/services/chart/charts/entities/active-users.ts index d6b49c86c3..3caa938d35 100644 --- a/packages/backend/src/services/chart/charts/entities/active-users.ts +++ b/packages/backend/src/services/chart/charts/entities/active-users.ts @@ -2,35 +2,9 @@ import Chart from '../../core'; export const name = 'activeUsers'; -const logSchema = { - /** - * アクティブユーザー - */ - users: { - type: 'array' as const, - optional: false as const, nullable: false as const, - items: { - type: 'string' as const, - optional: false as const, nullable: false as const, - }, - }, -}; - export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - local: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - remote: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - }, -}; + 'local.users': { uniqueIncrement: true }, + 'remote.users': { uniqueIncrement: true }, +} as const; export const entity = Chart.schemaToEntity(name, schema); diff --git a/packages/backend/src/services/chart/charts/entities/ap-request.ts b/packages/backend/src/services/chart/charts/entities/ap-request.ts new file mode 100644 index 0000000000..21fb40d138 --- /dev/null +++ b/packages/backend/src/services/chart/charts/entities/ap-request.ts @@ -0,0 +1,11 @@ +import Chart from '../../core'; + +export const name = 'apRequest'; + +export const schema = { + 'deliverFailed': { }, + 'deliverSucceeded': { }, + 'inboxReceived': { }, +} as const; + +export const entity = Chart.schemaToEntity(name, schema); diff --git a/packages/backend/src/services/chart/charts/entities/drive.ts b/packages/backend/src/services/chart/charts/entities/drive.ts index 3362cbd4cb..c5cdfd85bd 100644 --- a/packages/backend/src/services/chart/charts/entities/drive.ts +++ b/packages/backend/src/services/chart/charts/entities/drive.ts @@ -2,71 +2,15 @@ import Chart from '../../core'; export const name = 'drive'; -const logSchema = { - /** - * 集計期間時点での、全ドライブファイル数 - */ - totalCount: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 集計期間時点での、全ドライブファイルの合計サイズ - */ - totalSize: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 増加したドライブファイル数 - */ - incCount: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 増加したドライブ使用量 - */ - incSize: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 減少したドライブファイル数 - */ - decCount: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 減少したドライブ使用量 - */ - decSize: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, -}; - export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - local: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - remote: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - }, -}; + 'local.incCount': {}, + 'local.incSize': {}, // in kilobyte + 'local.decCount': {}, + 'local.decSize': {}, // in kilobyte + 'remote.incCount': {}, + 'remote.incSize': {}, // in kilobyte + 'remote.decCount': {}, + 'remote.decSize': {}, // in kilobyte +} as const; export const entity = Chart.schemaToEntity(name, schema); diff --git a/packages/backend/src/services/chart/charts/entities/federation.ts b/packages/backend/src/services/chart/charts/entities/federation.ts index 836116bd06..5a76c4918c 100644 --- a/packages/backend/src/services/chart/charts/entities/federation.ts +++ b/packages/backend/src/services/chart/charts/entities/federation.ts @@ -3,28 +3,11 @@ import Chart from '../../core'; export const name = 'federation'; export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - instance: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - }, -}; + 'instance.total': { accumulate: true }, + 'instance.inc': { range: 'small' }, + 'instance.dec': { range: 'small' }, + 'deliveredInstances': { uniqueIncrement: true, range: 'small' }, + 'inboxInstances': { uniqueIncrement: true, range: 'small' }, +} as const; export const entity = Chart.schemaToEntity(name, schema); diff --git a/packages/backend/src/services/chart/charts/entities/hashtag.ts b/packages/backend/src/services/chart/charts/entities/hashtag.ts index 43e15456a5..bd2ae38a16 100644 --- a/packages/backend/src/services/chart/charts/entities/hashtag.ts +++ b/packages/backend/src/services/chart/charts/entities/hashtag.ts @@ -2,35 +2,9 @@ import Chart from '../../core'; export const name = 'hashtag'; -const logSchema = { - /** - * 投稿したユーザー - */ - users: { - type: 'array' as const, - optional: false as const, nullable: false as const, - items: { - type: 'string' as const, - optional: false as const, nullable: false as const, - }, - }, -}; - export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - local: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - remote: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - }, -}; + 'local.users': { uniqueIncrement: true }, + 'remote.users': { uniqueIncrement: true }, +} as const; export const entity = Chart.schemaToEntity(name, schema, true); diff --git a/packages/backend/src/services/chart/charts/entities/instance.ts b/packages/backend/src/services/chart/charts/entities/instance.ts index 9d1f651dbb..0ff9e7b6b8 100644 --- a/packages/backend/src/services/chart/charts/entities/instance.ts +++ b/packages/backend/src/services/chart/charts/entities/instance.ts @@ -3,156 +3,29 @@ import Chart from '../../core'; export const name = 'instance'; export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - requests: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - failed: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - succeeded: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - received: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - - notes: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - diffs: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - normal: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - reply: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - renote: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - }, - }, - - users: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - - following: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - - followers: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - - drive: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - totalFiles: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - totalUsage: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - incFiles: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - incUsage: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - decFiles: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - decUsage: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - }, -}; + 'requests.failed': { range: 'small' }, + 'requests.succeeded': { range: 'small' }, + 'requests.received': { range: 'small' }, + 'notes.total': { accumulate: true }, + 'notes.inc': {}, + 'notes.dec': {}, + 'notes.diffs.normal': {}, + 'notes.diffs.reply': {}, + 'notes.diffs.renote': {}, + 'users.total': { accumulate: true }, + 'users.inc': { range: 'small' }, + 'users.dec': { range: 'small' }, + 'following.total': { accumulate: true }, + 'following.inc': { range: 'small' }, + 'following.dec': { range: 'small' }, + 'followers.total': { accumulate: true }, + 'followers.inc': { range: 'small' }, + 'followers.dec': { range: 'small' }, + 'drive.totalFiles': { accumulate: true }, + 'drive.incFiles': {}, + 'drive.decFiles': {}, + 'drive.incUsage': {}, // in kilobyte + 'drive.decUsage': {}, // in kilobyte +} as const; export const entity = Chart.schemaToEntity(name, schema, true); diff --git a/packages/backend/src/services/chart/charts/entities/network.ts b/packages/backend/src/services/chart/charts/entities/network.ts deleted file mode 100644 index 3d4fffb855..0000000000 --- a/packages/backend/src/services/chart/charts/entities/network.ts +++ /dev/null @@ -1,32 +0,0 @@ -import Chart from '../../core'; - -export const name = 'network'; - -export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - incomingRequests: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - outgoingRequests: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - totalTime: { // TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - incomingBytes: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - outgoingBytes: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, -}; - -export const entity = Chart.schemaToEntity(name, schema); diff --git a/packages/backend/src/services/chart/charts/entities/notes.ts b/packages/backend/src/services/chart/charts/entities/notes.ts index 554d3abe12..296328f102 100644 --- a/packages/backend/src/services/chart/charts/entities/notes.ts +++ b/packages/backend/src/services/chart/charts/entities/notes.ts @@ -2,59 +2,19 @@ import Chart from '../../core'; export const name = 'notes'; -const logSchema = { - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - diffs: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - normal: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - reply: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - renote: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, -}; - export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - local: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - remote: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - }, -}; + 'local.total': { accumulate: true }, + 'local.inc': {}, + 'local.dec': {}, + 'local.diffs.normal': {}, + 'local.diffs.reply': {}, + 'local.diffs.renote': {}, + 'remote.total': { accumulate: true }, + 'remote.inc': {}, + 'remote.dec': {}, + 'remote.diffs.normal': {}, + 'remote.diffs.reply': {}, + 'remote.diffs.renote': {}, +} as const; export const entity = Chart.schemaToEntity(name, schema); diff --git a/packages/backend/src/services/chart/charts/entities/per-user-drive.ts b/packages/backend/src/services/chart/charts/entities/per-user-drive.ts index ebf64e733e..00d85b1620 100644 --- a/packages/backend/src/services/chart/charts/entities/per-user-drive.ts +++ b/packages/backend/src/services/chart/charts/entities/per-user-drive.ts @@ -3,57 +3,12 @@ import Chart from '../../core'; export const name = 'perUserDrive'; export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - /** - * 集計期間時点での、全ドライブファイル数 - */ - totalCount: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 集計期間時点での、全ドライブファイルの合計サイズ - */ - totalSize: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 増加したドライブファイル数 - */ - incCount: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 増加したドライブ使用量 - */ - incSize: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 減少したドライブファイル数 - */ - decCount: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 減少したドライブ使用量 - */ - decSize: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, -}; + 'totalCount': { accumulate: true }, + 'totalSize': { accumulate: true }, // in kilobyte + 'incCount': { range: 'small' }, + 'incSize': {}, // in kilobyte + 'decCount': { range: 'small' }, + 'decSize': {}, // in kilobyte +} as const; export const entity = Chart.schemaToEntity(name, schema, true); diff --git a/packages/backend/src/services/chart/charts/entities/per-user-following.ts b/packages/backend/src/services/chart/charts/entities/per-user-following.ts index 8016c5fe97..1efd4977fc 100644 --- a/packages/backend/src/services/chart/charts/entities/per-user-following.ts +++ b/packages/backend/src/services/chart/charts/entities/per-user-following.ts @@ -2,89 +2,19 @@ import Chart from '../../core'; export const name = 'perUserFollowing'; -const logSchema = { - /** - * フォローしている - */ - followings: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - /** - * フォローしている合計 - */ - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * フォローした数 - */ - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * フォロー解除した数 - */ - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - - /** - * フォローされている - */ - followers: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - /** - * フォローされている合計 - */ - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * フォローされた数 - */ - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * フォロー解除された数 - */ - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, -}; - export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - local: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - remote: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - }, -}; + 'local.followings.total': { accumulate: true }, + 'local.followings.inc': { range: 'small' }, + 'local.followings.dec': { range: 'small' }, + 'local.followers.total': { accumulate: true }, + 'local.followers.inc': { range: 'small' }, + 'local.followers.dec': { range: 'small' }, + 'remote.followings.total': { accumulate: true }, + 'remote.followings.inc': { range: 'small' }, + 'remote.followings.dec': { range: 'small' }, + 'remote.followers.total': { accumulate: true }, + 'remote.followers.inc': { range: 'small' }, + 'remote.followers.dec': { range: 'small' }, +} as const; export const entity = Chart.schemaToEntity(name, schema, true); diff --git a/packages/backend/src/services/chart/charts/entities/per-user-notes.ts b/packages/backend/src/services/chart/charts/entities/per-user-notes.ts index d8f645b36e..bf7e4422be 100644 --- a/packages/backend/src/services/chart/charts/entities/per-user-notes.ts +++ b/packages/backend/src/services/chart/charts/entities/per-user-notes.ts @@ -3,45 +3,12 @@ import Chart from '../../core'; export const name = 'perUserNotes'; export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - diffs: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - normal: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - reply: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - renote: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - }, -}; + 'total': { accumulate: true }, + 'inc': { range: 'small' }, + 'dec': { range: 'small' }, + 'diffs.normal': { range: 'small' }, + 'diffs.reply': { range: 'small' }, + 'diffs.renote': { range: 'small' }, +} as const; export const entity = Chart.schemaToEntity(name, schema, true); diff --git a/packages/backend/src/services/chart/charts/entities/per-user-reactions.ts b/packages/backend/src/services/chart/charts/entities/per-user-reactions.ts index bcb7012661..ab315d24c9 100644 --- a/packages/backend/src/services/chart/charts/entities/per-user-reactions.ts +++ b/packages/backend/src/services/chart/charts/entities/per-user-reactions.ts @@ -2,31 +2,9 @@ import Chart from '../../core'; export const name = 'perUserReaction'; -const logSchema = { - /** - * 被リアクション数 - */ - count: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, -}; - export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - local: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - remote: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - }, -}; + 'local.count': { range: 'small' }, + 'remote.count': { range: 'small' }, +} as const; export const entity = Chart.schemaToEntity(name, schema, true); diff --git a/packages/backend/src/services/chart/charts/entities/test-grouped.ts b/packages/backend/src/services/chart/charts/entities/test-grouped.ts index ca1c8c5700..78c2bbd548 100644 --- a/packages/backend/src/services/chart/charts/entities/test-grouped.ts +++ b/packages/backend/src/services/chart/charts/entities/test-grouped.ts @@ -3,30 +3,9 @@ import Chart from '../../core'; export const name = 'testGrouped'; export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - foo: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - }, -}; + 'foo.total': { accumulate: true }, + 'foo.inc': {}, + 'foo.dec': {}, +} as const; export const entity = Chart.schemaToEntity(name, schema, true); diff --git a/packages/backend/src/services/chart/charts/entities/test-unique.ts b/packages/backend/src/services/chart/charts/entities/test-unique.ts index 2e917ee9ed..dc7c1520e1 100644 --- a/packages/backend/src/services/chart/charts/entities/test-unique.ts +++ b/packages/backend/src/services/chart/charts/entities/test-unique.ts @@ -3,18 +3,7 @@ import Chart from '../../core'; export const name = 'testUnique'; export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - foo: { - type: 'array' as const, - optional: false as const, nullable: false as const, - items: { - type: 'string' as const, - optional: false as const, nullable: false as const, - }, - }, - }, -}; + 'foo': { uniqueIncrement: true }, +} as const; export const entity = Chart.schemaToEntity(name, schema); diff --git a/packages/backend/src/services/chart/charts/entities/test.ts b/packages/backend/src/services/chart/charts/entities/test.ts index fa536ff2cf..edfa4c524b 100644 --- a/packages/backend/src/services/chart/charts/entities/test.ts +++ b/packages/backend/src/services/chart/charts/entities/test.ts @@ -3,30 +3,9 @@ import Chart from '../../core'; export const name = 'test'; export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - foo: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - }, - }, - }, -}; + 'foo.total': { accumulate: true }, + 'foo.inc': {}, + 'foo.dec': {}, +} as const; export const entity = Chart.schemaToEntity(name, schema); diff --git a/packages/backend/src/services/chart/charts/entities/users.ts b/packages/backend/src/services/chart/charts/entities/users.ts index 08d51c9414..d2cec72497 100644 --- a/packages/backend/src/services/chart/charts/entities/users.ts +++ b/packages/backend/src/services/chart/charts/entities/users.ts @@ -2,47 +2,13 @@ import Chart from '../../core'; export const name = 'users'; -const logSchema = { - /** - * 集計期間時点での、全ユーザー数 - */ - total: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 増加したユーザー数 - */ - inc: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, - - /** - * 減少したユーザー数 - */ - dec: { - type: 'number' as const, - optional: false as const, nullable: false as const, - }, -}; - export const schema = { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: { - local: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - remote: { - type: 'object' as const, - optional: false as const, nullable: false as const, - properties: logSchema, - }, - }, -}; + 'local.total': { accumulate: true }, + 'local.inc': { range: 'small' }, + 'local.dec': { range: 'small' }, + 'remote.total': { accumulate: true }, + 'remote.inc': { range: 'small' }, + 'remote.dec': { range: 'small' }, +} as const; export const entity = Chart.schemaToEntity(name, schema); diff --git a/packages/backend/src/services/chart/charts/federation.ts b/packages/backend/src/services/chart/charts/federation.ts index 8abb18b51f..3aa448e66f 100644 --- a/packages/backend/src/services/chart/charts/federation.ts +++ b/packages/backend/src/services/chart/charts/federation.ts @@ -1,66 +1,48 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; +import Chart, { KVs } from '../core'; import { Instances } from '@/models/index'; import { name, schema } from './entities/federation'; -type FederationLog = SchemaType; - /** * フェデレーションに関するチャート */ // eslint-disable-next-line import/no-default-export -export default class FederationChart extends Chart { +export default class FederationChart extends Chart { constructor() { super(name, schema); } @autobind - protected genNewLog(latest: FederationLog): DeepPartial { - return { - instance: { - total: latest.instance.total, - }, - }; - } - - @autobind - protected aggregate(logs: FederationLog[]): FederationLog { - return { - instance: { - total: logs[0].instance.total, - inc: logs.reduce((a, b) => a + b.instance.inc, 0), - dec: logs.reduce((a, b) => a + b.instance.dec, 0), - }, - }; - } - - @autobind - protected async fetchActual(): Promise> { + protected async queryCurrentState(): Promise>> { const [total] = await Promise.all([ Instances.count({}), ]); return { - instance: { - total: total, - }, + 'instance.total': total, }; } @autobind public async update(isAdditional: boolean): Promise { - const update: Obj = {}; + await this.commit({ + 'instance.total': isAdditional ? 1 : -1, + 'instance.inc': isAdditional ? 1 : 0, + 'instance.dec': isAdditional ? 0 : 1, + }); + } - update.total = isAdditional ? 1 : -1; - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } + @autobind + public async deliverd(host: string): Promise { + await this.commit({ + 'deliveredInstances': [host], + }); + } - await this.inc({ - instance: update, + @autobind + public async inbox(host: string): Promise { + await this.commit({ + 'inboxInstances': [host], }); } } diff --git a/packages/backend/src/services/chart/charts/hashtag.ts b/packages/backend/src/services/chart/charts/hashtag.ts index 34e0614643..0b7bc467d2 100644 --- a/packages/backend/src/services/chart/charts/hashtag.ts +++ b/packages/backend/src/services/chart/charts/hashtag.ts @@ -1,51 +1,28 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; +import Chart, { KVs } from '../core'; import { User } from '@/models/entities/user'; -import { SchemaType } from '@/misc/schema'; import { Users } from '@/models/index'; import { name, schema } from './entities/hashtag'; -type HashtagLog = SchemaType; - /** * ハッシュタグに関するチャート */ // eslint-disable-next-line import/no-default-export -export default class HashtagChart extends Chart { +export default class HashtagChart extends Chart { constructor() { super(name, schema, true); } @autobind - protected genNewLog(latest: HashtagLog): DeepPartial { - return {}; - } - - @autobind - protected aggregate(logs: HashtagLog[]): HashtagLog { - return { - local: { - users: logs.reduce((a, b) => a.concat(b.local.users), [] as HashtagLog['local']['users']), - }, - remote: { - users: logs.reduce((a, b) => a.concat(b.remote.users), [] as HashtagLog['remote']['users']), - }, - }; - } - - @autobind - protected async fetchActual(): Promise> { + protected async queryCurrentState(): Promise>> { return {}; } @autobind public async update(hashtag: string, user: { id: User['id'], host: User['host'] }): Promise { - const update: Obj = { - users: [user.id], - }; - - await this.inc({ - [Users.isLocalUser(user) ? 'local' : 'remote']: update, + await this.commit({ + 'local.users': Users.isLocalUser(user) ? [user.id] : [], + 'remote.users': Users.isLocalUser(user) ? [] : [user.id], }, hashtag); } } diff --git a/packages/backend/src/services/chart/charts/instance.ts b/packages/backend/src/services/chart/charts/instance.ts index 7f3419b69c..b893dde34c 100644 --- a/packages/backend/src/services/chart/charts/instance.ts +++ b/packages/backend/src/services/chart/charts/instance.ts @@ -1,158 +1,67 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; +import Chart, { KVs } from '../core'; import { DriveFiles, Followings, Users, Notes } from '@/models/index'; import { DriveFile } from '@/models/entities/drive-file'; import { Note } from '@/models/entities/note'; import { toPuny } from '@/misc/convert-host'; import { name, schema } from './entities/instance'; -type InstanceLog = SchemaType; - /** * インスタンスごとのチャート */ // eslint-disable-next-line import/no-default-export -export default class InstanceChart extends Chart { +export default class InstanceChart extends Chart { constructor() { super(name, schema, true); } @autobind - protected genNewLog(latest: InstanceLog): DeepPartial { - return { - notes: { - total: latest.notes.total, - }, - users: { - total: latest.users.total, - }, - following: { - total: latest.following.total, - }, - followers: { - total: latest.followers.total, - }, - drive: { - totalFiles: latest.drive.totalFiles, - totalUsage: latest.drive.totalUsage, - }, - }; - } - - @autobind - protected aggregate(logs: InstanceLog[]): InstanceLog { - return { - requests: { - failed: logs.reduce((a, b) => a + b.requests.failed, 0), - succeeded: logs.reduce((a, b) => a + b.requests.succeeded, 0), - received: logs.reduce((a, b) => a + b.requests.received, 0), - }, - notes: { - total: logs[0].notes.total, - inc: logs.reduce((a, b) => a + b.notes.inc, 0), - dec: logs.reduce((a, b) => a + b.notes.dec, 0), - diffs: { - reply: logs.reduce((a, b) => a + b.notes.diffs.reply, 0), - renote: logs.reduce((a, b) => a + b.notes.diffs.renote, 0), - normal: logs.reduce((a, b) => a + b.notes.diffs.normal, 0), - }, - }, - users: { - total: logs[0].users.total, - inc: logs.reduce((a, b) => a + b.users.inc, 0), - dec: logs.reduce((a, b) => a + b.users.dec, 0), - }, - following: { - total: logs[0].following.total, - inc: logs.reduce((a, b) => a + b.following.inc, 0), - dec: logs.reduce((a, b) => a + b.following.dec, 0), - }, - followers: { - total: logs[0].followers.total, - inc: logs.reduce((a, b) => a + b.followers.inc, 0), - dec: logs.reduce((a, b) => a + b.followers.dec, 0), - }, - drive: { - totalFiles: logs[0].drive.totalFiles, - totalUsage: logs[0].drive.totalUsage, - incFiles: logs.reduce((a, b) => a + b.drive.incFiles, 0), - incUsage: logs.reduce((a, b) => a + b.drive.incUsage, 0), - decFiles: logs.reduce((a, b) => a + b.drive.decFiles, 0), - decUsage: logs.reduce((a, b) => a + b.drive.decUsage, 0), - }, - }; - } - - @autobind - protected async fetchActual(group: string): Promise> { + protected async queryCurrentState(group: string): Promise>> { const [ notesCount, usersCount, followingCount, followersCount, driveFiles, - driveUsage, + //driveUsage, ] = await Promise.all([ Notes.count({ userHost: group }), Users.count({ host: group }), Followings.count({ followerHost: group }), Followings.count({ followeeHost: group }), DriveFiles.count({ userHost: group }), - DriveFiles.calcDriveUsageOfHost(group), + //DriveFiles.calcDriveUsageOfHost(group), ]); return { - notes: { - total: notesCount, - }, - users: { - total: usersCount, - }, - following: { - total: followingCount, - }, - followers: { - total: followersCount, - }, - drive: { - totalFiles: driveFiles, - totalUsage: driveUsage, - }, + 'notes.total': notesCount, + 'users.total': usersCount, + 'following.total': followingCount, + 'followers.total': followersCount, + 'drive.totalFiles': driveFiles, }; } @autobind public async requestReceived(host: string): Promise { - await this.inc({ - requests: { - received: 1, - }, + await this.commit({ + 'requests.received': 1, }, toPuny(host)); } @autobind public async requestSent(host: string, isSucceeded: boolean): Promise { - const update: Obj = {}; - - if (isSucceeded) { - update.succeeded = 1; - } else { - update.failed = 1; - } - - await this.inc({ - requests: update, + await this.commit({ + 'requests.succeeded': isSucceeded ? 1 : 0, + 'requests.failed': isSucceeded ? 0 : 1, }, toPuny(host)); } @autobind public async newUser(host: string): Promise { - await this.inc({ - users: { - total: 1, - inc: 1, - }, + await this.commit({ + 'users.total': 1, + 'users.inc': 1, }, toPuny(host)); } @@ -168,54 +77,43 @@ export default class InstanceChart extends Chart { diffs.normal = isAdditional ? 1 : -1; } - await this.inc({ - notes: { - total: isAdditional ? 1 : -1, - inc: isAdditional ? 1 : 0, - dec: isAdditional ? 0 : 1, - diffs: diffs, - }, + await this.commit({ + 'notes.total': isAdditional ? 1 : -1, + 'notes.inc': isAdditional ? 1 : 0, + 'notes.dec': isAdditional ? 0 : 1, + 'notes.diffs.normal': note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0, + 'notes.diffs.renote': note.renoteId != null ? (isAdditional ? 1 : -1) : 0, + 'notes.diffs.reply': note.replyId != null ? (isAdditional ? 1 : -1) : 0, }, toPuny(host)); } @autobind public async updateFollowing(host: string, isAdditional: boolean): Promise { - await this.inc({ - following: { - total: isAdditional ? 1 : -1, - inc: isAdditional ? 1 : 0, - dec: isAdditional ? 0 : 1, - }, + await this.commit({ + 'following.total': isAdditional ? 1 : -1, + 'following.inc': isAdditional ? 1 : 0, + 'following.dec': isAdditional ? 0 : 1, }, toPuny(host)); } @autobind public async updateFollowers(host: string, isAdditional: boolean): Promise { - await this.inc({ - followers: { - total: isAdditional ? 1 : -1, - inc: isAdditional ? 1 : 0, - dec: isAdditional ? 0 : 1, - }, + await this.commit({ + 'followers.total': isAdditional ? 1 : -1, + 'followers.inc': isAdditional ? 1 : 0, + 'followers.dec': isAdditional ? 0 : 1, }, toPuny(host)); } @autobind public async updateDrive(file: DriveFile, isAdditional: boolean): Promise { - const update: Obj = {}; - - update.totalFiles = isAdditional ? 1 : -1; - update.totalUsage = isAdditional ? file.size : -file.size; - if (isAdditional) { - update.incFiles = 1; - update.incUsage = file.size; - } else { - update.decFiles = 1; - update.decUsage = file.size; - } - - await this.inc({ - drive: update, + const fileSizeKb = file.size / 1000; + await this.commit({ + 'drive.totalFiles': isAdditional ? 1 : -1, + 'drive.incFiles': isAdditional ? 1 : 0, + 'drive.incUsage': isAdditional ? fileSizeKb : 0, + 'drive.decFiles': isAdditional ? 1 : 0, + 'drive.decUsage': isAdditional ? fileSizeKb : 0, }, file.userHost); } } diff --git a/packages/backend/src/services/chart/charts/network.ts b/packages/backend/src/services/chart/charts/network.ts deleted file mode 100644 index 73ea2f7e19..0000000000 --- a/packages/backend/src/services/chart/charts/network.ts +++ /dev/null @@ -1,49 +0,0 @@ -import autobind from 'autobind-decorator'; -import Chart, { DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; -import { name, schema } from './entities/network'; - -type NetworkLog = SchemaType; - -/** - * ネットワークに関するチャート - */ -// eslint-disable-next-line import/no-default-export -export default class NetworkChart extends Chart { - constructor() { - super(name, schema); - } - - @autobind - protected genNewLog(latest: NetworkLog): DeepPartial { - return {}; - } - - @autobind - protected aggregate(logs: NetworkLog[]): NetworkLog { - return { - incomingRequests: logs.reduce((a, b) => a + b.incomingRequests, 0), - outgoingRequests: logs.reduce((a, b) => a + b.outgoingRequests, 0), - totalTime: logs.reduce((a, b) => a + b.totalTime, 0), - incomingBytes: logs.reduce((a, b) => a + b.incomingBytes, 0), - outgoingBytes: logs.reduce((a, b) => a + b.outgoingBytes, 0), - }; - } - - @autobind - protected async fetchActual(): Promise> { - return {}; - } - - @autobind - public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number): Promise { - const inc: DeepPartial = { - incomingRequests: incomingRequests, - totalTime: time, - incomingBytes: incomingBytes, - outgoingBytes: outgoingBytes, - }; - - await this.inc(inc); - } -} diff --git a/packages/backend/src/services/chart/charts/notes.ts b/packages/backend/src/services/chart/charts/notes.ts index 86cda17225..4bbfa6704f 100644 --- a/packages/backend/src/services/chart/charts/notes.ts +++ b/packages/backend/src/services/chart/charts/notes.ts @@ -1,101 +1,43 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; +import Chart, { KVs } from '../core'; import { Notes } from '@/models/index'; import { Not, IsNull } from 'typeorm'; import { Note } from '@/models/entities/note'; import { name, schema } from './entities/notes'; -type NotesLog = SchemaType; - /** * ノートに関するチャート */ // eslint-disable-next-line import/no-default-export -export default class NotesChart extends Chart { +export default class NotesChart extends Chart { constructor() { super(name, schema); } @autobind - protected genNewLog(latest: NotesLog): DeepPartial { - return { - local: { - total: latest.local.total, - }, - remote: { - total: latest.remote.total, - }, - }; - } - - @autobind - protected aggregate(logs: NotesLog[]): NotesLog { - return { - local: { - total: logs[0].local.total, - inc: logs.reduce((a, b) => a + b.local.inc, 0), - dec: logs.reduce((a, b) => a + b.local.dec, 0), - diffs: { - reply: logs.reduce((a, b) => a + b.local.diffs.reply, 0), - renote: logs.reduce((a, b) => a + b.local.diffs.renote, 0), - normal: logs.reduce((a, b) => a + b.local.diffs.normal, 0), - }, - }, - remote: { - total: logs[0].remote.total, - inc: logs.reduce((a, b) => a + b.remote.inc, 0), - dec: logs.reduce((a, b) => a + b.remote.dec, 0), - diffs: { - reply: logs.reduce((a, b) => a + b.remote.diffs.reply, 0), - renote: logs.reduce((a, b) => a + b.remote.diffs.renote, 0), - normal: logs.reduce((a, b) => a + b.remote.diffs.normal, 0), - }, - }, - }; - } - - @autobind - protected async fetchActual(): Promise> { + protected async queryCurrentState(): Promise>> { const [localCount, remoteCount] = await Promise.all([ Notes.count({ userHost: null }), Notes.count({ userHost: Not(IsNull()) }), ]); return { - local: { - total: localCount, - }, - remote: { - total: remoteCount, - }, + 'local.total': localCount, + 'remote.total': remoteCount, }; } @autobind public async update(note: Note, isAdditional: boolean): Promise { - const update: Obj = { - diffs: {}, - }; - - update.total = isAdditional ? 1 : -1; - - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } - - if (note.replyId != null) { - update.diffs.reply = isAdditional ? 1 : -1; - } else if (note.renoteId != null) { - update.diffs.renote = isAdditional ? 1 : -1; - } else { - update.diffs.normal = isAdditional ? 1 : -1; - } - - await this.inc({ - [note.userHost === null ? 'local' : 'remote']: update, + const prefix = note.userHost === null ? 'local' : 'remote'; + + await this.commit({ + [`${prefix}.total`]: isAdditional ? 1 : -1, + [`${prefix}.inc`]: isAdditional ? 1 : 0, + [`${prefix}.dec`]: isAdditional ? 0 : 1, + [`${prefix}.diffs.normal`]: note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0, + [`${prefix}.diffs.renote`]: note.renoteId != null ? (isAdditional ? 1 : -1) : 0, + [`${prefix}.diffs.reply`]: note.replyId != null ? (isAdditional ? 1 : -1) : 0, }); } } diff --git a/packages/backend/src/services/chart/charts/per-user-drive.ts b/packages/backend/src/services/chart/charts/per-user-drive.ts index fff790367f..969ed018f9 100644 --- a/packages/backend/src/services/chart/charts/per-user-drive.ts +++ b/packages/backend/src/services/chart/charts/per-user-drive.ts @@ -1,68 +1,41 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; +import Chart, { KVs } from '../core'; import { DriveFiles } from '@/models/index'; import { DriveFile } from '@/models/entities/drive-file'; import { name, schema } from './entities/per-user-drive'; -type PerUserDriveLog = SchemaType; - /** * ユーザーごとのドライブに関するチャート */ // eslint-disable-next-line import/no-default-export -export default class PerUserDriveChart extends Chart { +export default class PerUserDriveChart extends Chart { constructor() { super(name, schema, true); } @autobind - protected genNewLog(latest: PerUserDriveLog): DeepPartial { - return { - totalCount: latest.totalCount, - totalSize: latest.totalSize, - }; - } - - @autobind - protected aggregate(logs: PerUserDriveLog[]): PerUserDriveLog { - return { - totalCount: logs[0].totalCount, - totalSize: logs[0].totalSize, - incCount: logs.reduce((a, b) => a + b.incCount, 0), - incSize: logs.reduce((a, b) => a + b.incSize, 0), - decCount: logs.reduce((a, b) => a + b.decCount, 0), - decSize: logs.reduce((a, b) => a + b.decSize, 0), - }; - } - - @autobind - protected async fetchActual(group: string): Promise> { + protected async queryCurrentState(group: string): Promise>> { const [count, size] = await Promise.all([ DriveFiles.count({ userId: group }), DriveFiles.calcDriveUsageOf(group), ]); return { - totalCount: count, - totalSize: size, + 'totalCount': count, + 'totalSize': size, }; } @autobind public async update(file: DriveFile, isAdditional: boolean): Promise { - const update: Obj = {}; - - update.totalCount = isAdditional ? 1 : -1; - update.totalSize = isAdditional ? file.size : -file.size; - if (isAdditional) { - update.incCount = 1; - update.incSize = file.size; - } else { - update.decCount = 1; - update.decSize = file.size; - } - - await this.inc(update, file.userId); + const fileSizeKb = file.size / 1000; + await this.commit({ + 'totalCount': isAdditional ? 1 : -1, + 'totalSize': isAdditional ? fileSizeKb : -fileSizeKb, + 'incCount': isAdditional ? 1 : 0, + 'incSize': isAdditional ? fileSizeKb : 0, + 'decCount': isAdditional ? 0 : 1, + 'decSize': isAdditional ? 0 : fileSizeKb, + }, file.userId); } } diff --git a/packages/backend/src/services/chart/charts/per-user-following.ts b/packages/backend/src/services/chart/charts/per-user-following.ts index d0a80abdaf..cdd0aad947 100644 --- a/packages/backend/src/services/chart/charts/per-user-following.ts +++ b/packages/backend/src/services/chart/charts/per-user-following.ts @@ -1,76 +1,21 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; +import Chart, { KVs } from '../core'; import { Followings, Users } from '@/models/index'; import { Not, IsNull } from 'typeorm'; import { User } from '@/models/entities/user'; import { name, schema } from './entities/per-user-following'; -type PerUserFollowingLog = SchemaType; - /** * ユーザーごとのフォローに関するチャート */ // eslint-disable-next-line import/no-default-export -export default class PerUserFollowingChart extends Chart { +export default class PerUserFollowingChart extends Chart { constructor() { super(name, schema, true); } @autobind - protected genNewLog(latest: PerUserFollowingLog): DeepPartial { - return { - local: { - followings: { - total: latest.local.followings.total, - }, - followers: { - total: latest.local.followers.total, - }, - }, - remote: { - followings: { - total: latest.remote.followings.total, - }, - followers: { - total: latest.remote.followers.total, - }, - }, - }; - } - - @autobind - protected aggregate(logs: PerUserFollowingLog[]): PerUserFollowingLog { - return { - local: { - followings: { - total: logs[0].local.followings.total, - inc: logs.reduce((a, b) => a + b.local.followings.inc, 0), - dec: logs.reduce((a, b) => a + b.local.followings.dec, 0), - }, - followers: { - total: logs[0].local.followers.total, - inc: logs.reduce((a, b) => a + b.local.followers.inc, 0), - dec: logs.reduce((a, b) => a + b.local.followers.dec, 0), - }, - }, - remote: { - followings: { - total: logs[0].remote.followings.total, - inc: logs.reduce((a, b) => a + b.remote.followings.inc, 0), - dec: logs.reduce((a, b) => a + b.remote.followings.dec, 0), - }, - followers: { - total: logs[0].remote.followers.total, - inc: logs.reduce((a, b) => a + b.remote.followers.inc, 0), - dec: logs.reduce((a, b) => a + b.remote.followers.dec, 0), - }, - }, - }; - } - - @autobind - protected async fetchActual(group: string): Promise> { + protected async queryCurrentState(group: string): Promise>> { const [ localFollowingsCount, localFollowersCount, @@ -84,42 +29,27 @@ export default class PerUserFollowingChart extends Chart { ]); return { - local: { - followings: { - total: localFollowingsCount, - }, - followers: { - total: localFollowersCount, - }, - }, - remote: { - followings: { - total: remoteFollowingsCount, - }, - followers: { - total: remoteFollowersCount, - }, - }, + 'local.followings.total': localFollowingsCount, + 'local.followers.total': localFollowersCount, + 'remote.followings.total': remoteFollowingsCount, + 'remote.followers.total': remoteFollowersCount, }; } @autobind public async update(follower: { id: User['id']; host: User['host']; }, followee: { id: User['id']; host: User['host']; }, isFollow: boolean): Promise { - const update: Obj = {}; - - update.total = isFollow ? 1 : -1; - - if (isFollow) { - update.inc = 1; - } else { - update.dec = 1; - } + const prefixFollower = Users.isLocalUser(follower) ? 'local' : 'remote'; + const prefixFollowee = Users.isLocalUser(followee) ? 'local' : 'remote'; - this.inc({ - [Users.isLocalUser(follower) ? 'local' : 'remote']: { followings: update }, + this.commit({ + [`${prefixFollower}.followings.total`]: isFollow ? 1 : -1, + [`${prefixFollower}.followings.inc`]: isFollow ? 1 : 0, + [`${prefixFollower}.followings.dec`]: isFollow ? 0 : 1, }, follower.id); - this.inc({ - [Users.isLocalUser(followee) ? 'local' : 'remote']: { followers: update }, + this.commit({ + [`${prefixFollowee}.followers.total`]: isFollow ? 1 : -1, + [`${prefixFollowee}.followers.inc`]: isFollow ? 1 : 0, + [`${prefixFollowee}.followers.dec`]: isFollow ? 0 : 1, }, followee.id); } } diff --git a/packages/backend/src/services/chart/charts/per-user-notes.ts b/packages/backend/src/services/chart/charts/per-user-notes.ts index d048c88885..ead353139b 100644 --- a/packages/backend/src/services/chart/charts/per-user-notes.ts +++ b/packages/backend/src/services/chart/charts/per-user-notes.ts @@ -1,45 +1,21 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; +import Chart, { KVs } from '../core'; import { User } from '@/models/entities/user'; -import { SchemaType } from '@/misc/schema'; import { Notes } from '@/models/index'; import { Note } from '@/models/entities/note'; import { name, schema } from './entities/per-user-notes'; -type PerUserNotesLog = SchemaType; - /** * ユーザーごとのノートに関するチャート */ // eslint-disable-next-line import/no-default-export -export default class PerUserNotesChart extends Chart { +export default class PerUserNotesChart extends Chart { constructor() { super(name, schema, true); } @autobind - protected genNewLog(latest: PerUserNotesLog): DeepPartial { - return { - total: latest.total, - }; - } - - @autobind - protected aggregate(logs: PerUserNotesLog[]): PerUserNotesLog { - return { - total: logs[0].total, - inc: logs.reduce((a, b) => a + b.inc, 0), - dec: logs.reduce((a, b) => a + b.dec, 0), - diffs: { - reply: logs.reduce((a, b) => a + b.diffs.reply, 0), - renote: logs.reduce((a, b) => a + b.diffs.renote, 0), - normal: logs.reduce((a, b) => a + b.diffs.normal, 0), - }, - }; - } - - @autobind - protected async fetchActual(group: string): Promise> { + protected async queryCurrentState(group: string): Promise>> { const [count] = await Promise.all([ Notes.count({ userId: group }), ]); @@ -51,26 +27,13 @@ export default class PerUserNotesChart extends Chart { @autobind public async update(user: { id: User['id'] }, note: Note, isAdditional: boolean): Promise { - const update: Obj = { - diffs: {}, - }; - - update.total = isAdditional ? 1 : -1; - - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } - - if (note.replyId != null) { - update.diffs.reply = isAdditional ? 1 : -1; - } else if (note.renoteId != null) { - update.diffs.renote = isAdditional ? 1 : -1; - } else { - update.diffs.normal = isAdditional ? 1 : -1; - } - - await this.inc(update, user.id); + await this.commit({ + 'total': isAdditional ? 1 : -1, + 'inc': isAdditional ? 1 : 0, + 'dec': isAdditional ? 0 : 1, + 'diffs.normal': note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0, + 'diffs.renote': note.renoteId != null ? (isAdditional ? 1 : -1) : 0, + 'diffs.reply': note.replyId != null ? (isAdditional ? 1 : -1) : 0, + }, user.id); } } diff --git a/packages/backend/src/services/chart/charts/per-user-reactions.ts b/packages/backend/src/services/chart/charts/per-user-reactions.ts index 2f5353340d..2ec347f40a 100644 --- a/packages/backend/src/services/chart/charts/per-user-reactions.ts +++ b/packages/backend/src/services/chart/charts/per-user-reactions.ts @@ -1,48 +1,29 @@ import autobind from 'autobind-decorator'; -import Chart, { DeepPartial } from '../core'; +import Chart, { KVs } from '../core'; import { User } from '@/models/entities/user'; import { Note } from '@/models/entities/note'; -import { SchemaType } from '@/misc/schema'; import { Users } from '@/models/index'; import { name, schema } from './entities/per-user-reactions'; -type PerUserReactionsLog = SchemaType; - /** * ユーザーごとのリアクションに関するチャート */ // eslint-disable-next-line import/no-default-export -export default class PerUserReactionsChart extends Chart { +export default class PerUserReactionsChart extends Chart { constructor() { super(name, schema, true); } @autobind - protected genNewLog(latest: PerUserReactionsLog): DeepPartial { - return {}; - } - - @autobind - protected aggregate(logs: PerUserReactionsLog[]): PerUserReactionsLog { - return { - local: { - count: logs.reduce((a, b) => a + b.local.count, 0), - }, - remote: { - count: logs.reduce((a, b) => a + b.remote.count, 0), - }, - }; - } - - @autobind - protected async fetchActual(group: string): Promise> { + protected async queryCurrentState(group: string): Promise>> { return {}; } @autobind public async update(user: { id: User['id'], host: User['host'] }, note: Note): Promise { - this.inc({ - [Users.isLocalUser(user) ? 'local' : 'remote']: { count: 1 }, + const prefix = Users.isLocalUser(user) ? 'local' : 'remote'; + this.commit({ + [`${prefix}.count`]: 1, }, note.userId); } } diff --git a/packages/backend/src/services/chart/charts/test-grouped.ts b/packages/backend/src/services/chart/charts/test-grouped.ts index c851d2df01..5f0b1aafdc 100644 --- a/packages/backend/src/services/chart/charts/test-grouped.ts +++ b/packages/backend/src/services/chart/charts/test-grouped.ts @@ -1,15 +1,12 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; +import Chart, { KVs } from '../core'; import { name, schema } from './entities/test-grouped'; -type TestGroupedLog = SchemaType; - /** * For testing */ // eslint-disable-next-line import/no-default-export -export default class TestGroupedChart extends Chart { +export default class TestGroupedChart extends Chart { private total = {} as Record; constructor() { @@ -17,31 +14,9 @@ export default class TestGroupedChart extends Chart { } @autobind - protected genNewLog(latest: TestGroupedLog): DeepPartial { - return { - foo: { - total: latest.foo.total, - }, - }; - } - - @autobind - protected aggregate(logs: TestGroupedLog[]): TestGroupedLog { + protected async queryCurrentState(group: string): Promise>> { return { - foo: { - total: logs[0].foo.total, - inc: logs.reduce((a, b) => a + b.foo.inc, 0), - dec: logs.reduce((a, b) => a + b.foo.dec, 0), - }, - }; - } - - @autobind - protected async fetchActual(group: string): Promise> { - return { - foo: { - total: this.total[group], - }, + 'foo.total': this.total[group], }; } @@ -49,14 +24,11 @@ export default class TestGroupedChart extends Chart { public async increment(group: string): Promise { if (this.total[group] == null) this.total[group] = 0; - const update: Obj = {}; - - update.total = 1; - update.inc = 1; this.total[group]++; - await this.inc({ - foo: update, + await this.commit({ + 'foo.total': 1, + 'foo.inc': 1, }, group); } } diff --git a/packages/backend/src/services/chart/charts/test-unique.ts b/packages/backend/src/services/chart/charts/test-unique.ts index 3564f675ad..e67036acef 100644 --- a/packages/backend/src/services/chart/charts/test-unique.ts +++ b/packages/backend/src/services/chart/charts/test-unique.ts @@ -1,39 +1,24 @@ import autobind from 'autobind-decorator'; -import Chart, { DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; +import Chart, { KVs } from '../core'; import { name, schema } from './entities/test-unique'; -type TestUniqueLog = SchemaType; - /** * For testing */ // eslint-disable-next-line import/no-default-export -export default class TestUniqueChart extends Chart { +export default class TestUniqueChart extends Chart { constructor() { super(name, schema); } @autobind - protected genNewLog(latest: TestUniqueLog): DeepPartial { - return {}; - } - - @autobind - protected aggregate(logs: TestUniqueLog[]): TestUniqueLog { - return { - foo: logs.reduce((a, b) => a.concat(b.foo), [] as TestUniqueLog['foo']), - }; - } - - @autobind - protected async fetchActual(): Promise> { + protected async queryCurrentState(): Promise>> { return {}; } @autobind public async uniqueIncrement(key: string): Promise { - await this.inc({ + await this.commit({ foo: [key], }); } diff --git a/packages/backend/src/services/chart/charts/test.ts b/packages/backend/src/services/chart/charts/test.ts index 06add7ede9..878acd51be 100644 --- a/packages/backend/src/services/chart/charts/test.ts +++ b/packages/backend/src/services/chart/charts/test.ts @@ -1,15 +1,12 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; +import Chart, { KVs } from '../core'; import { name, schema } from './entities/test'; -type TestLog = SchemaType; - /** * For testing */ // eslint-disable-next-line import/no-default-export -export default class TestChart extends Chart { +export default class TestChart extends Chart { public total = 0; // publicにするのはテストのため constructor() { @@ -17,57 +14,29 @@ export default class TestChart extends Chart { } @autobind - protected genNewLog(latest: TestLog): DeepPartial { + protected async queryCurrentState(): Promise>> { return { - foo: { - total: latest.foo.total, - }, - }; - } - - @autobind - protected aggregate(logs: TestLog[]): TestLog { - return { - foo: { - total: logs[0].foo.total, - inc: logs.reduce((a, b) => a + b.foo.inc, 0), - dec: logs.reduce((a, b) => a + b.foo.dec, 0), - }, - }; - } - - @autobind - protected async fetchActual(): Promise> { - return { - foo: { - total: this.total, - }, + 'foo.total': this.total, }; } @autobind public async increment(): Promise { - const update: Obj = {}; - - update.total = 1; - update.inc = 1; this.total++; - await this.inc({ - foo: update, + await this.commit({ + 'foo.total': 1, + 'foo.inc': 1, }); } @autobind public async decrement(): Promise { - const update: Obj = {}; - - update.total = -1; - update.dec = 1; this.total--; - await this.inc({ - foo: update, + await this.commit({ + 'foo.total': -1, + 'foo.dec': 1, }); } } diff --git a/packages/backend/src/services/chart/charts/users.ts b/packages/backend/src/services/chart/charts/users.ts index c36c6cd979..50fca3a8d6 100644 --- a/packages/backend/src/services/chart/charts/users.ts +++ b/packages/backend/src/services/chart/charts/users.ts @@ -1,80 +1,40 @@ import autobind from 'autobind-decorator'; -import Chart, { Obj, DeepPartial } from '../core'; -import { SchemaType } from '@/misc/schema'; +import Chart, { KVs } from '../core'; import { Users } from '@/models/index'; import { Not, IsNull } from 'typeorm'; import { User } from '@/models/entities/user'; import { name, schema } from './entities/users'; -type UsersLog = SchemaType; - /** * ユーザー数に関するチャート */ // eslint-disable-next-line import/no-default-export -export default class UsersChart extends Chart { +export default class UsersChart extends Chart { constructor() { super(name, schema); } @autobind - protected genNewLog(latest: UsersLog): DeepPartial { - return { - local: { - total: latest.local.total, - }, - remote: { - total: latest.remote.total, - }, - }; - } - - @autobind - protected aggregate(logs: UsersLog[]): UsersLog { - return { - local: { - total: logs[0].local.total, - inc: logs.reduce((a, b) => a + b.local.inc, 0), - dec: logs.reduce((a, b) => a + b.local.dec, 0), - }, - remote: { - total: logs[0].remote.total, - inc: logs.reduce((a, b) => a + b.remote.inc, 0), - dec: logs.reduce((a, b) => a + b.remote.dec, 0), - }, - }; - } - - @autobind - protected async fetchActual(): Promise> { + protected async queryCurrentState(): Promise>> { const [localCount, remoteCount] = await Promise.all([ Users.count({ host: null }), Users.count({ host: Not(IsNull()) }), ]); return { - local: { - total: localCount, - }, - remote: { - total: remoteCount, - }, + 'local.total': localCount, + 'remote.total': remoteCount, }; } @autobind public async update(user: { id: User['id'], host: User['host'] }, isAdditional: boolean): Promise { - const update: Obj = {}; - - update.total = isAdditional ? 1 : -1; - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } + const prefix = Users.isLocalUser(user) ? 'local' : 'remote'; - await this.inc({ - [Users.isLocalUser(user) ? 'local' : 'remote']: update, + await this.commit({ + [`${prefix}.total`]: isAdditional ? 1 : -1, + [`${prefix}.inc`]: isAdditional ? 1 : 0, + [`${prefix}.dec`]: isAdditional ? 0 : 1, }); } } diff --git a/packages/backend/src/services/chart/core.ts b/packages/backend/src/services/chart/core.ts index e406449f4f..0bc3d1a4c0 100644 --- a/packages/backend/src/services/chart/core.ts +++ b/packages/backend/src/services/chart/core.ts @@ -7,24 +7,19 @@ import * as nestedProperty from 'nested-property'; import autobind from 'autobind-decorator'; import Logger from '../logger'; -import { Schema } from '@/misc/schema'; import { EntitySchema, getRepository, Repository, LessThan, Between } from 'typeorm'; import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from '@/prelude/time'; import { getChartInsertLock } from '@/misc/app-lock'; const logger = new Logger('chart', 'white', process.env.NODE_ENV !== 'test'); -export type Obj = { [key: string]: any }; +const columnPrefix = '___' as const; +const uniqueTempColumnPrefix = 'unique_temp___' as const; +const columnDot = '_' as const; -export type DeepPartial = { - [P in keyof T]?: DeepPartial; -}; - -type ArrayValue = { - [P in keyof T]: T[P] extends number ? T[P][] : ArrayValue; -}; +type KeyToColumnName = T extends `${infer R1}.${infer R2}` ? `${R1}${typeof columnDot}${KeyToColumnName}` : T; -type Log = { +type RawRecord = { id: number; /** @@ -36,6 +31,10 @@ type Log = { * 集計日時のUnixタイムスタンプ(秒) */ date: number; +} & { + [K in keyof S as `${typeof uniqueTempColumnPrefix}${KeyToColumnName}`]: S[K]['uniqueIncrement'] extends true ? string[] : never; +} & { + [K in keyof S as `${typeof columnPrefix}${KeyToColumnName}`]: number; }; const camelToSnake = (str: string): string => { @@ -44,123 +43,72 @@ const camelToSnake = (str: string): string => { const removeDuplicates = (array: any[]) => Array.from(new Set(array)); +type Schema = Record; + +type Commit = { + [K in keyof S]?: S[K]['uniqueIncrement'] extends true ? string[] : number; +}; + +export type KVs = { + [K in keyof S]: number; +}; + +type ChartResult = { + [P in keyof T]: number[]; +}; + /** * 様々なチャートの管理を司るクラス */ // eslint-disable-next-line import/no-default-export -export default abstract class Chart> { - private static readonly columnPrefix = '___'; - private static readonly columnDot = '_'; +export default abstract class Chart { + public schema: T; private name: string; private buffer: { - diff: DeepPartial; + diff: Commit; group: string | null; }[] = []; - public schema: Schema; - protected repositoryForHour: Repository; - protected repositoryForDay: Repository; - - protected abstract genNewLog(latest: T): DeepPartial; - - /** - * @param logs 日時が新しい方が先頭 - */ - protected abstract aggregate(logs: T[]): T; + protected repositoryForHour: Repository>; + protected repositoryForDay: Repository>; - protected abstract fetchActual(group: string | null): Promise>; + protected abstract queryCurrentState(group: string | null): Promise>>; @autobind - private static convertSchemaToFlatColumnDefinitions(schema: Schema) { - const columns = {} as Record; - const flatColumns = (x: Obj, path?: string) => { - for (const [k, v] of Object.entries(x)) { - const p = path ? `${path}${this.columnDot}${k}` : k; - if (v.type === 'object') { - flatColumns(v.properties, p); - } else if (v.type === 'number') { - columns[this.columnPrefix + p] = { - type: 'bigint', - }; - } else if (v.type === 'array' && v.items.type === 'string') { - columns[this.columnPrefix + p] = { - type: 'varchar', - array: true, - }; - } + private static convertSchemaToColumnDefinitions(schema: Schema): Record { + const columns = {} as Record; + for (const [k, v] of Object.entries(schema)) { + const name = k.replaceAll('.', columnDot); + const type = v.range === 'big' ? 'bigint' : v.range === 'small' ? 'smallint' : 'integer'; + if (v.uniqueIncrement) { + columns[uniqueTempColumnPrefix + name] = { + type: 'varchar', + array: true, + default: '{}', + }; + columns[columnPrefix + name] = { + type, + default: 0, + }; + } else { + columns[columnPrefix + name] = { + type, + default: 0, + }; } - }; - flatColumns(schema.properties!); - return columns; - } - - @autobind - private static convertFlattenColumnsToObject(x: Record): Record { - const obj = {} as Record; - for (const k of Object.keys(x).filter(k => k.startsWith(Chart.columnPrefix))) { - // now k is ___x_y_z - const path = k.substr(Chart.columnPrefix.length).split(Chart.columnDot).join('.'); - nestedProperty.set(obj, path, x[k]); } - return obj; - } - - @autobind - private static convertObjectToFlattenColumns(x: Record) { - const columns = {} as Record; - const flatten = (x: Obj, path?: string) => { - for (const [k, v] of Object.entries(x)) { - const p = path ? `${path}${this.columnDot}${k}` : k; - if (typeof v === 'object' && !Array.isArray(v)) { - flatten(v, p); - } else { - columns[this.columnPrefix + p] = v; - } - } - }; - flatten(x); return columns; } @autobind - private static countUniqueFields(x: Record) { - const exec = (x: Obj) => { - const res = {} as Record; - for (const [k, v] of Object.entries(x)) { - if (typeof v === 'object' && !Array.isArray(v)) { - res[k] = exec(v); - } else if (Array.isArray(v)) { - res[k] = Array.from(new Set(v)).length; - } else { - res[k] = v; - } - } - return res; - }; - return exec(x); - } - - @autobind - private static convertQuery(diff: Record) { - const query: Record string> = {}; - - for (const [k, v] of Object.entries(diff)) { - if (typeof v === 'number') { - if (v > 0) query[k] = () => `"${k}" + ${v}`; - if (v < 0) query[k] = () => `"${k}" - ${Math.abs(v)}`; - } else if (Array.isArray(v)) { - // TODO: item が文字列以外の場合も対応 - // TODO: item をSQLエスケープ - const items = v.map(item => `"${item}"`).join(','); - query[k] = () => `array_cat("${k}", '{${items}}'::varchar[])`; - } - } - - return query; - } - - @autobind - private static dateToTimestamp(x: Date): Log['date'] { + private static dateToTimestamp(x: Date): number { return Math.floor(x.getTime() / 1000); } @@ -207,7 +155,7 @@ export default abstract class Chart> { length: 128, }, } : {}), - ...Chart.convertSchemaToFlatColumnDefinitions(schema), + ...Chart.convertSchemaToColumnDefinitions(schema), }, indices: [{ columns: grouped ? ['date', 'group'] : ['date'], @@ -233,37 +181,39 @@ export default abstract class Chart> { }; } - constructor(name: string, schema: Schema, grouped = false) { + constructor(name: string, schema: T, grouped = false) { this.name = name; this.schema = schema; const { hour, day } = Chart.schemaToEntity(name, schema, grouped); - this.repositoryForHour = getRepository(hour); - this.repositoryForDay = getRepository(day); + this.repositoryForHour = getRepository>(hour); + this.repositoryForDay = getRepository>(day); } @autobind - private getNewLog(latest: T | null): T { - const log = latest ? this.genNewLog(latest) : {}; - const flatColumns = (x: Obj, path?: string) => { - for (const [k, v] of Object.entries(x)) { - const p = path ? `${path}.${k}` : k; - if (v.type === 'object') { - flatColumns(v.properties, p); - } else { - if (nestedProperty.get(log, p) == null) { - const emptyValue = v.type === 'number' ? 0 : []; - nestedProperty.set(log, p, emptyValue); - } - } + private convertRawRecord(x: RawRecord): KVs { + const kvs = {} as KVs; + for (const k of Object.keys(x).filter(k => k.startsWith(columnPrefix))) { + kvs[k.substr(columnPrefix.length).split(columnDot).join('.')] = x[k]; + } + return kvs; + } + + @autobind + private getNewLog(latest: KVs | null): KVs { + const log = {} as Record; + for (const [k, v] of Object.entries(this.schema)) { + if (v.accumulate && latest) { + log[k] = latest[k]; + } else { + log[k] = 0; } - }; - flatColumns(this.schema.properties!); - return log as T; + } + return log as KVs; } @autobind - private getLatestLog(group: string | null, span: 'hour' | 'day'): Promise { + private getLatestLog(group: string | null, span: 'hour' | 'day'): Promise | null> { const repository = span === 'hour' ? this.repositoryForHour : span === 'day' ? this.repositoryForDay : @@ -282,7 +232,7 @@ export default abstract class Chart> { * 現在(=今のHour or Day)のログをデータベースから探して、あればそれを返し、なければ作成して返します。 */ @autobind - private async claimCurrentLog(group: string | null, span: 'hour' | 'day'): Promise { + private async claimCurrentLog(group: string | null, span: 'hour' | 'day'): Promise> { const [y, m, d, h] = Chart.getCurrentDate(); const current = dateUTC( @@ -306,8 +256,8 @@ export default abstract class Chart> { return currentLog; } - let log: Log; - let data: T; + let log: RawRecord; + let data: KVs; // 集計期間が変わってから、初めてのチャート更新なら // 最も最近のログを持ってくる @@ -318,10 +268,8 @@ export default abstract class Chart> { const latest = await this.getLatestLog(group, span); if (latest != null) { - const obj = Chart.convertFlattenColumnsToObject(latest) as T; - // 空ログデータを作成 - data = this.getNewLog(obj); + data = this.getNewLog(this.convertRawRecord(latest)); } else { // ログが存在しなかったら // (Misskeyインスタンスを建てて初めてのチャート更新時など) @@ -346,11 +294,17 @@ export default abstract class Chart> { // ログがあればそれを返して終了 if (currentLog != null) return currentLog; + const columns = {} as Record; + for (const [k, v] of Object.entries(data)) { + const name = k.replaceAll('.', columnDot); + columns[columnPrefix + name] = v; + } + // 新規ログ挿入 log = await repository.insert({ date: date, ...(group ? { group: group } : {}), - ...Chart.convertObjectToFlattenColumns(data), + ...columns, }).then(x => repository.findOneOrFail(x.identifiers[0])); logger.info(`${this.name + (group ? `:${group}` : '')}(${span}): New commit created`); @@ -362,7 +316,10 @@ export default abstract class Chart> { } @autobind - protected commit(diff: DeepPartial, group: string | null = null): void { + protected commit(diff: Commit, group: string | null = null): void { + for (const [k, v] of Object.entries(diff)) { + if (v == null || v === 0 || (Array.isArray(v) && v.length === 0)) delete diff[k]; + } this.buffer.push({ diff, group, }); @@ -381,13 +338,11 @@ export default abstract class Chart> { // そのログは本来は 01:00~ のログとしてDBに保存されて欲しいのに、02:00~ のログ扱いになってしまう。 // これを回避するための実装は複雑になりそうなため、一旦保留。 - const update = async (logHour: Log, logDay: Log): Promise => { + const update = async (logHour: RawRecord, logDay: RawRecord): Promise => { const finalDiffs = {} as Record; for (const diff of this.buffer.filter(q => q.group == null || (q.group === logHour.group)).map(q => q.diff)) { - const columns = Chart.convertObjectToFlattenColumns(diff); - - for (const [k, v] of Object.entries(columns)) { + for (const [k, v] of Object.entries(diff)) { if (finalDiffs[k] == null) { finalDiffs[k] = v; } else { @@ -400,18 +355,45 @@ export default abstract class Chart> { } } - const query = Chart.convertQuery(finalDiffs); + const queryForHour: Record string)> = {}; + const queryForDay: Record string)> = {}; + for (const [k, v] of Object.entries(finalDiffs)) { + if (typeof v === 'number') { + const name = columnPrefix + k.replaceAll('.', columnDot); + if (v > 0) queryForHour[name] = () => `"${name}" + ${v}`; + if (v < 0) queryForHour[name] = () => `"${name}" - ${Math.abs(v)}`; + if (v > 0) queryForDay[name] = () => `"${name}" + ${v}`; + if (v < 0) queryForDay[name] = () => `"${name}" - ${Math.abs(v)}`; + } else if (Array.isArray(v) && v.length > 0) { // ユニークインクリメント + const name = uniqueTempColumnPrefix + k.replaceAll('.', columnDot); + // TODO: item が文字列以外の場合も対応 + // TODO: item をSQLエスケープ + // TODO: 値が重複しないようにしたい + const items = v.map(item => `"${item}"`).join(','); + queryForHour[name] = () => `array_cat("${name}", '{${items}}'::varchar[])`; + queryForDay[name] = () => `array_cat("${name}", '{${items}}'::varchar[])`; + } + } + + for (const [k, v] of Object.entries(this.schema)) { + const name = columnPrefix + k.replaceAll('.', columnDot); + if (v.uniqueIncrement) { + const tempColumnName = uniqueTempColumnPrefix + k.replaceAll('.', columnDot); + queryForHour[name] = new Set([...finalDiffs[k], ...logHour[tempColumnName]]).size; + queryForDay[name] = new Set([...finalDiffs[k], ...logDay[tempColumnName]]).size; + } + } // ログ更新 await Promise.all([ this.repositoryForHour.createQueryBuilder() .update() - .set(query) + .set(queryForHour) .where('id = :id', { id: logHour.id }) .execute(), this.repositoryForDay.createQueryBuilder() .update() - .set(query) + .set(queryForDay) .where('id = :id', { id: logDay.id }) .execute(), ]); @@ -435,18 +417,24 @@ export default abstract class Chart> { @autobind public async resync(group: string | null = null): Promise { - const data = await this.fetchActual(group); + const data = await this.queryCurrentState(group); + + const columns = {} as Record; + for (const [k, v] of Object.entries(data)) { + const name = k.replaceAll('.', columnDot); + columns[columnPrefix + name] = v; + } - const update = async (logHour: Log, logDay: Log): Promise => { + const update = async (logHour: RawRecord, logDay: RawRecord): Promise => { await Promise.all([ this.repositoryForHour.createQueryBuilder() .update() - .set(Chart.convertObjectToFlattenColumns(data)) + .set(columns as any) .where('id = :id', { id: logHour.id }) .execute(), this.repositoryForDay.createQueryBuilder() .update() - .set(Chart.convertObjectToFlattenColumns(data)) + .set(columns as any) .where('id = :id', { id: logDay.id }) .execute(), ]); @@ -460,12 +448,39 @@ export default abstract class Chart> { } @autobind - protected async inc(inc: DeepPartial, group: string | null = null): Promise { - await this.commit(inc, group); + public async clean(): Promise { + const current = dateUTC(Chart.getCurrentDate()); + + // 一日以上前かつ三日以内 + const gt = Chart.dateToTimestamp(current) - (1000 * 60 * 60 * 24 * 3); + const lt = Chart.dateToTimestamp(current) - (1000 * 60 * 60 * 24); + + const columns = {} as Record; + for (const [k, v] of Object.entries(this.schema)) { + if (v.uniqueIncrement) { + const name = k.replaceAll('.', columnDot); + columns[uniqueTempColumnPrefix + name] = []; + } + } + + await Promise.all([ + this.repositoryForHour.createQueryBuilder() + .update() + .set(columns as any) + .where('date > :gt', { gt }) + .andWhere('date < :lt', { lt }) + .execute(), + this.repositoryForDay.createQueryBuilder() + .update() + .set(columns as any) + .where('date > :gt', { gt }) + .andWhere('date < :lt', { lt }) + .execute(), + ]); } @autobind - public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise> { + public async getChartRaw(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise> { const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate(); const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [] as never; @@ -526,7 +541,7 @@ export default abstract class Chart> { } } - const chart: T[] = []; + const chart: KVs[] = []; for (let i = (amount - 1); i >= 0; i--) { const current = @@ -537,17 +552,16 @@ export default abstract class Chart> { const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current)); if (log) { - const data = Chart.convertFlattenColumnsToObject(log); - chart.unshift(Chart.countUniqueFields(data) as T); + chart.unshift(this.convertRawRecord(log)); } else { // 隙間埋め const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current)); - const data = latest ? Chart.convertFlattenColumnsToObject(latest) as T : null; - chart.unshift(Chart.countUniqueFields(this.getNewLog(data)) as T); + const data = latest ? this.convertRawRecord(latest) : null; + chart.unshift(this.getNewLog(data)); } } - const res = {} as Record; + const res = {} as ChartResult; /** * [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }] @@ -555,36 +569,26 @@ export default abstract class Chart> { * { foo: [1, 2, 3], bar: [5, 6, 7] } * にする */ - const compact = (x: Obj, path?: string): void => { - for (const [k, v] of Object.entries(x)) { - const p = path ? `${path}.${k}` : k; - if (typeof v === 'object' && !Array.isArray(v)) { - compact(v, p); + for (const record of chart) { + for (const [k, v] of Object.entries(record)) { + if (res[k]) { + res[k].push(v); } else { - const values = chart.map(s => nestedProperty.get(s, p)); - nestedProperty.set(res, p, values); + res[k] = [v]; } } - }; - - compact(chart[0]); + } - return res as ArrayValue; + return res; } -} -export function convertLog(logSchema: Schema): Schema { - const v: Schema = JSON.parse(JSON.stringify(logSchema)); // copy - if (v.type === 'number') { - v.type = 'array'; - v.items = { - type: 'number' as const, - optional: false as const, nullable: false as const, - }; - } else if (v.type === 'object') { - for (const k of Object.keys(v.properties!)) { - v.properties![k] = convertLog(v.properties![k]); + @autobind + public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise> { + const result = await this.getChartRaw(span, amount, cursor, group); + const object = {}; + for (const [k, v] of Object.entries(result)) { + nestedProperty.set(object, k, v); } + return object; } - return v; } diff --git a/packages/backend/src/services/chart/entities.ts b/packages/backend/src/services/chart/entities.ts index dedbd47080..569b328557 100644 --- a/packages/backend/src/services/chart/entities.ts +++ b/packages/backend/src/services/chart/entities.ts @@ -1,7 +1,6 @@ import { entity as FederationChart } from './charts/entities/federation'; import { entity as NotesChart } from './charts/entities/notes'; import { entity as UsersChart } from './charts/entities/users'; -import { entity as NetworkChart } from './charts/entities/network'; import { entity as ActiveUsersChart } from './charts/entities/active-users'; import { entity as InstanceChart } from './charts/entities/instance'; import { entity as PerUserNotesChart } from './charts/entities/per-user-notes'; @@ -10,12 +9,12 @@ import { entity as PerUserReactionsChart } from './charts/entities/per-user-reac import { entity as HashtagChart } from './charts/entities/hashtag'; import { entity as PerUserFollowingChart } from './charts/entities/per-user-following'; import { entity as PerUserDriveChart } from './charts/entities/per-user-drive'; +import { entity as ApRequestChart } from './charts/entities/ap-request'; export const entities = [ FederationChart.hour, FederationChart.day, NotesChart.hour, NotesChart.day, UsersChart.hour, UsersChart.day, - NetworkChart.hour, NetworkChart.day, ActiveUsersChart.hour, ActiveUsersChart.day, InstanceChart.hour, InstanceChart.day, PerUserNotesChart.hour, PerUserNotesChart.day, @@ -24,4 +23,5 @@ export const entities = [ HashtagChart.hour, HashtagChart.day, PerUserFollowingChart.hour, PerUserFollowingChart.day, PerUserDriveChart.hour, PerUserDriveChart.day, + ApRequestChart.hour, ApRequestChart.day, ]; diff --git a/packages/backend/src/services/chart/index.ts b/packages/backend/src/services/chart/index.ts index 0b9887b36f..1c0f7aadc1 100644 --- a/packages/backend/src/services/chart/index.ts +++ b/packages/backend/src/services/chart/index.ts @@ -3,7 +3,6 @@ import { beforeShutdown } from '@/misc/before-shutdown'; import FederationChart from './charts/federation'; import NotesChart from './charts/notes'; import UsersChart from './charts/users'; -import NetworkChart from './charts/network'; import ActiveUsersChart from './charts/active-users'; import InstanceChart from './charts/instance'; import PerUserNotesChart from './charts/per-user-notes'; @@ -12,11 +11,11 @@ import PerUserReactionsChart from './charts/per-user-reactions'; import HashtagChart from './charts/hashtag'; import PerUserFollowingChart from './charts/per-user-following'; import PerUserDriveChart from './charts/per-user-drive'; +import ApRequestChart from './charts/ap-request'; export const federationChart = new FederationChart(); export const notesChart = new NotesChart(); export const usersChart = new UsersChart(); -export const networkChart = new NetworkChart(); export const activeUsersChart = new ActiveUsersChart(); export const instanceChart = new InstanceChart(); export const perUserNotesChart = new PerUserNotesChart(); @@ -25,12 +24,12 @@ export const perUserReactionsChart = new PerUserReactionsChart(); export const hashtagChart = new HashtagChart(); export const perUserFollowingChart = new PerUserFollowingChart(); export const perUserDriveChart = new PerUserDriveChart(); +export const apRequestChart = new ApRequestChart(); const charts = [ federationChart, notesChart, usersChart, - networkChart, activeUsersChart, instanceChart, perUserNotesChart, @@ -39,6 +38,7 @@ const charts = [ hashtagChart, perUserFollowingChart, perUserDriveChart, + apRequestChart, ]; // 20分おきにメモリ情報をDBに書き込み -- cgit v1.2.3-freya