summaryrefslogtreecommitdiff
path: root/packages/backend/src
diff options
context:
space:
mode:
authorsyuilo <Syuilotan@yahoo.co.jp>2022-02-06 00:13:52 +0900
committerGitHub <noreply@github.com>2022-02-06 00:13:52 +0900
commitc1b264e4e9686804e1b8ea17ba39753c41bd205b (patch)
tree041e794b683cf986f98fcc04a3d5a50a8dcb7554 /packages/backend/src
parentenhance(client): improve chart rendering (diff)
downloadsharkey-c1b264e4e9686804e1b8ea17ba39753c41bd205b.tar.gz
sharkey-c1b264e4e9686804e1b8ea17ba39753c41bd205b.tar.bz2
sharkey-c1b264e4e9686804e1b8ea17ba39753c41bd205b.zip
Improve chart engine (#8253)
* wip * wip * wip * wip * wip * wip * wip * Update core.ts * wip * wip * #7361 * delete network chart * federationChart強化 apRequestChart追加 * tweak
Diffstat (limited to 'packages/backend/src')
-rw-r--r--packages/backend/src/queue/index.ts5
-rw-r--r--packages/backend/src/queue/processors/deliver.ts5
-rw-r--r--packages/backend/src/queue/processors/inbox.ts4
-rw-r--r--packages/backend/src/queue/processors/system/clean-charts.ts28
-rw-r--r--packages/backend/src/queue/processors/system/index.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/active-users.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/ap-request.ts (renamed from packages/backend/src/server/api/endpoints/charts/network.ts)6
-rw-r--r--packages/backend/src/server/api/endpoints/charts/drive.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/federation.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/hashtag.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/instance.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/notes.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/user/drive.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/user/following.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/user/notes.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/user/reactions.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/charts/users.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/stats.ts4
-rw-r--r--packages/backend/src/server/index.ts26
-rw-r--r--packages/backend/src/services/chart/charts/active-users.ts35
-rw-r--r--packages/backend/src/services/chart/charts/ap-request.ts39
-rw-r--r--packages/backend/src/services/chart/charts/drive.ts88
-rw-r--r--packages/backend/src/services/chart/charts/entities/active-users.ts32
-rw-r--r--packages/backend/src/services/chart/charts/entities/ap-request.ts11
-rw-r--r--packages/backend/src/services/chart/charts/entities/drive.ts74
-rw-r--r--packages/backend/src/services/chart/charts/entities/federation.ts29
-rw-r--r--packages/backend/src/services/chart/charts/entities/hashtag.ts32
-rw-r--r--packages/backend/src/services/chart/charts/entities/instance.ts175
-rw-r--r--packages/backend/src/services/chart/charts/entities/network.ts32
-rw-r--r--packages/backend/src/services/chart/charts/entities/notes.ts66
-rw-r--r--packages/backend/src/services/chart/charts/entities/per-user-drive.ts59
-rw-r--r--packages/backend/src/services/chart/charts/entities/per-user-following.ts96
-rw-r--r--packages/backend/src/services/chart/charts/entities/per-user-notes.ts47
-rw-r--r--packages/backend/src/services/chart/charts/entities/per-user-reactions.ts28
-rw-r--r--packages/backend/src/services/chart/charts/entities/test-grouped.ts29
-rw-r--r--packages/backend/src/services/chart/charts/entities/test-unique.ts15
-rw-r--r--packages/backend/src/services/chart/charts/entities/test.ts29
-rw-r--r--packages/backend/src/services/chart/charts/entities/users.ts48
-rw-r--r--packages/backend/src/services/chart/charts/federation.ts58
-rw-r--r--packages/backend/src/services/chart/charts/hashtag.ts35
-rw-r--r--packages/backend/src/services/chart/charts/instance.ts182
-rw-r--r--packages/backend/src/services/chart/charts/network.ts49
-rw-r--r--packages/backend/src/services/chart/charts/notes.ts84
-rw-r--r--packages/backend/src/services/chart/charts/per-user-drive.ts55
-rw-r--r--packages/backend/src/services/chart/charts/per-user-following.ts104
-rw-r--r--packages/backend/src/services/chart/charts/per-user-notes.ts59
-rw-r--r--packages/backend/src/services/chart/charts/per-user-reactions.ts31
-rw-r--r--packages/backend/src/services/chart/charts/test-grouped.ts42
-rw-r--r--packages/backend/src/services/chart/charts/test-unique.ts23
-rw-r--r--packages/backend/src/services/chart/charts/test.ts51
-rw-r--r--packages/backend/src/services/chart/charts/users.ts60
-rw-r--r--packages/backend/src/services/chart/core.ts368
-rw-r--r--packages/backend/src/services/chart/entities.ts4
-rw-r--r--packages/backend/src/services/chart/index.ts6
54 files changed, 583 insertions, 1694 deletions
diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts
index f9994c3b59..795e520feb 100644
--- a/packages/backend/src/queue/index.ts
+++ b/packages/backend/src/queue/index.ts
@@ -261,6 +261,11 @@ export default function() {
}, {
repeat: { cron: '0 0 * * *' },
});
+
+ systemQueue.add('cleanCharts', {
+ }, {
+ repeat: { cron: '0 0 * * *' },
+ });
}
export function destroy() {
diff --git a/packages/backend/src/queue/processors/deliver.ts b/packages/backend/src/queue/processors/deliver.ts
index 46aeb8cb7e..e29f991307 100644
--- a/packages/backend/src/queue/processors/deliver.ts
+++ b/packages/backend/src/queue/processors/deliver.ts
@@ -4,7 +4,7 @@ import request from '@/remote/activitypub/request';
import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc';
import Logger from '@/services/logger';
import { Instances } from '@/models/index';
-import { instanceChart } from '@/services/chart/index';
+import { apRequestChart, federationChart, instanceChart } from '@/services/chart/index';
import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata';
import { fetchMeta } from '@/misc/fetch-meta';
import { toPuny } from '@/misc/convert-host';
@@ -61,6 +61,8 @@ export default async (job: Bull.Job<DeliverJobData>) => {
fetchInstanceMetadata(i);
instanceChart.requestSent(i.host, true);
+ apRequestChart.deliverSucc();
+ federationChart.deliverd(i.host);
});
return 'Success';
@@ -74,6 +76,7 @@ export default async (job: Bull.Job<DeliverJobData>) => {
});
instanceChart.requestSent(i.host, false);
+ apRequestChart.deliverFail();
});
if (res instanceof StatusError) {
diff --git a/packages/backend/src/queue/processors/inbox.ts b/packages/backend/src/queue/processors/inbox.ts
index 82a0de9663..c189256c33 100644
--- a/packages/backend/src/queue/processors/inbox.ts
+++ b/packages/backend/src/queue/processors/inbox.ts
@@ -5,7 +5,7 @@ import perform from '@/remote/activitypub/perform';
import Logger from '@/services/logger';
import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc';
import { Instances } from '@/models/index';
-import { instanceChart } from '@/services/chart/index';
+import { apRequestChart, federationChart, instanceChart } from '@/services/chart/index';
import { fetchMeta } from '@/misc/fetch-meta';
import { toPuny, extractDbHost } from '@/misc/convert-host';
import { getApId } from '@/remote/activitypub/type';
@@ -143,6 +143,8 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => {
fetchInstanceMetadata(i);
instanceChart.requestReceived(i.host);
+ apRequestChart.inbox();
+ federationChart.inbox(i.host);
});
// アクティビティを処理
diff --git a/packages/backend/src/queue/processors/system/clean-charts.ts b/packages/backend/src/queue/processors/system/clean-charts.ts
new file mode 100644
index 0000000000..fd156def9e
--- /dev/null
+++ b/packages/backend/src/queue/processors/system/clean-charts.ts
@@ -0,0 +1,28 @@
+import * as Bull from 'bull';
+
+import { queueLogger } from '../../logger';
+import { activeUsersChart, driveChart, federationChart, hashtagChart, instanceChart, notesChart, perUserDriveChart, perUserFollowingChart, perUserNotesChart, perUserReactionsChart, usersChart, apRequestChart } from '@/services/chart/index';
+
+const logger = queueLogger.createSubLogger('clean-charts');
+
+export async function cleanCharts(job: Bull.Job<Record<string, unknown>>, done: any): Promise<void> {
+ logger.info(`Resync charts...`);
+
+ await Promise.all([
+ federationChart.clean(),
+ notesChart.clean(),
+ usersChart.clean(),
+ activeUsersChart.clean(),
+ instanceChart.clean(),
+ perUserNotesChart.clean(),
+ driveChart.clean(),
+ perUserReactionsChart.clean(),
+ hashtagChart.clean(),
+ perUserFollowingChart.clean(),
+ perUserDriveChart.clean(),
+ apRequestChart.clean(),
+ ]);
+
+ logger.succ(`All charts successfully cleaned.`);
+ done();
+}
diff --git a/packages/backend/src/queue/processors/system/index.ts b/packages/backend/src/queue/processors/system/index.ts
index 8460ea0a9b..636fefc402 100644
--- a/packages/backend/src/queue/processors/system/index.ts
+++ b/packages/backend/src/queue/processors/system/index.ts
@@ -1,8 +1,10 @@
import * as Bull from 'bull';
import { resyncCharts } from './resync-charts';
+import { cleanCharts } from './clean-charts';
const jobs = {
resyncCharts,
+ cleanCharts,
} as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>> | Bull.ProcessPromiseFunction<Record<string, unknown>>>;
export default function(dbQueue: Bull.Queue<Record<string, unknown>>) {
diff --git a/packages/backend/src/server/api/endpoints/charts/active-users.ts b/packages/backend/src/server/api/endpoints/charts/active-users.ts
index f7eadc7089..a51cd00351 100644
--- a/packages/backend/src/server/api/endpoints/charts/active-users.ts
+++ b/packages/backend/src/server/api/endpoints/charts/active-users.ts
@@ -22,7 +22,7 @@ export const meta = {
},
},
- res: convertLog(activeUsersChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/charts/network.ts b/packages/backend/src/server/api/endpoints/charts/ap-request.ts
index c5a39bbd76..38bbddb27a 100644
--- a/packages/backend/src/server/api/endpoints/charts/network.ts
+++ b/packages/backend/src/server/api/endpoints/charts/ap-request.ts
@@ -1,7 +1,7 @@
import $ from 'cafy';
import define from '../../define';
import { convertLog } from '@/services/chart/core';
-import { networkChart } from '@/services/chart/index';
+import { apRequestChart } from '@/services/chart/index';
export const meta = {
tags: ['charts'],
@@ -22,10 +22,10 @@ export const meta = {
},
},
- res: convertLog(networkChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
export default define(meta, async (ps) => {
- return await networkChart.getChart(ps.span as any, ps.limit!, ps.offset ? new Date(ps.offset) : null);
+ return await apRequestChart.getChart(ps.span as any, ps.limit!, ps.offset ? new Date(ps.offset) : null);
});
diff --git a/packages/backend/src/server/api/endpoints/charts/drive.ts b/packages/backend/src/server/api/endpoints/charts/drive.ts
index 364279da95..4bbb9861f8 100644
--- a/packages/backend/src/server/api/endpoints/charts/drive.ts
+++ b/packages/backend/src/server/api/endpoints/charts/drive.ts
@@ -22,7 +22,7 @@ export const meta = {
},
},
- res: convertLog(driveChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/charts/federation.ts b/packages/backend/src/server/api/endpoints/charts/federation.ts
index 6feb82b6d9..237678ffed 100644
--- a/packages/backend/src/server/api/endpoints/charts/federation.ts
+++ b/packages/backend/src/server/api/endpoints/charts/federation.ts
@@ -22,7 +22,7 @@ export const meta = {
},
},
- res: convertLog(federationChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/charts/hashtag.ts b/packages/backend/src/server/api/endpoints/charts/hashtag.ts
index 99dc77998e..6c12cb063e 100644
--- a/packages/backend/src/server/api/endpoints/charts/hashtag.ts
+++ b/packages/backend/src/server/api/endpoints/charts/hashtag.ts
@@ -26,7 +26,7 @@ export const meta = {
},
},
- res: convertLog(hashtagChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/charts/instance.ts b/packages/backend/src/server/api/endpoints/charts/instance.ts
index 23e6fbf2b0..32a10d5a28 100644
--- a/packages/backend/src/server/api/endpoints/charts/instance.ts
+++ b/packages/backend/src/server/api/endpoints/charts/instance.ts
@@ -26,7 +26,7 @@ export const meta = {
},
},
- res: convertLog(instanceChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/charts/notes.ts b/packages/backend/src/server/api/endpoints/charts/notes.ts
index dcbd80c3e9..09255f1d2d 100644
--- a/packages/backend/src/server/api/endpoints/charts/notes.ts
+++ b/packages/backend/src/server/api/endpoints/charts/notes.ts
@@ -22,7 +22,7 @@ export const meta = {
},
},
- res: convertLog(notesChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/charts/user/drive.ts b/packages/backend/src/server/api/endpoints/charts/user/drive.ts
index 94787b4a57..89379858d1 100644
--- a/packages/backend/src/server/api/endpoints/charts/user/drive.ts
+++ b/packages/backend/src/server/api/endpoints/charts/user/drive.ts
@@ -27,7 +27,7 @@ export const meta = {
},
},
- res: convertLog(perUserDriveChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/charts/user/following.ts b/packages/backend/src/server/api/endpoints/charts/user/following.ts
index effe0c54b9..26c3f5c126 100644
--- a/packages/backend/src/server/api/endpoints/charts/user/following.ts
+++ b/packages/backend/src/server/api/endpoints/charts/user/following.ts
@@ -27,7 +27,7 @@ export const meta = {
},
},
- res: convertLog(perUserFollowingChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/charts/user/notes.ts b/packages/backend/src/server/api/endpoints/charts/user/notes.ts
index df68a5fe52..ecf85becfa 100644
--- a/packages/backend/src/server/api/endpoints/charts/user/notes.ts
+++ b/packages/backend/src/server/api/endpoints/charts/user/notes.ts
@@ -27,7 +27,7 @@ export const meta = {
},
},
- res: convertLog(perUserNotesChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/charts/user/reactions.ts b/packages/backend/src/server/api/endpoints/charts/user/reactions.ts
index dcd067305f..6e08d07511 100644
--- a/packages/backend/src/server/api/endpoints/charts/user/reactions.ts
+++ b/packages/backend/src/server/api/endpoints/charts/user/reactions.ts
@@ -27,7 +27,7 @@ export const meta = {
},
},
- res: convertLog(perUserReactionsChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/charts/users.ts b/packages/backend/src/server/api/endpoints/charts/users.ts
index d32e14ad61..15a9ec038e 100644
--- a/packages/backend/src/server/api/endpoints/charts/users.ts
+++ b/packages/backend/src/server/api/endpoints/charts/users.ts
@@ -22,7 +22,7 @@ export const meta = {
},
},
- res: convertLog(usersChart.schema),
+ // TODO: response definition
} as const;
// eslint-disable-next-line import/no-default-export
diff --git a/packages/backend/src/server/api/endpoints/stats.ts b/packages/backend/src/server/api/endpoints/stats.ts
index 9879ef2adf..164ca1c53f 100644
--- a/packages/backend/src/server/api/endpoints/stats.ts
+++ b/packages/backend/src/server/api/endpoints/stats.ts
@@ -66,8 +66,8 @@ export default define(meta, async () => {
NoteReactions.count({ cache: 3600000 }), // 1 hour
//NoteReactions.count({ where: { userHost: null }, cache: 3600000 }),
federationChart.getChart('hour', 1, null).then(chart => chart.instance.total[0]),
- driveChart.getChart('hour', 1, null).then(chart => chart.local.totalSize[0]),
- driveChart.getChart('hour', 1, null).then(chart => chart.remote.totalSize[0]),
+ driveChart.getChart('hour', 1, null).then(chart => chart.local.totalSize[0] * 1000),
+ driveChart.getChart('hour', 1, null).then(chart => chart.remote.totalSize[0] * 1000),
]);
return {
diff --git a/packages/backend/src/server/index.ts b/packages/backend/src/server/index.ts
index 764306c7d8..4d6b402e64 100644
--- a/packages/backend/src/server/index.ts
+++ b/packages/backend/src/server/index.ts
@@ -10,7 +10,6 @@ import * as Koa from 'koa';
import * as Router from '@koa/router';
import * as mount from 'koa-mount';
import * as koaLogger from 'koa-logger';
-import * as requestStats from 'request-stats';
import * as slow from 'koa-slow';
import activityPub from './activitypub';
@@ -18,11 +17,9 @@ import nodeinfo from './nodeinfo';
import wellKnown from './well-known';
import config from '@/config/index';
import apiServer from './api/index';
-import { sum } from '@/prelude/array';
import Logger from '@/services/logger';
import { envOption } from '../env';
import { UserProfiles, Users } from '@/models/index';
-import { networkChart } from '@/services/chart/index';
import { genIdenticon } from '@/misc/gen-identicon';
import { createTemp } from '@/misc/create-temp';
import { publishMainStream } from '@/services/stream';
@@ -153,27 +150,4 @@ export default () => new Promise(resolve => {
// Listen
server.listen(config.port, resolve);
-
- //#region Network stats
- let queue: any[] = [];
-
- requestStats(server, (stats: any) => {
- if (stats.ok) {
- queue.push(stats);
- }
- });
-
- // Bulk write
- setInterval(() => {
- if (queue.length === 0) return;
-
- const requests = queue.length;
- const time = sum(queue.map(x => x.time));
- const incomingBytes = sum(queue.map(x => x.req.byets));
- const outgoingBytes = sum(queue.map(x => x.res.byets));
- queue = [];
-
- networkChart.update(requests, time, incomingBytes, outgoingBytes);
- }, 5000);
- //#endregion
});
diff --git a/packages/backend/src/services/chart/charts/active-users.ts b/packages/backend/src/services/chart/charts/active-users.ts
index 9490101e36..9e5c332d3b 100644
--- a/packages/backend/src/services/chart/charts/active-users.ts
+++ b/packages/backend/src/services/chart/charts/active-users.ts
@@ -1,51 +1,28 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
+import Chart, { KVs } from '../core';
import { User } from '@/models/entities/user';
-import { SchemaType } from '@/misc/schema';
import { Users } from '@/models/index';
import { name, schema } from './entities/active-users';
-type ActiveUsersLog = SchemaType<typeof schema>;
-
/**
* アクティブユーザーに関するチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class ActiveUsersChart extends Chart<ActiveUsersLog> {
+export default class ActiveUsersChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
- protected genNewLog(latest: ActiveUsersLog): DeepPartial<ActiveUsersLog> {
- return {};
- }
-
- @autobind
- protected aggregate(logs: ActiveUsersLog[]): ActiveUsersLog {
- return {
- local: {
- users: logs.reduce((a, b) => a.concat(b.local.users), [] as ActiveUsersLog['local']['users']),
- },
- remote: {
- users: logs.reduce((a, b) => a.concat(b.remote.users), [] as ActiveUsersLog['remote']['users']),
- },
- };
- }
-
- @autobind
- protected async fetchActual(): Promise<DeepPartial<ActiveUsersLog>> {
+ protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async update(user: { id: User['id'], host: User['host'] }): Promise<void> {
- const update: Obj = {
- users: [user.id],
- };
-
- await this.inc({
- [Users.isLocalUser(user) ? 'local' : 'remote']: update,
+ await this.commit({
+ 'local.users': Users.isLocalUser(user) ? [user.id] : [],
+ 'remote.users': Users.isLocalUser(user) ? [] : [user.id],
});
}
}
diff --git a/packages/backend/src/services/chart/charts/ap-request.ts b/packages/backend/src/services/chart/charts/ap-request.ts
new file mode 100644
index 0000000000..bac5e425c8
--- /dev/null
+++ b/packages/backend/src/services/chart/charts/ap-request.ts
@@ -0,0 +1,39 @@
+import autobind from 'autobind-decorator';
+import Chart, { KVs } from '../core';
+import { name, schema } from './entities/ap-request';
+
+/**
+ * Chart about ActivityPub requests
+ */
+// eslint-disable-next-line import/no-default-export
+export default class ApRequestChart extends Chart<typeof schema> {
+ constructor() {
+ super(name, schema);
+ }
+
+ @autobind
+ protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+ return {};
+ }
+
+ @autobind
+ public async deliverSucc(): Promise<void> {
+ await this.commit({
+ 'deliverSucceeded': 1,
+ });
+ }
+
+ @autobind
+ public async deliverFail(): Promise<void> {
+ await this.commit({
+ 'deliverFailed': 1,
+ });
+ }
+
+ @autobind
+ public async inbox(): Promise<void> {
+ await this.commit({
+ 'inboxReceived': 1,
+ });
+ }
+}
diff --git a/packages/backend/src/services/chart/charts/drive.ts b/packages/backend/src/services/chart/charts/drive.ts
index 06cf7ebeeb..2f00adae2b 100644
--- a/packages/backend/src/services/chart/charts/drive.ts
+++ b/packages/backend/src/services/chart/charts/drive.ts
@@ -1,95 +1,37 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
+import Chart, { KVs } from '../core';
import { DriveFiles } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { DriveFile } from '@/models/entities/drive-file';
import { name, schema } from './entities/drive';
-type DriveLog = SchemaType<typeof schema>;
-
/**
* ドライブに関するチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class DriveChart extends Chart<DriveLog> {
+export default class DriveChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
- protected genNewLog(latest: DriveLog): DeepPartial<DriveLog> {
- return {
- local: {
- totalCount: latest.local.totalCount,
- totalSize: latest.local.totalSize,
- },
- remote: {
- totalCount: latest.remote.totalCount,
- totalSize: latest.remote.totalSize,
- },
- };
- }
-
- @autobind
- protected aggregate(logs: DriveLog[]): DriveLog {
- return {
- local: {
- totalCount: logs[0].local.totalCount,
- totalSize: logs[0].local.totalSize,
- incCount: logs.reduce((a, b) => a + b.local.incCount, 0),
- incSize: logs.reduce((a, b) => a + b.local.incSize, 0),
- decCount: logs.reduce((a, b) => a + b.local.decCount, 0),
- decSize: logs.reduce((a, b) => a + b.local.decSize, 0),
- },
- remote: {
- totalCount: logs[0].remote.totalCount,
- totalSize: logs[0].remote.totalSize,
- incCount: logs.reduce((a, b) => a + b.remote.incCount, 0),
- incSize: logs.reduce((a, b) => a + b.remote.incSize, 0),
- decCount: logs.reduce((a, b) => a + b.remote.decCount, 0),
- decSize: logs.reduce((a, b) => a + b.remote.decSize, 0),
- },
- };
- }
-
- @autobind
- protected async fetchActual(): Promise<DeepPartial<DriveLog>> {
- const [localCount, remoteCount, localSize, remoteSize] = await Promise.all([
- DriveFiles.count({ userHost: null }),
- DriveFiles.count({ userHost: Not(IsNull()) }),
- DriveFiles.calcDriveUsageOfLocal(),
- DriveFiles.calcDriveUsageOfRemote(),
- ]);
-
- return {
- local: {
- totalCount: localCount,
- totalSize: localSize,
- },
- remote: {
- totalCount: remoteCount,
- totalSize: remoteSize,
- },
- };
+ protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+ return {};
}
@autobind
public async update(file: DriveFile, isAdditional: boolean): Promise<void> {
- const update: Obj = {};
-
- update.totalCount = isAdditional ? 1 : -1;
- update.totalSize = isAdditional ? file.size : -file.size;
- if (isAdditional) {
- update.incCount = 1;
- update.incSize = file.size;
- } else {
- update.decCount = 1;
- update.decSize = file.size;
- }
-
- await this.inc({
- [file.userHost === null ? 'local' : 'remote']: update,
+ const fileSizeKb = file.size / 1000;
+ await this.commit(file.userHost === null ? {
+ 'local.incCount': isAdditional ? 1 : 0,
+ 'local.incSize': isAdditional ? fileSizeKb : 0,
+ 'local.decCount': isAdditional ? 0 : 1,
+ 'local.decSize': isAdditional ? 0 : fileSizeKb,
+ } : {
+ 'remote.incCount': isAdditional ? 1 : 0,
+ 'remote.incSize': isAdditional ? fileSizeKb : 0,
+ 'remote.decCount': isAdditional ? 0 : 1,
+ 'remote.decSize': isAdditional ? 0 : fileSizeKb,
});
}
}
diff --git a/packages/backend/src/services/chart/charts/entities/active-users.ts b/packages/backend/src/services/chart/charts/entities/active-users.ts
index d6b49c86c3..3caa938d35 100644
--- a/packages/backend/src/services/chart/charts/entities/active-users.ts
+++ b/packages/backend/src/services/chart/charts/entities/active-users.ts
@@ -2,35 +2,9 @@ import Chart from '../../core';
export const name = 'activeUsers';
-const logSchema = {
- /**
- * アクティブユーザー
- */
- users: {
- type: 'array' as const,
- optional: false as const, nullable: false as const,
- items: {
- type: 'string' as const,
- optional: false as const, nullable: false as const,
- },
- },
-};
-
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- local: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- remote: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- },
-};
+ 'local.users': { uniqueIncrement: true },
+ 'remote.users': { uniqueIncrement: true },
+} as const;
export const entity = Chart.schemaToEntity(name, schema);
diff --git a/packages/backend/src/services/chart/charts/entities/ap-request.ts b/packages/backend/src/services/chart/charts/entities/ap-request.ts
new file mode 100644
index 0000000000..21fb40d138
--- /dev/null
+++ b/packages/backend/src/services/chart/charts/entities/ap-request.ts
@@ -0,0 +1,11 @@
+import Chart from '../../core';
+
+export const name = 'apRequest';
+
+export const schema = {
+ 'deliverFailed': { },
+ 'deliverSucceeded': { },
+ 'inboxReceived': { },
+} as const;
+
+export const entity = Chart.schemaToEntity(name, schema);
diff --git a/packages/backend/src/services/chart/charts/entities/drive.ts b/packages/backend/src/services/chart/charts/entities/drive.ts
index 3362cbd4cb..c5cdfd85bd 100644
--- a/packages/backend/src/services/chart/charts/entities/drive.ts
+++ b/packages/backend/src/services/chart/charts/entities/drive.ts
@@ -2,71 +2,15 @@ import Chart from '../../core';
export const name = 'drive';
-const logSchema = {
- /**
- * 集計期間時点での、全ドライブファイル数
- */
- totalCount: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 集計期間時点での、全ドライブファイルの合計サイズ
- */
- totalSize: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 増加したドライブファイル数
- */
- incCount: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 増加したドライブ使用量
- */
- incSize: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 減少したドライブファイル数
- */
- decCount: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 減少したドライブ使用量
- */
- decSize: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-};
-
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- local: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- remote: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- },
-};
+ 'local.incCount': {},
+ 'local.incSize': {}, // in kilobyte
+ 'local.decCount': {},
+ 'local.decSize': {}, // in kilobyte
+ 'remote.incCount': {},
+ 'remote.incSize': {}, // in kilobyte
+ 'remote.decCount': {},
+ 'remote.decSize': {}, // in kilobyte
+} as const;
export const entity = Chart.schemaToEntity(name, schema);
diff --git a/packages/backend/src/services/chart/charts/entities/federation.ts b/packages/backend/src/services/chart/charts/entities/federation.ts
index 836116bd06..5a76c4918c 100644
--- a/packages/backend/src/services/chart/charts/entities/federation.ts
+++ b/packages/backend/src/services/chart/charts/entities/federation.ts
@@ -3,28 +3,11 @@ import Chart from '../../core';
export const name = 'federation';
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- instance: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
- },
-};
+ 'instance.total': { accumulate: true },
+ 'instance.inc': { range: 'small' },
+ 'instance.dec': { range: 'small' },
+ 'deliveredInstances': { uniqueIncrement: true, range: 'small' },
+ 'inboxInstances': { uniqueIncrement: true, range: 'small' },
+} as const;
export const entity = Chart.schemaToEntity(name, schema);
diff --git a/packages/backend/src/services/chart/charts/entities/hashtag.ts b/packages/backend/src/services/chart/charts/entities/hashtag.ts
index 43e15456a5..bd2ae38a16 100644
--- a/packages/backend/src/services/chart/charts/entities/hashtag.ts
+++ b/packages/backend/src/services/chart/charts/entities/hashtag.ts
@@ -2,35 +2,9 @@ import Chart from '../../core';
export const name = 'hashtag';
-const logSchema = {
- /**
- * 投稿したユーザー
- */
- users: {
- type: 'array' as const,
- optional: false as const, nullable: false as const,
- items: {
- type: 'string' as const,
- optional: false as const, nullable: false as const,
- },
- },
-};
-
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- local: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- remote: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- },
-};
+ 'local.users': { uniqueIncrement: true },
+ 'remote.users': { uniqueIncrement: true },
+} as const;
export const entity = Chart.schemaToEntity(name, schema, true);
diff --git a/packages/backend/src/services/chart/charts/entities/instance.ts b/packages/backend/src/services/chart/charts/entities/instance.ts
index 9d1f651dbb..0ff9e7b6b8 100644
--- a/packages/backend/src/services/chart/charts/entities/instance.ts
+++ b/packages/backend/src/services/chart/charts/entities/instance.ts
@@ -3,156 +3,29 @@ import Chart from '../../core';
export const name = 'instance';
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- requests: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- failed: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- succeeded: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- received: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
-
- notes: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- diffs: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- normal: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- reply: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- renote: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
- },
- },
-
- users: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
-
- following: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
-
- followers: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
-
- drive: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- totalFiles: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- totalUsage: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- incFiles: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- incUsage: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- decFiles: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- decUsage: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
- },
-};
+ 'requests.failed': { range: 'small' },
+ 'requests.succeeded': { range: 'small' },
+ 'requests.received': { range: 'small' },
+ 'notes.total': { accumulate: true },
+ 'notes.inc': {},
+ 'notes.dec': {},
+ 'notes.diffs.normal': {},
+ 'notes.diffs.reply': {},
+ 'notes.diffs.renote': {},
+ 'users.total': { accumulate: true },
+ 'users.inc': { range: 'small' },
+ 'users.dec': { range: 'small' },
+ 'following.total': { accumulate: true },
+ 'following.inc': { range: 'small' },
+ 'following.dec': { range: 'small' },
+ 'followers.total': { accumulate: true },
+ 'followers.inc': { range: 'small' },
+ 'followers.dec': { range: 'small' },
+ 'drive.totalFiles': { accumulate: true },
+ 'drive.incFiles': {},
+ 'drive.decFiles': {},
+ 'drive.incUsage': {}, // in kilobyte
+ 'drive.decUsage': {}, // in kilobyte
+} as const;
export const entity = Chart.schemaToEntity(name, schema, true);
diff --git a/packages/backend/src/services/chart/charts/entities/network.ts b/packages/backend/src/services/chart/charts/entities/network.ts
deleted file mode 100644
index 3d4fffb855..0000000000
--- a/packages/backend/src/services/chart/charts/entities/network.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import Chart from '../../core';
-
-export const name = 'network';
-
-export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- incomingRequests: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- outgoingRequests: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- totalTime: { // TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- incomingBytes: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- outgoingBytes: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
-};
-
-export const entity = Chart.schemaToEntity(name, schema);
diff --git a/packages/backend/src/services/chart/charts/entities/notes.ts b/packages/backend/src/services/chart/charts/entities/notes.ts
index 554d3abe12..296328f102 100644
--- a/packages/backend/src/services/chart/charts/entities/notes.ts
+++ b/packages/backend/src/services/chart/charts/entities/notes.ts
@@ -2,59 +2,19 @@ import Chart from '../../core';
export const name = 'notes';
-const logSchema = {
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- diffs: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- normal: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- reply: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- renote: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
-};
-
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- local: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- remote: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- },
-};
+ 'local.total': { accumulate: true },
+ 'local.inc': {},
+ 'local.dec': {},
+ 'local.diffs.normal': {},
+ 'local.diffs.reply': {},
+ 'local.diffs.renote': {},
+ 'remote.total': { accumulate: true },
+ 'remote.inc': {},
+ 'remote.dec': {},
+ 'remote.diffs.normal': {},
+ 'remote.diffs.reply': {},
+ 'remote.diffs.renote': {},
+} as const;
export const entity = Chart.schemaToEntity(name, schema);
diff --git a/packages/backend/src/services/chart/charts/entities/per-user-drive.ts b/packages/backend/src/services/chart/charts/entities/per-user-drive.ts
index ebf64e733e..00d85b1620 100644
--- a/packages/backend/src/services/chart/charts/entities/per-user-drive.ts
+++ b/packages/backend/src/services/chart/charts/entities/per-user-drive.ts
@@ -3,57 +3,12 @@ import Chart from '../../core';
export const name = 'perUserDrive';
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- /**
- * 集計期間時点での、全ドライブファイル数
- */
- totalCount: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 集計期間時点での、全ドライブファイルの合計サイズ
- */
- totalSize: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 増加したドライブファイル数
- */
- incCount: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 増加したドライブ使用量
- */
- incSize: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 減少したドライブファイル数
- */
- decCount: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 減少したドライブ使用量
- */
- decSize: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
-};
+ 'totalCount': { accumulate: true },
+ 'totalSize': { accumulate: true }, // in kilobyte
+ 'incCount': { range: 'small' },
+ 'incSize': {}, // in kilobyte
+ 'decCount': { range: 'small' },
+ 'decSize': {}, // in kilobyte
+} as const;
export const entity = Chart.schemaToEntity(name, schema, true);
diff --git a/packages/backend/src/services/chart/charts/entities/per-user-following.ts b/packages/backend/src/services/chart/charts/entities/per-user-following.ts
index 8016c5fe97..1efd4977fc 100644
--- a/packages/backend/src/services/chart/charts/entities/per-user-following.ts
+++ b/packages/backend/src/services/chart/charts/entities/per-user-following.ts
@@ -2,89 +2,19 @@ import Chart from '../../core';
export const name = 'perUserFollowing';
-const logSchema = {
- /**
- * フォローしている
- */
- followings: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- /**
- * フォローしている合計
- */
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * フォローした数
- */
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * フォロー解除した数
- */
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
-
- /**
- * フォローされている
- */
- followers: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- /**
- * フォローされている合計
- */
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * フォローされた数
- */
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * フォロー解除された数
- */
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
-};
-
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- local: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- remote: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- },
-};
+ 'local.followings.total': { accumulate: true },
+ 'local.followings.inc': { range: 'small' },
+ 'local.followings.dec': { range: 'small' },
+ 'local.followers.total': { accumulate: true },
+ 'local.followers.inc': { range: 'small' },
+ 'local.followers.dec': { range: 'small' },
+ 'remote.followings.total': { accumulate: true },
+ 'remote.followings.inc': { range: 'small' },
+ 'remote.followings.dec': { range: 'small' },
+ 'remote.followers.total': { accumulate: true },
+ 'remote.followers.inc': { range: 'small' },
+ 'remote.followers.dec': { range: 'small' },
+} as const;
export const entity = Chart.schemaToEntity(name, schema, true);
diff --git a/packages/backend/src/services/chart/charts/entities/per-user-notes.ts b/packages/backend/src/services/chart/charts/entities/per-user-notes.ts
index d8f645b36e..bf7e4422be 100644
--- a/packages/backend/src/services/chart/charts/entities/per-user-notes.ts
+++ b/packages/backend/src/services/chart/charts/entities/per-user-notes.ts
@@ -3,45 +3,12 @@ import Chart from '../../core';
export const name = 'perUserNotes';
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- diffs: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- normal: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- reply: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- renote: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
- },
-};
+ 'total': { accumulate: true },
+ 'inc': { range: 'small' },
+ 'dec': { range: 'small' },
+ 'diffs.normal': { range: 'small' },
+ 'diffs.reply': { range: 'small' },
+ 'diffs.renote': { range: 'small' },
+} as const;
export const entity = Chart.schemaToEntity(name, schema, true);
diff --git a/packages/backend/src/services/chart/charts/entities/per-user-reactions.ts b/packages/backend/src/services/chart/charts/entities/per-user-reactions.ts
index bcb7012661..ab315d24c9 100644
--- a/packages/backend/src/services/chart/charts/entities/per-user-reactions.ts
+++ b/packages/backend/src/services/chart/charts/entities/per-user-reactions.ts
@@ -2,31 +2,9 @@ import Chart from '../../core';
export const name = 'perUserReaction';
-const logSchema = {
- /**
- * 被リアクション数
- */
- count: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-};
-
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- local: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- remote: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- },
-};
+ 'local.count': { range: 'small' },
+ 'remote.count': { range: 'small' },
+} as const;
export const entity = Chart.schemaToEntity(name, schema, true);
diff --git a/packages/backend/src/services/chart/charts/entities/test-grouped.ts b/packages/backend/src/services/chart/charts/entities/test-grouped.ts
index ca1c8c5700..78c2bbd548 100644
--- a/packages/backend/src/services/chart/charts/entities/test-grouped.ts
+++ b/packages/backend/src/services/chart/charts/entities/test-grouped.ts
@@ -3,30 +3,9 @@ import Chart from '../../core';
export const name = 'testGrouped';
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- foo: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
- },
-};
+ 'foo.total': { accumulate: true },
+ 'foo.inc': {},
+ 'foo.dec': {},
+} as const;
export const entity = Chart.schemaToEntity(name, schema, true);
diff --git a/packages/backend/src/services/chart/charts/entities/test-unique.ts b/packages/backend/src/services/chart/charts/entities/test-unique.ts
index 2e917ee9ed..dc7c1520e1 100644
--- a/packages/backend/src/services/chart/charts/entities/test-unique.ts
+++ b/packages/backend/src/services/chart/charts/entities/test-unique.ts
@@ -3,18 +3,7 @@ import Chart from '../../core';
export const name = 'testUnique';
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- foo: {
- type: 'array' as const,
- optional: false as const, nullable: false as const,
- items: {
- type: 'string' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
-};
+ 'foo': { uniqueIncrement: true },
+} as const;
export const entity = Chart.schemaToEntity(name, schema);
diff --git a/packages/backend/src/services/chart/charts/entities/test.ts b/packages/backend/src/services/chart/charts/entities/test.ts
index fa536ff2cf..edfa4c524b 100644
--- a/packages/backend/src/services/chart/charts/entities/test.ts
+++ b/packages/backend/src/services/chart/charts/entities/test.ts
@@ -3,30 +3,9 @@ import Chart from '../../core';
export const name = 'test';
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- foo: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
- },
- },
- },
-};
+ 'foo.total': { accumulate: true },
+ 'foo.inc': {},
+ 'foo.dec': {},
+} as const;
export const entity = Chart.schemaToEntity(name, schema);
diff --git a/packages/backend/src/services/chart/charts/entities/users.ts b/packages/backend/src/services/chart/charts/entities/users.ts
index 08d51c9414..d2cec72497 100644
--- a/packages/backend/src/services/chart/charts/entities/users.ts
+++ b/packages/backend/src/services/chart/charts/entities/users.ts
@@ -2,47 +2,13 @@ import Chart from '../../core';
export const name = 'users';
-const logSchema = {
- /**
- * 集計期間時点での、全ユーザー数
- */
- total: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 増加したユーザー数
- */
- inc: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-
- /**
- * 減少したユーザー数
- */
- dec: {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- },
-};
-
export const schema = {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: {
- local: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- remote: {
- type: 'object' as const,
- optional: false as const, nullable: false as const,
- properties: logSchema,
- },
- },
-};
+ 'local.total': { accumulate: true },
+ 'local.inc': { range: 'small' },
+ 'local.dec': { range: 'small' },
+ 'remote.total': { accumulate: true },
+ 'remote.inc': { range: 'small' },
+ 'remote.dec': { range: 'small' },
+} as const;
export const entity = Chart.schemaToEntity(name, schema);
diff --git a/packages/backend/src/services/chart/charts/federation.ts b/packages/backend/src/services/chart/charts/federation.ts
index 8abb18b51f..3aa448e66f 100644
--- a/packages/backend/src/services/chart/charts/federation.ts
+++ b/packages/backend/src/services/chart/charts/federation.ts
@@ -1,66 +1,48 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
+import Chart, { KVs } from '../core';
import { Instances } from '@/models/index';
import { name, schema } from './entities/federation';
-type FederationLog = SchemaType<typeof schema>;
-
/**
* フェデレーションに関するチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class FederationChart extends Chart<FederationLog> {
+export default class FederationChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
- protected genNewLog(latest: FederationLog): DeepPartial<FederationLog> {
- return {
- instance: {
- total: latest.instance.total,
- },
- };
- }
-
- @autobind
- protected aggregate(logs: FederationLog[]): FederationLog {
- return {
- instance: {
- total: logs[0].instance.total,
- inc: logs.reduce((a, b) => a + b.instance.inc, 0),
- dec: logs.reduce((a, b) => a + b.instance.dec, 0),
- },
- };
- }
-
- @autobind
- protected async fetchActual(): Promise<DeepPartial<FederationLog>> {
+ protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
const [total] = await Promise.all([
Instances.count({}),
]);
return {
- instance: {
- total: total,
- },
+ 'instance.total': total,
};
}
@autobind
public async update(isAdditional: boolean): Promise<void> {
- const update: Obj = {};
+ await this.commit({
+ 'instance.total': isAdditional ? 1 : -1,
+ 'instance.inc': isAdditional ? 1 : 0,
+ 'instance.dec': isAdditional ? 0 : 1,
+ });
+ }
- update.total = isAdditional ? 1 : -1;
- if (isAdditional) {
- update.inc = 1;
- } else {
- update.dec = 1;
- }
+ @autobind
+ public async deliverd(host: string): Promise<void> {
+ await this.commit({
+ 'deliveredInstances': [host],
+ });
+ }
- await this.inc({
- instance: update,
+ @autobind
+ public async inbox(host: string): Promise<void> {
+ await this.commit({
+ 'inboxInstances': [host],
});
}
}
diff --git a/packages/backend/src/services/chart/charts/hashtag.ts b/packages/backend/src/services/chart/charts/hashtag.ts
index 34e0614643..0b7bc467d2 100644
--- a/packages/backend/src/services/chart/charts/hashtag.ts
+++ b/packages/backend/src/services/chart/charts/hashtag.ts
@@ -1,51 +1,28 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
+import Chart, { KVs } from '../core';
import { User } from '@/models/entities/user';
-import { SchemaType } from '@/misc/schema';
import { Users } from '@/models/index';
import { name, schema } from './entities/hashtag';
-type HashtagLog = SchemaType<typeof schema>;
-
/**
* ハッシュタグに関するチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class HashtagChart extends Chart<HashtagLog> {
+export default class HashtagChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
- protected genNewLog(latest: HashtagLog): DeepPartial<HashtagLog> {
- return {};
- }
-
- @autobind
- protected aggregate(logs: HashtagLog[]): HashtagLog {
- return {
- local: {
- users: logs.reduce((a, b) => a.concat(b.local.users), [] as HashtagLog['local']['users']),
- },
- remote: {
- users: logs.reduce((a, b) => a.concat(b.remote.users), [] as HashtagLog['remote']['users']),
- },
- };
- }
-
- @autobind
- protected async fetchActual(): Promise<DeepPartial<HashtagLog>> {
+ protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async update(hashtag: string, user: { id: User['id'], host: User['host'] }): Promise<void> {
- const update: Obj = {
- users: [user.id],
- };
-
- await this.inc({
- [Users.isLocalUser(user) ? 'local' : 'remote']: update,
+ await this.commit({
+ 'local.users': Users.isLocalUser(user) ? [user.id] : [],
+ 'remote.users': Users.isLocalUser(user) ? [] : [user.id],
}, hashtag);
}
}
diff --git a/packages/backend/src/services/chart/charts/instance.ts b/packages/backend/src/services/chart/charts/instance.ts
index 7f3419b69c..b893dde34c 100644
--- a/packages/backend/src/services/chart/charts/instance.ts
+++ b/packages/backend/src/services/chart/charts/instance.ts
@@ -1,158 +1,67 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
+import Chart, { KVs } from '../core';
import { DriveFiles, Followings, Users, Notes } from '@/models/index';
import { DriveFile } from '@/models/entities/drive-file';
import { Note } from '@/models/entities/note';
import { toPuny } from '@/misc/convert-host';
import { name, schema } from './entities/instance';
-type InstanceLog = SchemaType<typeof schema>;
-
/**
* インスタンスごとのチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class InstanceChart extends Chart<InstanceLog> {
+export default class InstanceChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
- protected genNewLog(latest: InstanceLog): DeepPartial<InstanceLog> {
- return {
- notes: {
- total: latest.notes.total,
- },
- users: {
- total: latest.users.total,
- },
- following: {
- total: latest.following.total,
- },
- followers: {
- total: latest.followers.total,
- },
- drive: {
- totalFiles: latest.drive.totalFiles,
- totalUsage: latest.drive.totalUsage,
- },
- };
- }
-
- @autobind
- protected aggregate(logs: InstanceLog[]): InstanceLog {
- return {
- requests: {
- failed: logs.reduce((a, b) => a + b.requests.failed, 0),
- succeeded: logs.reduce((a, b) => a + b.requests.succeeded, 0),
- received: logs.reduce((a, b) => a + b.requests.received, 0),
- },
- notes: {
- total: logs[0].notes.total,
- inc: logs.reduce((a, b) => a + b.notes.inc, 0),
- dec: logs.reduce((a, b) => a + b.notes.dec, 0),
- diffs: {
- reply: logs.reduce((a, b) => a + b.notes.diffs.reply, 0),
- renote: logs.reduce((a, b) => a + b.notes.diffs.renote, 0),
- normal: logs.reduce((a, b) => a + b.notes.diffs.normal, 0),
- },
- },
- users: {
- total: logs[0].users.total,
- inc: logs.reduce((a, b) => a + b.users.inc, 0),
- dec: logs.reduce((a, b) => a + b.users.dec, 0),
- },
- following: {
- total: logs[0].following.total,
- inc: logs.reduce((a, b) => a + b.following.inc, 0),
- dec: logs.reduce((a, b) => a + b.following.dec, 0),
- },
- followers: {
- total: logs[0].followers.total,
- inc: logs.reduce((a, b) => a + b.followers.inc, 0),
- dec: logs.reduce((a, b) => a + b.followers.dec, 0),
- },
- drive: {
- totalFiles: logs[0].drive.totalFiles,
- totalUsage: logs[0].drive.totalUsage,
- incFiles: logs.reduce((a, b) => a + b.drive.incFiles, 0),
- incUsage: logs.reduce((a, b) => a + b.drive.incUsage, 0),
- decFiles: logs.reduce((a, b) => a + b.drive.decFiles, 0),
- decUsage: logs.reduce((a, b) => a + b.drive.decUsage, 0),
- },
- };
- }
-
- @autobind
- protected async fetchActual(group: string): Promise<DeepPartial<InstanceLog>> {
+ protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
const [
notesCount,
usersCount,
followingCount,
followersCount,
driveFiles,
- driveUsage,
+ //driveUsage,
] = await Promise.all([
Notes.count({ userHost: group }),
Users.count({ host: group }),
Followings.count({ followerHost: group }),
Followings.count({ followeeHost: group }),
DriveFiles.count({ userHost: group }),
- DriveFiles.calcDriveUsageOfHost(group),
+ //DriveFiles.calcDriveUsageOfHost(group),
]);
return {
- notes: {
- total: notesCount,
- },
- users: {
- total: usersCount,
- },
- following: {
- total: followingCount,
- },
- followers: {
- total: followersCount,
- },
- drive: {
- totalFiles: driveFiles,
- totalUsage: driveUsage,
- },
+ 'notes.total': notesCount,
+ 'users.total': usersCount,
+ 'following.total': followingCount,
+ 'followers.total': followersCount,
+ 'drive.totalFiles': driveFiles,
};
}
@autobind
public async requestReceived(host: string): Promise<void> {
- await this.inc({
- requests: {
- received: 1,
- },
+ await this.commit({
+ 'requests.received': 1,
}, toPuny(host));
}
@autobind
public async requestSent(host: string, isSucceeded: boolean): Promise<void> {
- const update: Obj = {};
-
- if (isSucceeded) {
- update.succeeded = 1;
- } else {
- update.failed = 1;
- }
-
- await this.inc({
- requests: update,
+ await this.commit({
+ 'requests.succeeded': isSucceeded ? 1 : 0,
+ 'requests.failed': isSucceeded ? 0 : 1,
}, toPuny(host));
}
@autobind
public async newUser(host: string): Promise<void> {
- await this.inc({
- users: {
- total: 1,
- inc: 1,
- },
+ await this.commit({
+ 'users.total': 1,
+ 'users.inc': 1,
}, toPuny(host));
}
@@ -168,54 +77,43 @@ export default class InstanceChart extends Chart<InstanceLog> {
diffs.normal = isAdditional ? 1 : -1;
}
- await this.inc({
- notes: {
- total: isAdditional ? 1 : -1,
- inc: isAdditional ? 1 : 0,
- dec: isAdditional ? 0 : 1,
- diffs: diffs,
- },
+ await this.commit({
+ 'notes.total': isAdditional ? 1 : -1,
+ 'notes.inc': isAdditional ? 1 : 0,
+ 'notes.dec': isAdditional ? 0 : 1,
+ 'notes.diffs.normal': note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0,
+ 'notes.diffs.renote': note.renoteId != null ? (isAdditional ? 1 : -1) : 0,
+ 'notes.diffs.reply': note.replyId != null ? (isAdditional ? 1 : -1) : 0,
}, toPuny(host));
}
@autobind
public async updateFollowing(host: string, isAdditional: boolean): Promise<void> {
- await this.inc({
- following: {
- total: isAdditional ? 1 : -1,
- inc: isAdditional ? 1 : 0,
- dec: isAdditional ? 0 : 1,
- },
+ await this.commit({
+ 'following.total': isAdditional ? 1 : -1,
+ 'following.inc': isAdditional ? 1 : 0,
+ 'following.dec': isAdditional ? 0 : 1,
}, toPuny(host));
}
@autobind
public async updateFollowers(host: string, isAdditional: boolean): Promise<void> {
- await this.inc({
- followers: {
- total: isAdditional ? 1 : -1,
- inc: isAdditional ? 1 : 0,
- dec: isAdditional ? 0 : 1,
- },
+ await this.commit({
+ 'followers.total': isAdditional ? 1 : -1,
+ 'followers.inc': isAdditional ? 1 : 0,
+ 'followers.dec': isAdditional ? 0 : 1,
}, toPuny(host));
}
@autobind
public async updateDrive(file: DriveFile, isAdditional: boolean): Promise<void> {
- const update: Obj = {};
-
- update.totalFiles = isAdditional ? 1 : -1;
- update.totalUsage = isAdditional ? file.size : -file.size;
- if (isAdditional) {
- update.incFiles = 1;
- update.incUsage = file.size;
- } else {
- update.decFiles = 1;
- update.decUsage = file.size;
- }
-
- await this.inc({
- drive: update,
+ const fileSizeKb = file.size / 1000;
+ await this.commit({
+ 'drive.totalFiles': isAdditional ? 1 : -1,
+ 'drive.incFiles': isAdditional ? 1 : 0,
+ 'drive.incUsage': isAdditional ? fileSizeKb : 0,
+ 'drive.decFiles': isAdditional ? 1 : 0,
+ 'drive.decUsage': isAdditional ? fileSizeKb : 0,
}, file.userHost);
}
}
diff --git a/packages/backend/src/services/chart/charts/network.ts b/packages/backend/src/services/chart/charts/network.ts
deleted file mode 100644
index 73ea2f7e19..0000000000
--- a/packages/backend/src/services/chart/charts/network.ts
+++ /dev/null
@@ -1,49 +0,0 @@
-import autobind from 'autobind-decorator';
-import Chart, { DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
-import { name, schema } from './entities/network';
-
-type NetworkLog = SchemaType<typeof schema>;
-
-/**
- * ネットワークに関するチャート
- */
-// eslint-disable-next-line import/no-default-export
-export default class NetworkChart extends Chart<NetworkLog> {
- constructor() {
- super(name, schema);
- }
-
- @autobind
- protected genNewLog(latest: NetworkLog): DeepPartial<NetworkLog> {
- return {};
- }
-
- @autobind
- protected aggregate(logs: NetworkLog[]): NetworkLog {
- return {
- incomingRequests: logs.reduce((a, b) => a + b.incomingRequests, 0),
- outgoingRequests: logs.reduce((a, b) => a + b.outgoingRequests, 0),
- totalTime: logs.reduce((a, b) => a + b.totalTime, 0),
- incomingBytes: logs.reduce((a, b) => a + b.incomingBytes, 0),
- outgoingBytes: logs.reduce((a, b) => a + b.outgoingBytes, 0),
- };
- }
-
- @autobind
- protected async fetchActual(): Promise<DeepPartial<NetworkLog>> {
- return {};
- }
-
- @autobind
- public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number): Promise<void> {
- const inc: DeepPartial<NetworkLog> = {
- incomingRequests: incomingRequests,
- totalTime: time,
- incomingBytes: incomingBytes,
- outgoingBytes: outgoingBytes,
- };
-
- await this.inc(inc);
- }
-}
diff --git a/packages/backend/src/services/chart/charts/notes.ts b/packages/backend/src/services/chart/charts/notes.ts
index 86cda17225..4bbfa6704f 100644
--- a/packages/backend/src/services/chart/charts/notes.ts
+++ b/packages/backend/src/services/chart/charts/notes.ts
@@ -1,101 +1,43 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
+import Chart, { KVs } from '../core';
import { Notes } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { Note } from '@/models/entities/note';
import { name, schema } from './entities/notes';
-type NotesLog = SchemaType<typeof schema>;
-
/**
* ノートに関するチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class NotesChart extends Chart<NotesLog> {
+export default class NotesChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
- protected genNewLog(latest: NotesLog): DeepPartial<NotesLog> {
- return {
- local: {
- total: latest.local.total,
- },
- remote: {
- total: latest.remote.total,
- },
- };
- }
-
- @autobind
- protected aggregate(logs: NotesLog[]): NotesLog {
- return {
- local: {
- total: logs[0].local.total,
- inc: logs.reduce((a, b) => a + b.local.inc, 0),
- dec: logs.reduce((a, b) => a + b.local.dec, 0),
- diffs: {
- reply: logs.reduce((a, b) => a + b.local.diffs.reply, 0),
- renote: logs.reduce((a, b) => a + b.local.diffs.renote, 0),
- normal: logs.reduce((a, b) => a + b.local.diffs.normal, 0),
- },
- },
- remote: {
- total: logs[0].remote.total,
- inc: logs.reduce((a, b) => a + b.remote.inc, 0),
- dec: logs.reduce((a, b) => a + b.remote.dec, 0),
- diffs: {
- reply: logs.reduce((a, b) => a + b.remote.diffs.reply, 0),
- renote: logs.reduce((a, b) => a + b.remote.diffs.renote, 0),
- normal: logs.reduce((a, b) => a + b.remote.diffs.normal, 0),
- },
- },
- };
- }
-
- @autobind
- protected async fetchActual(): Promise<DeepPartial<NotesLog>> {
+ protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
const [localCount, remoteCount] = await Promise.all([
Notes.count({ userHost: null }),
Notes.count({ userHost: Not(IsNull()) }),
]);
return {
- local: {
- total: localCount,
- },
- remote: {
- total: remoteCount,
- },
+ 'local.total': localCount,
+ 'remote.total': remoteCount,
};
}
@autobind
public async update(note: Note, isAdditional: boolean): Promise<void> {
- const update: Obj = {
- diffs: {},
- };
-
- update.total = isAdditional ? 1 : -1;
-
- if (isAdditional) {
- update.inc = 1;
- } else {
- update.dec = 1;
- }
-
- if (note.replyId != null) {
- update.diffs.reply = isAdditional ? 1 : -1;
- } else if (note.renoteId != null) {
- update.diffs.renote = isAdditional ? 1 : -1;
- } else {
- update.diffs.normal = isAdditional ? 1 : -1;
- }
+ const prefix = note.userHost === null ? 'local' : 'remote';
- await this.inc({
- [note.userHost === null ? 'local' : 'remote']: update,
+ await this.commit({
+ [`${prefix}.total`]: isAdditional ? 1 : -1,
+ [`${prefix}.inc`]: isAdditional ? 1 : 0,
+ [`${prefix}.dec`]: isAdditional ? 0 : 1,
+ [`${prefix}.diffs.normal`]: note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0,
+ [`${prefix}.diffs.renote`]: note.renoteId != null ? (isAdditional ? 1 : -1) : 0,
+ [`${prefix}.diffs.reply`]: note.replyId != null ? (isAdditional ? 1 : -1) : 0,
});
}
}
diff --git a/packages/backend/src/services/chart/charts/per-user-drive.ts b/packages/backend/src/services/chart/charts/per-user-drive.ts
index fff790367f..969ed018f9 100644
--- a/packages/backend/src/services/chart/charts/per-user-drive.ts
+++ b/packages/backend/src/services/chart/charts/per-user-drive.ts
@@ -1,68 +1,41 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
+import Chart, { KVs } from '../core';
import { DriveFiles } from '@/models/index';
import { DriveFile } from '@/models/entities/drive-file';
import { name, schema } from './entities/per-user-drive';
-type PerUserDriveLog = SchemaType<typeof schema>;
-
/**
* ユーザーごとのドライブに関するチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class PerUserDriveChart extends Chart<PerUserDriveLog> {
+export default class PerUserDriveChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
- protected genNewLog(latest: PerUserDriveLog): DeepPartial<PerUserDriveLog> {
- return {
- totalCount: latest.totalCount,
- totalSize: latest.totalSize,
- };
- }
-
- @autobind
- protected aggregate(logs: PerUserDriveLog[]): PerUserDriveLog {
- return {
- totalCount: logs[0].totalCount,
- totalSize: logs[0].totalSize,
- incCount: logs.reduce((a, b) => a + b.incCount, 0),
- incSize: logs.reduce((a, b) => a + b.incSize, 0),
- decCount: logs.reduce((a, b) => a + b.decCount, 0),
- decSize: logs.reduce((a, b) => a + b.decSize, 0),
- };
- }
-
- @autobind
- protected async fetchActual(group: string): Promise<DeepPartial<PerUserDriveLog>> {
+ protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
const [count, size] = await Promise.all([
DriveFiles.count({ userId: group }),
DriveFiles.calcDriveUsageOf(group),
]);
return {
- totalCount: count,
- totalSize: size,
+ 'totalCount': count,
+ 'totalSize': size,
};
}
@autobind
public async update(file: DriveFile, isAdditional: boolean): Promise<void> {
- const update: Obj = {};
-
- update.totalCount = isAdditional ? 1 : -1;
- update.totalSize = isAdditional ? file.size : -file.size;
- if (isAdditional) {
- update.incCount = 1;
- update.incSize = file.size;
- } else {
- update.decCount = 1;
- update.decSize = file.size;
- }
-
- await this.inc(update, file.userId);
+ const fileSizeKb = file.size / 1000;
+ await this.commit({
+ 'totalCount': isAdditional ? 1 : -1,
+ 'totalSize': isAdditional ? fileSizeKb : -fileSizeKb,
+ 'incCount': isAdditional ? 1 : 0,
+ 'incSize': isAdditional ? fileSizeKb : 0,
+ 'decCount': isAdditional ? 0 : 1,
+ 'decSize': isAdditional ? 0 : fileSizeKb,
+ }, file.userId);
}
}
diff --git a/packages/backend/src/services/chart/charts/per-user-following.ts b/packages/backend/src/services/chart/charts/per-user-following.ts
index d0a80abdaf..cdd0aad947 100644
--- a/packages/backend/src/services/chart/charts/per-user-following.ts
+++ b/packages/backend/src/services/chart/charts/per-user-following.ts
@@ -1,76 +1,21 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
+import Chart, { KVs } from '../core';
import { Followings, Users } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { User } from '@/models/entities/user';
import { name, schema } from './entities/per-user-following';
-type PerUserFollowingLog = SchemaType<typeof schema>;
-
/**
* ユーザーごとのフォローに関するチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class PerUserFollowingChart extends Chart<PerUserFollowingLog> {
+export default class PerUserFollowingChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
- protected genNewLog(latest: PerUserFollowingLog): DeepPartial<PerUserFollowingLog> {
- return {
- local: {
- followings: {
- total: latest.local.followings.total,
- },
- followers: {
- total: latest.local.followers.total,
- },
- },
- remote: {
- followings: {
- total: latest.remote.followings.total,
- },
- followers: {
- total: latest.remote.followers.total,
- },
- },
- };
- }
-
- @autobind
- protected aggregate(logs: PerUserFollowingLog[]): PerUserFollowingLog {
- return {
- local: {
- followings: {
- total: logs[0].local.followings.total,
- inc: logs.reduce((a, b) => a + b.local.followings.inc, 0),
- dec: logs.reduce((a, b) => a + b.local.followings.dec, 0),
- },
- followers: {
- total: logs[0].local.followers.total,
- inc: logs.reduce((a, b) => a + b.local.followers.inc, 0),
- dec: logs.reduce((a, b) => a + b.local.followers.dec, 0),
- },
- },
- remote: {
- followings: {
- total: logs[0].remote.followings.total,
- inc: logs.reduce((a, b) => a + b.remote.followings.inc, 0),
- dec: logs.reduce((a, b) => a + b.remote.followings.dec, 0),
- },
- followers: {
- total: logs[0].remote.followers.total,
- inc: logs.reduce((a, b) => a + b.remote.followers.inc, 0),
- dec: logs.reduce((a, b) => a + b.remote.followers.dec, 0),
- },
- },
- };
- }
-
- @autobind
- protected async fetchActual(group: string): Promise<DeepPartial<PerUserFollowingLog>> {
+ protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
const [
localFollowingsCount,
localFollowersCount,
@@ -84,42 +29,27 @@ export default class PerUserFollowingChart extends Chart<PerUserFollowingLog> {
]);
return {
- local: {
- followings: {
- total: localFollowingsCount,
- },
- followers: {
- total: localFollowersCount,
- },
- },
- remote: {
- followings: {
- total: remoteFollowingsCount,
- },
- followers: {
- total: remoteFollowersCount,
- },
- },
+ 'local.followings.total': localFollowingsCount,
+ 'local.followers.total': localFollowersCount,
+ 'remote.followings.total': remoteFollowingsCount,
+ 'remote.followers.total': remoteFollowersCount,
};
}
@autobind
public async update(follower: { id: User['id']; host: User['host']; }, followee: { id: User['id']; host: User['host']; }, isFollow: boolean): Promise<void> {
- const update: Obj = {};
-
- update.total = isFollow ? 1 : -1;
-
- if (isFollow) {
- update.inc = 1;
- } else {
- update.dec = 1;
- }
+ const prefixFollower = Users.isLocalUser(follower) ? 'local' : 'remote';
+ const prefixFollowee = Users.isLocalUser(followee) ? 'local' : 'remote';
- this.inc({
- [Users.isLocalUser(follower) ? 'local' : 'remote']: { followings: update },
+ this.commit({
+ [`${prefixFollower}.followings.total`]: isFollow ? 1 : -1,
+ [`${prefixFollower}.followings.inc`]: isFollow ? 1 : 0,
+ [`${prefixFollower}.followings.dec`]: isFollow ? 0 : 1,
}, follower.id);
- this.inc({
- [Users.isLocalUser(followee) ? 'local' : 'remote']: { followers: update },
+ this.commit({
+ [`${prefixFollowee}.followers.total`]: isFollow ? 1 : -1,
+ [`${prefixFollowee}.followers.inc`]: isFollow ? 1 : 0,
+ [`${prefixFollowee}.followers.dec`]: isFollow ? 0 : 1,
}, followee.id);
}
}
diff --git a/packages/backend/src/services/chart/charts/per-user-notes.ts b/packages/backend/src/services/chart/charts/per-user-notes.ts
index d048c88885..ead353139b 100644
--- a/packages/backend/src/services/chart/charts/per-user-notes.ts
+++ b/packages/backend/src/services/chart/charts/per-user-notes.ts
@@ -1,45 +1,21 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
+import Chart, { KVs } from '../core';
import { User } from '@/models/entities/user';
-import { SchemaType } from '@/misc/schema';
import { Notes } from '@/models/index';
import { Note } from '@/models/entities/note';
import { name, schema } from './entities/per-user-notes';
-type PerUserNotesLog = SchemaType<typeof schema>;
-
/**
* ユーザーごとのノートに関するチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class PerUserNotesChart extends Chart<PerUserNotesLog> {
+export default class PerUserNotesChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
- protected genNewLog(latest: PerUserNotesLog): DeepPartial<PerUserNotesLog> {
- return {
- total: latest.total,
- };
- }
-
- @autobind
- protected aggregate(logs: PerUserNotesLog[]): PerUserNotesLog {
- return {
- total: logs[0].total,
- inc: logs.reduce((a, b) => a + b.inc, 0),
- dec: logs.reduce((a, b) => a + b.dec, 0),
- diffs: {
- reply: logs.reduce((a, b) => a + b.diffs.reply, 0),
- renote: logs.reduce((a, b) => a + b.diffs.renote, 0),
- normal: logs.reduce((a, b) => a + b.diffs.normal, 0),
- },
- };
- }
-
- @autobind
- protected async fetchActual(group: string): Promise<DeepPartial<PerUserNotesLog>> {
+ protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
const [count] = await Promise.all([
Notes.count({ userId: group }),
]);
@@ -51,26 +27,13 @@ export default class PerUserNotesChart extends Chart<PerUserNotesLog> {
@autobind
public async update(user: { id: User['id'] }, note: Note, isAdditional: boolean): Promise<void> {
- const update: Obj = {
- diffs: {},
- };
-
- update.total = isAdditional ? 1 : -1;
-
- if (isAdditional) {
- update.inc = 1;
- } else {
- update.dec = 1;
- }
-
- if (note.replyId != null) {
- update.diffs.reply = isAdditional ? 1 : -1;
- } else if (note.renoteId != null) {
- update.diffs.renote = isAdditional ? 1 : -1;
- } else {
- update.diffs.normal = isAdditional ? 1 : -1;
- }
-
- await this.inc(update, user.id);
+ await this.commit({
+ 'total': isAdditional ? 1 : -1,
+ 'inc': isAdditional ? 1 : 0,
+ 'dec': isAdditional ? 0 : 1,
+ 'diffs.normal': note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0,
+ 'diffs.renote': note.renoteId != null ? (isAdditional ? 1 : -1) : 0,
+ 'diffs.reply': note.replyId != null ? (isAdditional ? 1 : -1) : 0,
+ }, user.id);
}
}
diff --git a/packages/backend/src/services/chart/charts/per-user-reactions.ts b/packages/backend/src/services/chart/charts/per-user-reactions.ts
index 2f5353340d..2ec347f40a 100644
--- a/packages/backend/src/services/chart/charts/per-user-reactions.ts
+++ b/packages/backend/src/services/chart/charts/per-user-reactions.ts
@@ -1,48 +1,29 @@
import autobind from 'autobind-decorator';
-import Chart, { DeepPartial } from '../core';
+import Chart, { KVs } from '../core';
import { User } from '@/models/entities/user';
import { Note } from '@/models/entities/note';
-import { SchemaType } from '@/misc/schema';
import { Users } from '@/models/index';
import { name, schema } from './entities/per-user-reactions';
-type PerUserReactionsLog = SchemaType<typeof schema>;
-
/**
* ユーザーごとのリアクションに関するチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class PerUserReactionsChart extends Chart<PerUserReactionsLog> {
+export default class PerUserReactionsChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
- protected genNewLog(latest: PerUserReactionsLog): DeepPartial<PerUserReactionsLog> {
- return {};
- }
-
- @autobind
- protected aggregate(logs: PerUserReactionsLog[]): PerUserReactionsLog {
- return {
- local: {
- count: logs.reduce((a, b) => a + b.local.count, 0),
- },
- remote: {
- count: logs.reduce((a, b) => a + b.remote.count, 0),
- },
- };
- }
-
- @autobind
- protected async fetchActual(group: string): Promise<DeepPartial<PerUserReactionsLog>> {
+ protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async update(user: { id: User['id'], host: User['host'] }, note: Note): Promise<void> {
- this.inc({
- [Users.isLocalUser(user) ? 'local' : 'remote']: { count: 1 },
+ const prefix = Users.isLocalUser(user) ? 'local' : 'remote';
+ this.commit({
+ [`${prefix}.count`]: 1,
}, note.userId);
}
}
diff --git a/packages/backend/src/services/chart/charts/test-grouped.ts b/packages/backend/src/services/chart/charts/test-grouped.ts
index c851d2df01..5f0b1aafdc 100644
--- a/packages/backend/src/services/chart/charts/test-grouped.ts
+++ b/packages/backend/src/services/chart/charts/test-grouped.ts
@@ -1,15 +1,12 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
+import Chart, { KVs } from '../core';
import { name, schema } from './entities/test-grouped';
-type TestGroupedLog = SchemaType<typeof schema>;
-
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
-export default class TestGroupedChart extends Chart<TestGroupedLog> {
+export default class TestGroupedChart extends Chart<typeof schema> {
private total = {} as Record<string, number>;
constructor() {
@@ -17,31 +14,9 @@ export default class TestGroupedChart extends Chart<TestGroupedLog> {
}
@autobind
- protected genNewLog(latest: TestGroupedLog): DeepPartial<TestGroupedLog> {
- return {
- foo: {
- total: latest.foo.total,
- },
- };
- }
-
- @autobind
- protected aggregate(logs: TestGroupedLog[]): TestGroupedLog {
+ protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
return {
- foo: {
- total: logs[0].foo.total,
- inc: logs.reduce((a, b) => a + b.foo.inc, 0),
- dec: logs.reduce((a, b) => a + b.foo.dec, 0),
- },
- };
- }
-
- @autobind
- protected async fetchActual(group: string): Promise<DeepPartial<TestGroupedLog>> {
- return {
- foo: {
- total: this.total[group],
- },
+ 'foo.total': this.total[group],
};
}
@@ -49,14 +24,11 @@ export default class TestGroupedChart extends Chart<TestGroupedLog> {
public async increment(group: string): Promise<void> {
if (this.total[group] == null) this.total[group] = 0;
- const update: Obj = {};
-
- update.total = 1;
- update.inc = 1;
this.total[group]++;
- await this.inc({
- foo: update,
+ await this.commit({
+ 'foo.total': 1,
+ 'foo.inc': 1,
}, group);
}
}
diff --git a/packages/backend/src/services/chart/charts/test-unique.ts b/packages/backend/src/services/chart/charts/test-unique.ts
index 3564f675ad..e67036acef 100644
--- a/packages/backend/src/services/chart/charts/test-unique.ts
+++ b/packages/backend/src/services/chart/charts/test-unique.ts
@@ -1,39 +1,24 @@
import autobind from 'autobind-decorator';
-import Chart, { DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
+import Chart, { KVs } from '../core';
import { name, schema } from './entities/test-unique';
-type TestUniqueLog = SchemaType<typeof schema>;
-
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
-export default class TestUniqueChart extends Chart<TestUniqueLog> {
+export default class TestUniqueChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
- protected genNewLog(latest: TestUniqueLog): DeepPartial<TestUniqueLog> {
- return {};
- }
-
- @autobind
- protected aggregate(logs: TestUniqueLog[]): TestUniqueLog {
- return {
- foo: logs.reduce((a, b) => a.concat(b.foo), [] as TestUniqueLog['foo']),
- };
- }
-
- @autobind
- protected async fetchActual(): Promise<DeepPartial<TestUniqueLog>> {
+ protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async uniqueIncrement(key: string): Promise<void> {
- await this.inc({
+ await this.commit({
foo: [key],
});
}
diff --git a/packages/backend/src/services/chart/charts/test.ts b/packages/backend/src/services/chart/charts/test.ts
index 06add7ede9..878acd51be 100644
--- a/packages/backend/src/services/chart/charts/test.ts
+++ b/packages/backend/src/services/chart/charts/test.ts
@@ -1,15 +1,12 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
+import Chart, { KVs } from '../core';
import { name, schema } from './entities/test';
-type TestLog = SchemaType<typeof schema>;
-
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
-export default class TestChart extends Chart<TestLog> {
+export default class TestChart extends Chart<typeof schema> {
public total = 0; // publicにするのはテストのため
constructor() {
@@ -17,57 +14,29 @@ export default class TestChart extends Chart<TestLog> {
}
@autobind
- protected genNewLog(latest: TestLog): DeepPartial<TestLog> {
+ protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {
- foo: {
- total: latest.foo.total,
- },
- };
- }
-
- @autobind
- protected aggregate(logs: TestLog[]): TestLog {
- return {
- foo: {
- total: logs[0].foo.total,
- inc: logs.reduce((a, b) => a + b.foo.inc, 0),
- dec: logs.reduce((a, b) => a + b.foo.dec, 0),
- },
- };
- }
-
- @autobind
- protected async fetchActual(): Promise<DeepPartial<TestLog>> {
- return {
- foo: {
- total: this.total,
- },
+ 'foo.total': this.total,
};
}
@autobind
public async increment(): Promise<void> {
- const update: Obj = {};
-
- update.total = 1;
- update.inc = 1;
this.total++;
- await this.inc({
- foo: update,
+ await this.commit({
+ 'foo.total': 1,
+ 'foo.inc': 1,
});
}
@autobind
public async decrement(): Promise<void> {
- const update: Obj = {};
-
- update.total = -1;
- update.dec = 1;
this.total--;
- await this.inc({
- foo: update,
+ await this.commit({
+ 'foo.total': -1,
+ 'foo.dec': 1,
});
}
}
diff --git a/packages/backend/src/services/chart/charts/users.ts b/packages/backend/src/services/chart/charts/users.ts
index c36c6cd979..50fca3a8d6 100644
--- a/packages/backend/src/services/chart/charts/users.ts
+++ b/packages/backend/src/services/chart/charts/users.ts
@@ -1,80 +1,40 @@
import autobind from 'autobind-decorator';
-import Chart, { Obj, DeepPartial } from '../core';
-import { SchemaType } from '@/misc/schema';
+import Chart, { KVs } from '../core';
import { Users } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { User } from '@/models/entities/user';
import { name, schema } from './entities/users';
-type UsersLog = SchemaType<typeof schema>;
-
/**
* ユーザー数に関するチャート
*/
// eslint-disable-next-line import/no-default-export
-export default class UsersChart extends Chart<UsersLog> {
+export default class UsersChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
- protected genNewLog(latest: UsersLog): DeepPartial<UsersLog> {
- return {
- local: {
- total: latest.local.total,
- },
- remote: {
- total: latest.remote.total,
- },
- };
- }
-
- @autobind
- protected aggregate(logs: UsersLog[]): UsersLog {
- return {
- local: {
- total: logs[0].local.total,
- inc: logs.reduce((a, b) => a + b.local.inc, 0),
- dec: logs.reduce((a, b) => a + b.local.dec, 0),
- },
- remote: {
- total: logs[0].remote.total,
- inc: logs.reduce((a, b) => a + b.remote.inc, 0),
- dec: logs.reduce((a, b) => a + b.remote.dec, 0),
- },
- };
- }
-
- @autobind
- protected async fetchActual(): Promise<DeepPartial<UsersLog>> {
+ protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
const [localCount, remoteCount] = await Promise.all([
Users.count({ host: null }),
Users.count({ host: Not(IsNull()) }),
]);
return {
- local: {
- total: localCount,
- },
- remote: {
- total: remoteCount,
- },
+ 'local.total': localCount,
+ 'remote.total': remoteCount,
};
}
@autobind
public async update(user: { id: User['id'], host: User['host'] }, isAdditional: boolean): Promise<void> {
- const update: Obj = {};
-
- update.total = isAdditional ? 1 : -1;
- if (isAdditional) {
- update.inc = 1;
- } else {
- update.dec = 1;
- }
+ const prefix = Users.isLocalUser(user) ? 'local' : 'remote';
- await this.inc({
- [Users.isLocalUser(user) ? 'local' : 'remote']: update,
+ await this.commit({
+ [`${prefix}.total`]: isAdditional ? 1 : -1,
+ [`${prefix}.inc`]: isAdditional ? 1 : 0,
+ [`${prefix}.dec`]: isAdditional ? 0 : 1,
});
}
}
diff --git a/packages/backend/src/services/chart/core.ts b/packages/backend/src/services/chart/core.ts
index e406449f4f..0bc3d1a4c0 100644
--- a/packages/backend/src/services/chart/core.ts
+++ b/packages/backend/src/services/chart/core.ts
@@ -7,24 +7,19 @@
import * as nestedProperty from 'nested-property';
import autobind from 'autobind-decorator';
import Logger from '../logger';
-import { Schema } from '@/misc/schema';
import { EntitySchema, getRepository, Repository, LessThan, Between } from 'typeorm';
import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from '@/prelude/time';
import { getChartInsertLock } from '@/misc/app-lock';
const logger = new Logger('chart', 'white', process.env.NODE_ENV !== 'test');
-export type Obj = { [key: string]: any };
+const columnPrefix = '___' as const;
+const uniqueTempColumnPrefix = 'unique_temp___' as const;
+const columnDot = '_' as const;
-export type DeepPartial<T> = {
- [P in keyof T]?: DeepPartial<T[P]>;
-};
-
-type ArrayValue<T> = {
- [P in keyof T]: T[P] extends number ? T[P][] : ArrayValue<T[P]>;
-};
+type KeyToColumnName<T extends string> = T extends `${infer R1}.${infer R2}` ? `${R1}${typeof columnDot}${KeyToColumnName<R2>}` : T;
-type Log = {
+type RawRecord<S extends Schema> = {
id: number;
/**
@@ -36,6 +31,10 @@ type Log = {
* 集計日時のUnixタイムスタンプ(秒)
*/
date: number;
+} & {
+ [K in keyof S as `${typeof uniqueTempColumnPrefix}${KeyToColumnName<string & K>}`]: S[K]['uniqueIncrement'] extends true ? string[] : never;
+} & {
+ [K in keyof S as `${typeof columnPrefix}${KeyToColumnName<string & K>}`]: number;
};
const camelToSnake = (str: string): string => {
@@ -44,123 +43,72 @@ const camelToSnake = (str: string): string => {
const removeDuplicates = (array: any[]) => Array.from(new Set(array));
+type Schema = Record<string, {
+ uniqueIncrement?: boolean;
+
+ range?: 'big' | 'small' | 'medium';
+
+ // previousな値を引き継ぐかどうか
+ accumulate?: boolean;
+}>;
+
+type Commit<S extends Schema> = {
+ [K in keyof S]?: S[K]['uniqueIncrement'] extends true ? string[] : number;
+};
+
+export type KVs<S extends Schema> = {
+ [K in keyof S]: number;
+};
+
+type ChartResult<T extends Schema> = {
+ [P in keyof T]: number[];
+};
+
/**
* 様々なチャートの管理を司るクラス
*/
// eslint-disable-next-line import/no-default-export
-export default abstract class Chart<T extends Record<string, any>> {
- private static readonly columnPrefix = '___';
- private static readonly columnDot = '_';
+export default abstract class Chart<T extends Schema> {
+ public schema: T;
private name: string;
private buffer: {
- diff: DeepPartial<T>;
+ diff: Commit<T>;
group: string | null;
}[] = [];
- public schema: Schema;
- protected repositoryForHour: Repository<Log>;
- protected repositoryForDay: Repository<Log>;
-
- protected abstract genNewLog(latest: T): DeepPartial<T>;
-
- /**
- * @param logs 日時が新しい方が先頭
- */
- protected abstract aggregate(logs: T[]): T;
+ protected repositoryForHour: Repository<RawRecord<T>>;
+ protected repositoryForDay: Repository<RawRecord<T>>;
- protected abstract fetchActual(group: string | null): Promise<DeepPartial<T>>;
+ protected abstract queryCurrentState(group: string | null): Promise<Partial<KVs<T>>>;
@autobind
- private static convertSchemaToFlatColumnDefinitions(schema: Schema) {
- const columns = {} as Record<string, unknown>;
- const flatColumns = (x: Obj, path?: string) => {
- for (const [k, v] of Object.entries(x)) {
- const p = path ? `${path}${this.columnDot}${k}` : k;
- if (v.type === 'object') {
- flatColumns(v.properties, p);
- } else if (v.type === 'number') {
- columns[this.columnPrefix + p] = {
- type: 'bigint',
- };
- } else if (v.type === 'array' && v.items.type === 'string') {
- columns[this.columnPrefix + p] = {
- type: 'varchar',
- array: true,
- };
- }
+ private static convertSchemaToColumnDefinitions(schema: Schema): Record<string, { type: string; array?: boolean; default?: any; }> {
+ const columns = {} as Record<string, { type: string; array?: boolean; default?: any; }>;
+ for (const [k, v] of Object.entries(schema)) {
+ const name = k.replaceAll('.', columnDot);
+ const type = v.range === 'big' ? 'bigint' : v.range === 'small' ? 'smallint' : 'integer';
+ if (v.uniqueIncrement) {
+ columns[uniqueTempColumnPrefix + name] = {
+ type: 'varchar',
+ array: true,
+ default: '{}',
+ };
+ columns[columnPrefix + name] = {
+ type,
+ default: 0,
+ };
+ } else {
+ columns[columnPrefix + name] = {
+ type,
+ default: 0,
+ };
}
- };
- flatColumns(schema.properties!);
- return columns;
- }
-
- @autobind
- private static convertFlattenColumnsToObject(x: Record<string, unknown>): Record<string, unknown> {
- const obj = {} as Record<string, unknown>;
- for (const k of Object.keys(x).filter(k => k.startsWith(Chart.columnPrefix))) {
- // now k is ___x_y_z
- const path = k.substr(Chart.columnPrefix.length).split(Chart.columnDot).join('.');
- nestedProperty.set(obj, path, x[k]);
}
- return obj;
- }
-
- @autobind
- private static convertObjectToFlattenColumns(x: Record<string, unknown>) {
- const columns = {} as Record<string, number | unknown[]>;
- const flatten = (x: Obj, path?: string) => {
- for (const [k, v] of Object.entries(x)) {
- const p = path ? `${path}${this.columnDot}${k}` : k;
- if (typeof v === 'object' && !Array.isArray(v)) {
- flatten(v, p);
- } else {
- columns[this.columnPrefix + p] = v;
- }
- }
- };
- flatten(x);
return columns;
}
@autobind
- private static countUniqueFields(x: Record<string, unknown>) {
- const exec = (x: Obj) => {
- const res = {} as Record<string, unknown>;
- for (const [k, v] of Object.entries(x)) {
- if (typeof v === 'object' && !Array.isArray(v)) {
- res[k] = exec(v);
- } else if (Array.isArray(v)) {
- res[k] = Array.from(new Set(v)).length;
- } else {
- res[k] = v;
- }
- }
- return res;
- };
- return exec(x);
- }
-
- @autobind
- private static convertQuery(diff: Record<string, number | unknown[]>) {
- const query: Record<string, () => string> = {};
-
- for (const [k, v] of Object.entries(diff)) {
- if (typeof v === 'number') {
- if (v > 0) query[k] = () => `"${k}" + ${v}`;
- if (v < 0) query[k] = () => `"${k}" - ${Math.abs(v)}`;
- } else if (Array.isArray(v)) {
- // TODO: item が文字列以外の場合も対応
- // TODO: item をSQLエスケープ
- const items = v.map(item => `"${item}"`).join(',');
- query[k] = () => `array_cat("${k}", '{${items}}'::varchar[])`;
- }
- }
-
- return query;
- }
-
- @autobind
- private static dateToTimestamp(x: Date): Log['date'] {
+ private static dateToTimestamp(x: Date): number {
return Math.floor(x.getTime() / 1000);
}
@@ -207,7 +155,7 @@ export default abstract class Chart<T extends Record<string, any>> {
length: 128,
},
} : {}),
- ...Chart.convertSchemaToFlatColumnDefinitions(schema),
+ ...Chart.convertSchemaToColumnDefinitions(schema),
},
indices: [{
columns: grouped ? ['date', 'group'] : ['date'],
@@ -233,37 +181,39 @@ export default abstract class Chart<T extends Record<string, any>> {
};
}
- constructor(name: string, schema: Schema, grouped = false) {
+ constructor(name: string, schema: T, grouped = false) {
this.name = name;
this.schema = schema;
const { hour, day } = Chart.schemaToEntity(name, schema, grouped);
- this.repositoryForHour = getRepository<Log>(hour);
- this.repositoryForDay = getRepository<Log>(day);
+ this.repositoryForHour = getRepository<RawRecord<T>>(hour);
+ this.repositoryForDay = getRepository<RawRecord<T>>(day);
}
@autobind
- private getNewLog(latest: T | null): T {
- const log = latest ? this.genNewLog(latest) : {};
- const flatColumns = (x: Obj, path?: string) => {
- for (const [k, v] of Object.entries(x)) {
- const p = path ? `${path}.${k}` : k;
- if (v.type === 'object') {
- flatColumns(v.properties, p);
- } else {
- if (nestedProperty.get(log, p) == null) {
- const emptyValue = v.type === 'number' ? 0 : [];
- nestedProperty.set(log, p, emptyValue);
- }
- }
+ private convertRawRecord(x: RawRecord<T>): KVs<T> {
+ const kvs = {} as KVs<T>;
+ for (const k of Object.keys(x).filter(k => k.startsWith(columnPrefix))) {
+ kvs[k.substr(columnPrefix.length).split(columnDot).join('.')] = x[k];
+ }
+ return kvs;
+ }
+
+ @autobind
+ private getNewLog(latest: KVs<T> | null): KVs<T> {
+ const log = {} as Record<keyof T, number>;
+ for (const [k, v] of Object.entries(this.schema)) {
+ if (v.accumulate && latest) {
+ log[k] = latest[k];
+ } else {
+ log[k] = 0;
}
- };
- flatColumns(this.schema.properties!);
- return log as T;
+ }
+ return log as KVs<T>;
}
@autobind
- private getLatestLog(group: string | null, span: 'hour' | 'day'): Promise<Log | null> {
+ private getLatestLog(group: string | null, span: 'hour' | 'day'): Promise<RawRecord<T> | null> {
const repository =
span === 'hour' ? this.repositoryForHour :
span === 'day' ? this.repositoryForDay :
@@ -282,7 +232,7 @@ export default abstract class Chart<T extends Record<string, any>> {
* 現在(=今のHour or Day)のログをデータベースから探して、あればそれを返し、なければ作成して返します。
*/
@autobind
- private async claimCurrentLog(group: string | null, span: 'hour' | 'day'): Promise<Log> {
+ private async claimCurrentLog(group: string | null, span: 'hour' | 'day'): Promise<RawRecord<T>> {
const [y, m, d, h] = Chart.getCurrentDate();
const current = dateUTC(
@@ -306,8 +256,8 @@ export default abstract class Chart<T extends Record<string, any>> {
return currentLog;
}
- let log: Log;
- let data: T;
+ let log: RawRecord<T>;
+ let data: KVs<T>;
// 集計期間が変わってから、初めてのチャート更新なら
// 最も最近のログを持ってくる
@@ -318,10 +268,8 @@ export default abstract class Chart<T extends Record<string, any>> {
const latest = await this.getLatestLog(group, span);
if (latest != null) {
- const obj = Chart.convertFlattenColumnsToObject(latest) as T;
-
// 空ログデータを作成
- data = this.getNewLog(obj);
+ data = this.getNewLog(this.convertRawRecord(latest));
} else {
// ログが存在しなかったら
// (Misskeyインスタンスを建てて初めてのチャート更新時など)
@@ -346,11 +294,17 @@ export default abstract class Chart<T extends Record<string, any>> {
// ログがあればそれを返して終了
if (currentLog != null) return currentLog;
+ const columns = {} as Record<string, number | unknown[]>;
+ for (const [k, v] of Object.entries(data)) {
+ const name = k.replaceAll('.', columnDot);
+ columns[columnPrefix + name] = v;
+ }
+
// 新規ログ挿入
log = await repository.insert({
date: date,
...(group ? { group: group } : {}),
- ...Chart.convertObjectToFlattenColumns(data),
+ ...columns,
}).then(x => repository.findOneOrFail(x.identifiers[0]));
logger.info(`${this.name + (group ? `:${group}` : '')}(${span}): New commit created`);
@@ -362,7 +316,10 @@ export default abstract class Chart<T extends Record<string, any>> {
}
@autobind
- protected commit(diff: DeepPartial<T>, group: string | null = null): void {
+ protected commit(diff: Commit<T>, group: string | null = null): void {
+ for (const [k, v] of Object.entries(diff)) {
+ if (v == null || v === 0 || (Array.isArray(v) && v.length === 0)) delete diff[k];
+ }
this.buffer.push({
diff, group,
});
@@ -381,13 +338,11 @@ export default abstract class Chart<T extends Record<string, any>> {
// そのログは本来は 01:00~ のログとしてDBに保存されて欲しいのに、02:00~ のログ扱いになってしまう。
// これを回避するための実装は複雑になりそうなため、一旦保留。
- const update = async (logHour: Log, logDay: Log): Promise<void> => {
+ const update = async (logHour: RawRecord<T>, logDay: RawRecord<T>): Promise<void> => {
const finalDiffs = {} as Record<string, number | unknown[]>;
for (const diff of this.buffer.filter(q => q.group == null || (q.group === logHour.group)).map(q => q.diff)) {
- const columns = Chart.convertObjectToFlattenColumns(diff);
-
- for (const [k, v] of Object.entries(columns)) {
+ for (const [k, v] of Object.entries(diff)) {
if (finalDiffs[k] == null) {
finalDiffs[k] = v;
} else {
@@ -400,18 +355,45 @@ export default abstract class Chart<T extends Record<string, any>> {
}
}
- const query = Chart.convertQuery(finalDiffs);
+ const queryForHour: Record<string, number | (() => string)> = {};
+ const queryForDay: Record<string, number | (() => string)> = {};
+ for (const [k, v] of Object.entries(finalDiffs)) {
+ if (typeof v === 'number') {
+ const name = columnPrefix + k.replaceAll('.', columnDot);
+ if (v > 0) queryForHour[name] = () => `"${name}" + ${v}`;
+ if (v < 0) queryForHour[name] = () => `"${name}" - ${Math.abs(v)}`;
+ if (v > 0) queryForDay[name] = () => `"${name}" + ${v}`;
+ if (v < 0) queryForDay[name] = () => `"${name}" - ${Math.abs(v)}`;
+ } else if (Array.isArray(v) && v.length > 0) { // ユニークインクリメント
+ const name = uniqueTempColumnPrefix + k.replaceAll('.', columnDot);
+ // TODO: item が文字列以外の場合も対応
+ // TODO: item をSQLエスケープ
+ // TODO: 値が重複しないようにしたい
+ const items = v.map(item => `"${item}"`).join(',');
+ queryForHour[name] = () => `array_cat("${name}", '{${items}}'::varchar[])`;
+ queryForDay[name] = () => `array_cat("${name}", '{${items}}'::varchar[])`;
+ }
+ }
+
+ for (const [k, v] of Object.entries(this.schema)) {
+ const name = columnPrefix + k.replaceAll('.', columnDot);
+ if (v.uniqueIncrement) {
+ const tempColumnName = uniqueTempColumnPrefix + k.replaceAll('.', columnDot);
+ queryForHour[name] = new Set([...finalDiffs[k], ...logHour[tempColumnName]]).size;
+ queryForDay[name] = new Set([...finalDiffs[k], ...logDay[tempColumnName]]).size;
+ }
+ }
// ログ更新
await Promise.all([
this.repositoryForHour.createQueryBuilder()
.update()
- .set(query)
+ .set(queryForHour)
.where('id = :id', { id: logHour.id })
.execute(),
this.repositoryForDay.createQueryBuilder()
.update()
- .set(query)
+ .set(queryForDay)
.where('id = :id', { id: logDay.id })
.execute(),
]);
@@ -435,18 +417,24 @@ export default abstract class Chart<T extends Record<string, any>> {
@autobind
public async resync(group: string | null = null): Promise<void> {
- const data = await this.fetchActual(group);
+ const data = await this.queryCurrentState(group);
- const update = async (logHour: Log, logDay: Log): Promise<void> => {
+ const columns = {} as Record<string, number>;
+ for (const [k, v] of Object.entries(data)) {
+ const name = k.replaceAll('.', columnDot);
+ columns[columnPrefix + name] = v;
+ }
+
+ const update = async (logHour: RawRecord<T>, logDay: RawRecord<T>): Promise<void> => {
await Promise.all([
this.repositoryForHour.createQueryBuilder()
.update()
- .set(Chart.convertObjectToFlattenColumns(data))
+ .set(columns as any)
.where('id = :id', { id: logHour.id })
.execute(),
this.repositoryForDay.createQueryBuilder()
.update()
- .set(Chart.convertObjectToFlattenColumns(data))
+ .set(columns as any)
.where('id = :id', { id: logDay.id })
.execute(),
]);
@@ -460,12 +448,39 @@ export default abstract class Chart<T extends Record<string, any>> {
}
@autobind
- protected async inc(inc: DeepPartial<T>, group: string | null = null): Promise<void> {
- await this.commit(inc, group);
+ public async clean(): Promise<void> {
+ const current = dateUTC(Chart.getCurrentDate());
+
+ // 一日以上前かつ三日以内
+ const gt = Chart.dateToTimestamp(current) - (1000 * 60 * 60 * 24 * 3);
+ const lt = Chart.dateToTimestamp(current) - (1000 * 60 * 60 * 24);
+
+ const columns = {} as Record<string, number>;
+ for (const [k, v] of Object.entries(this.schema)) {
+ if (v.uniqueIncrement) {
+ const name = k.replaceAll('.', columnDot);
+ columns[uniqueTempColumnPrefix + name] = [];
+ }
+ }
+
+ await Promise.all([
+ this.repositoryForHour.createQueryBuilder()
+ .update()
+ .set(columns as any)
+ .where('date > :gt', { gt })
+ .andWhere('date < :lt', { lt })
+ .execute(),
+ this.repositoryForDay.createQueryBuilder()
+ .update()
+ .set(columns as any)
+ .where('date > :gt', { gt })
+ .andWhere('date < :lt', { lt })
+ .execute(),
+ ]);
}
@autobind
- public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<ArrayValue<T>> {
+ public async getChartRaw(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<ChartResult<T>> {
const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate();
const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [] as never;
@@ -526,7 +541,7 @@ export default abstract class Chart<T extends Record<string, any>> {
}
}
- const chart: T[] = [];
+ const chart: KVs<T>[] = [];
for (let i = (amount - 1); i >= 0; i--) {
const current =
@@ -537,17 +552,16 @@ export default abstract class Chart<T extends Record<string, any>> {
const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current));
if (log) {
- const data = Chart.convertFlattenColumnsToObject(log);
- chart.unshift(Chart.countUniqueFields(data) as T);
+ chart.unshift(this.convertRawRecord(log));
} else {
// 隙間埋め
const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current));
- const data = latest ? Chart.convertFlattenColumnsToObject(latest) as T : null;
- chart.unshift(Chart.countUniqueFields(this.getNewLog(data)) as T);
+ const data = latest ? this.convertRawRecord(latest) : null;
+ chart.unshift(this.getNewLog(data));
}
}
- const res = {} as Record<string, unknown>;
+ const res = {} as ChartResult<T>;
/**
* [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }]
@@ -555,36 +569,26 @@ export default abstract class Chart<T extends Record<string, any>> {
* { foo: [1, 2, 3], bar: [5, 6, 7] }
* にする
*/
- const compact = (x: Obj, path?: string): void => {
- for (const [k, v] of Object.entries(x)) {
- const p = path ? `${path}.${k}` : k;
- if (typeof v === 'object' && !Array.isArray(v)) {
- compact(v, p);
+ for (const record of chart) {
+ for (const [k, v] of Object.entries(record)) {
+ if (res[k]) {
+ res[k].push(v);
} else {
- const values = chart.map(s => nestedProperty.get(s, p));
- nestedProperty.set(res, p, values);
+ res[k] = [v];
}
}
- };
-
- compact(chart[0]);
+ }
- return res as ArrayValue<T>;
+ return res;
}
-}
-export function convertLog(logSchema: Schema): Schema {
- const v: Schema = JSON.parse(JSON.stringify(logSchema)); // copy
- if (v.type === 'number') {
- v.type = 'array';
- v.items = {
- type: 'number' as const,
- optional: false as const, nullable: false as const,
- };
- } else if (v.type === 'object') {
- for (const k of Object.keys(v.properties!)) {
- v.properties![k] = convertLog(v.properties![k]);
+ @autobind
+ public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<Record<string, unknown>> {
+ const result = await this.getChartRaw(span, amount, cursor, group);
+ const object = {};
+ for (const [k, v] of Object.entries(result)) {
+ nestedProperty.set(object, k, v);
}
+ return object;
}
- return v;
}
diff --git a/packages/backend/src/services/chart/entities.ts b/packages/backend/src/services/chart/entities.ts
index dedbd47080..569b328557 100644
--- a/packages/backend/src/services/chart/entities.ts
+++ b/packages/backend/src/services/chart/entities.ts
@@ -1,7 +1,6 @@
import { entity as FederationChart } from './charts/entities/federation';
import { entity as NotesChart } from './charts/entities/notes';
import { entity as UsersChart } from './charts/entities/users';
-import { entity as NetworkChart } from './charts/entities/network';
import { entity as ActiveUsersChart } from './charts/entities/active-users';
import { entity as InstanceChart } from './charts/entities/instance';
import { entity as PerUserNotesChart } from './charts/entities/per-user-notes';
@@ -10,12 +9,12 @@ import { entity as PerUserReactionsChart } from './charts/entities/per-user-reac
import { entity as HashtagChart } from './charts/entities/hashtag';
import { entity as PerUserFollowingChart } from './charts/entities/per-user-following';
import { entity as PerUserDriveChart } from './charts/entities/per-user-drive';
+import { entity as ApRequestChart } from './charts/entities/ap-request';
export const entities = [
FederationChart.hour, FederationChart.day,
NotesChart.hour, NotesChart.day,
UsersChart.hour, UsersChart.day,
- NetworkChart.hour, NetworkChart.day,
ActiveUsersChart.hour, ActiveUsersChart.day,
InstanceChart.hour, InstanceChart.day,
PerUserNotesChart.hour, PerUserNotesChart.day,
@@ -24,4 +23,5 @@ export const entities = [
HashtagChart.hour, HashtagChart.day,
PerUserFollowingChart.hour, PerUserFollowingChart.day,
PerUserDriveChart.hour, PerUserDriveChart.day,
+ ApRequestChart.hour, ApRequestChart.day,
];
diff --git a/packages/backend/src/services/chart/index.ts b/packages/backend/src/services/chart/index.ts
index 0b9887b36f..1c0f7aadc1 100644
--- a/packages/backend/src/services/chart/index.ts
+++ b/packages/backend/src/services/chart/index.ts
@@ -3,7 +3,6 @@ import { beforeShutdown } from '@/misc/before-shutdown';
import FederationChart from './charts/federation';
import NotesChart from './charts/notes';
import UsersChart from './charts/users';
-import NetworkChart from './charts/network';
import ActiveUsersChart from './charts/active-users';
import InstanceChart from './charts/instance';
import PerUserNotesChart from './charts/per-user-notes';
@@ -12,11 +11,11 @@ import PerUserReactionsChart from './charts/per-user-reactions';
import HashtagChart from './charts/hashtag';
import PerUserFollowingChart from './charts/per-user-following';
import PerUserDriveChart from './charts/per-user-drive';
+import ApRequestChart from './charts/ap-request';
export const federationChart = new FederationChart();
export const notesChart = new NotesChart();
export const usersChart = new UsersChart();
-export const networkChart = new NetworkChart();
export const activeUsersChart = new ActiveUsersChart();
export const instanceChart = new InstanceChart();
export const perUserNotesChart = new PerUserNotesChart();
@@ -25,12 +24,12 @@ export const perUserReactionsChart = new PerUserReactionsChart();
export const hashtagChart = new HashtagChart();
export const perUserFollowingChart = new PerUserFollowingChart();
export const perUserDriveChart = new PerUserDriveChart();
+export const apRequestChart = new ApRequestChart();
const charts = [
federationChart,
notesChart,
usersChart,
- networkChart,
activeUsersChart,
instanceChart,
perUserNotesChart,
@@ -39,6 +38,7 @@ const charts = [
hashtagChart,
perUserFollowingChart,
perUserDriveChart,
+ apRequestChart,
];
// 20分おきにメモリ情報をDBに書き込み