Improve chart performance (#7360)

* wip

* wip

* wip

* wip

* wip

* Update chart.ts

* wip

* Improve server performance

* wip

* wip

1 files changed, 165 insertions, 111 deletions

diff --git a/src/services/chart/core.ts b/src/services/chart/core.ts
index dc09923ae4..10621be073 100644
--- a/src/services/chart/core.ts
+++ b/src/services/chart/core.ts
@@ -24,8 +24,6 @@ type ArrayValue<T> = {
 	[P in keyof T]: T[P] extends number ? T[P][] : ArrayValue<T[P]>;
 };
 
-type Span = 'day' | 'hour';
-
 type Log = {
 	id: number;
 
@@ -38,22 +36,14 @@ type Log = {
 	 * 集計日時のUnixタイムスタンプ(秒)
 	 */
 	date: number;
-
-	/**
-	 * 集計期間
-	 */
-	span: Span;
-
-	/**
-	 * ユニークインクリメント用
-	 */
-	unique?: Record<string, any>;
 };
 
 const camelToSnake = (str: string) => {
 	return str.replace(/([A-Z])/g, s => '_' + s.charAt(0).toLowerCase());
 };
 
+const removeDuplicates = (array: any[]) => Array.from(new Set(array));
+
 /**
  * 様々なチャートの管理を司るクラス
  */
@@ -62,10 +52,21 @@ export default abstract class Chart<T extends Record<string, any>> {
 	private static readonly columnDot = '_';
 
 	private name: string;
+	private queue: {
+		diff: DeepPartial<T>;
+		group: string | null;
+	}[] = [];
 	public schema: Schema;
 	protected repository: Repository<Log>;
+
 	protected abstract genNewLog(latest: T): DeepPartial<T>;
-	protected abstract async fetchActual(group: string | null): Promise<DeepPartial<T>>;
+
+	/**
+	 * @param logs 日時が新しい方が先頭
+	 */
+	protected abstract aggregate(logs: T[]): T;
+
+	protected abstract fetchActual(group: string | null): Promise<DeepPartial<T>>;
 
 	@autobind
 	private static convertSchemaToFlatColumnDefinitions(schema: Schema) {
@@ -75,10 +76,15 @@ export default abstract class Chart<T extends Record<string, any>> {
 				const p = path ? `${path}${this.columnDot}${k}` : k;
 				if (v.type === 'object') {
 					flatColumns(v.properties, p);
-				} else {
+				} else if (v.type === 'number') {
 					columns[this.columnPrefix + p] = {
 						type: 'bigint',
 					};
+				} else if (v.type === 'array' && v.items.type === 'string') {
+					columns[this.columnPrefix + p] = {
+						type: 'varchar',
+						array: true,
+					};
 				}
 			}
 		};
@@ -99,11 +105,11 @@ export default abstract class Chart<T extends Record<string, any>> {
 
 	@autobind
 	private static convertObjectToFlattenColumns(x: Record<string, any>) {
-		const columns = {} as Record<string, number>;
+		const columns = {} as Record<string, number | unknown[]>;
 		const flatten = (x: Obj, path?: string) => {
 			for (const [k, v] of Object.entries(x)) {
 				const p = path ? `${path}${this.columnDot}${k}` : k;
-				if (typeof v === 'object') {
+				if (typeof v === 'object' && !Array.isArray(v)) {
 					flatten(v, p);
 				} else {
 					columns[this.columnPrefix + p] = v;
@@ -115,14 +121,37 @@ export default abstract class Chart<T extends Record<string, any>> {
 	}
 
 	@autobind
-	private static convertQuery(x: Record<string, any>) {
-		const query: Record<string, Function> = {};
+	private static countUniqueFields(x: Record<string, any>) {
+		const exec = (x: Obj) => {
+			const res = {} as Record<string, any>;
+			for (const [k, v] of Object.entries(x)) {
+				if (typeof v === 'object' && !Array.isArray(v)) {
+					res[k] = exec(v);
+				} else if (Array.isArray(v)) {
+					res[k] = Array.from(new Set(v)).length;
+				} else {
+					res[k] = v;
+				}
+			}
+			return res;
+		};
+		return exec(x);
+	}
 
-		const columns = Chart.convertObjectToFlattenColumns(x);
+	@autobind
+	private static convertQuery(diff: Record<string, number | unknown[]>) {
+		const query: Record<string, Function> = {};
 
-		for (const [k, v] of Object.entries(columns)) {
-			if (v > 0) query[k] = () => `"${k}" + ${v}`;
-			if (v < 0) query[k] = () => `"${k}" - ${Math.abs(v)}`;
+		for (const [k, v] of Object.entries(diff)) {
+			if (typeof v === 'number') {
+				if (v > 0) query[k] = () => `"${k}" + ${v}`;
+				if (v < 0) query[k] = () => `"${k}" - ${Math.abs(v)}`;
+			} else if (Array.isArray(v)) {
+				// TODO: item が文字列以外の場合も対応
+				// TODO: item をSQLエスケープ
+				const items = v.map(item => `"${item}"`).join(',');
+				query[k] = () => `array_cat("${k}", '{${items}}'::varchar[])`;
+			}
 		}
 
 		return query;
@@ -169,28 +198,14 @@ export default abstract class Chart<T extends Record<string, any>> {
 					length: 128,
 					nullable: true
 				},
-				span: {
-					type: 'enum',
-					enum: ['hour', 'day']
-				},
-				unique: {
-					type: 'jsonb',
-					default: {}
-				},
 				...Chart.convertSchemaToFlatColumnDefinitions(schema)
 			},
 			indices: [{
 				columns: ['date']
 			}, {
-				columns: ['span']
-			}, {
 				columns: ['group']
 			}, {
-				columns: ['span', 'date']
-			}, {
 				columns: ['date', 'group']
-			}, {
-				columns: ['span', 'date', 'group']
 			}]
 		});
 	}
@@ -200,7 +215,7 @@ export default abstract class Chart<T extends Record<string, any>> {
 		this.schema = schema;
 		const entity = Chart.schemaToEntity(name, schema);
 
-		const keys = ['span', 'date'];
+		const keys = ['date'];
 		if (grouped) keys.push('group');
 
 		entity.options.uniques = [{
@@ -220,7 +235,8 @@ export default abstract class Chart<T extends Record<string, any>> {
 					flatColumns(v.properties, p);
 				} else {
 					if (nestedProperty.get(log, p) == null) {
-						nestedProperty.set(log, p, 0);
+						const emptyValue = v.type === 'number' ? 0 : [];
+						nestedProperty.set(log, p, emptyValue);
 					}
 				}
 			}
@@ -230,10 +246,9 @@ export default abstract class Chart<T extends Record<string, any>> {
 	}
 
 	@autobind
-	private getLatestLog(span: Span, group: string | null = null): Promise<Log | null> {
+	private getLatestLog(group: string | null = null): Promise<Log | null> {
 		return this.repository.findOne({
 			group: group,
-			span: span
 		}, {
 			order: {
 				date: -1
@@ -242,17 +257,13 @@ export default abstract class Chart<T extends Record<string, any>> {
 	}
 
 	@autobind
-	private async getCurrentLog(span: Span, group: string | null = null): Promise<Log> {
+	private async getCurrentLog(group: string | null = null): Promise<Log> {
 		const [y, m, d, h] = Chart.getCurrentDate();
 
-		const current =
-			span == 'day' ? dateUTC([y, m, d, 0]) :
-			span == 'hour' ? dateUTC([y, m, d, h]) :
-			null as never;
+		const current = dateUTC([y, m, d, h]);
 
-		// 現在(今日または今のHour)のログ
+		// 現在(=今のHour)のログ
 		const currentLog = await this.repository.findOne({
-			span: span,
 			date: Chart.dateToTimestamp(current),
 			...(group ? { group: group } : {})
 		});
@@ -271,7 +282,7 @@ export default abstract class Chart<T extends Record<string, any>> {
 		// * 昨日何もチャートを更新するような出来事がなかった場合は、
 		// * ログがそもそも作られずドキュメントが存在しないということがあり得るため、
 		// * 「昨日の」と決め打ちせずに「もっとも最近の」とします
-		const latest = await this.getLatestLog(span, group);
+		const latest = await this.getLatestLog(group);
 
 		if (latest != null) {
 			const obj = Chart.convertFlattenColumnsToObject(
@@ -286,17 +297,16 @@ export default abstract class Chart<T extends Record<string, any>> {
 			// 初期ログデータを作成
 			data = this.getNewLog(null);
 
-			logger.info(`${this.name + (group ? `:${group}` : '')} (${span}): Initial commit created`);
+			logger.info(`${this.name + (group ? `:${group}` : '')}: Initial commit created`);
 		}
 
 		const date = Chart.dateToTimestamp(current);
-		const lockKey = `${this.name}:${date}:${group}:${span}`;
+		const lockKey = `${this.name}:${date}:${group}`;
 
 		const unlock = await getChartInsertLock(lockKey);
 		try {
 			// ロック内でもう1回チェックする
 			const currentLog = await this.repository.findOne({
-				span: span,
 				date: date,
 				...(group ? { group: group } : {})
 			});
@@ -307,12 +317,11 @@ export default abstract class Chart<T extends Record<string, any>> {
 			// 新規ログ挿入
 			log = await this.repository.save({
 				group: group,
-				span: span,
 				date: date,
 				...Chart.convertObjectToFlattenColumns(data)
 			});
 
-			logger.info(`${this.name + (group ? `:${group}` : '')} (${span}): New commit created`);
+			logger.info(`${this.name + (group ? `:${group}` : '')}: New commit created`);
 
 			return log;
 		} finally {
@@ -321,38 +330,62 @@ export default abstract class Chart<T extends Record<string, any>> {
 	}
 
 	@autobind
-	protected commit(query: Record<string, Function>, group: string | null = null, uniqueKey?: string, uniqueValue?: string): Promise<any> {
+	protected commit(diff: DeepPartial<T>, group: string | null = null): void {
+		this.queue.push({
+			diff, group,
+		});
+	}
+
+	@autobind
+	public async save() {
+		if (this.queue.length === 0) {
+			logger.info(`${this.name}: Write skipped`);
+			return;
+		}
+
+		// TODO: 前の時間のログがqueueにあった場合のハンドリング
+		// 例えば、save が20分ごとに行われるとして、前回行われたのは 01:50 だったとする。
+		// 次に save が行われるのは 02:10 ということになるが、もし 01:55 に新規ログが queue に追加されたとすると、
+		// そのログは本来は 01:00~ のログとしてDBに保存されて欲しいのに、02:00~ のログ扱いになってしまう。
+		// これを回避するための実装は複雑になりそうなため、一旦保留。
+
 		const update = async (log: Log) => {
-			// ユニークインクリメントの場合、指定のキーに指定の値が既に存在していたら弾く
-			if (
-				uniqueKey && log.unique &&
-				log.unique[uniqueKey] &&
-				log.unique[uniqueKey].includes(uniqueValue)
-			) return;
+			const finalDiffs = {} as Record<string, number | unknown[]>;
 
-			// ユニークインクリメントの指定のキーに値を追加
-			if (uniqueKey && log.unique) {
-				if (log.unique[uniqueKey]) {
-					const sql = `jsonb_set("unique", '{${uniqueKey}}', ("unique"->>'${uniqueKey}')::jsonb || '["${uniqueValue}"]'::jsonb)`;
-					query['unique'] = () => sql;
-				} else {
-					const sql = `jsonb_set("unique", '{${uniqueKey}}', '["${uniqueValue}"]')`;
-					query['unique'] = () => sql;
+			for (const diff of this.queue.filter(q => q.group === log.group).map(q => q.diff)) {
+				const columns = Chart.convertObjectToFlattenColumns(diff);
+
+				for (const [k, v] of Object.entries(columns)) {
+					if (finalDiffs[k] == null) {
+						finalDiffs[k] = v;
+					} else {
+						if (typeof finalDiffs[k] === 'number') {
+							(finalDiffs[k] as number) += v as number;
+						} else {
+							(finalDiffs[k] as unknown[]) = (finalDiffs[k] as unknown[]).concat(v);
+						}
+					}
 				}
 			}
 
+			const query = Chart.convertQuery(finalDiffs);
+
 			// ログ更新
 			await this.repository.createQueryBuilder()
 				.update()
 				.set(query)
 				.where('id = :id', { id: log.id })
 				.execute();
+
+			logger.info(`${this.name + (log.group ? `:${log.group}` : '')}: Updated`);
+
+			// TODO: この一連の処理が始まった後に新たにqueueに入ったものは消さないようにする
+			this.queue = this.queue.filter(q => q.group !== log.group);
 		};
 
-		return Promise.all([
-			this.getCurrentLog('day', group).then(log => update(log)),
-			this.getCurrentLog('hour', group).then(log => update(log)),
-		]);
+		const groups = removeDuplicates(this.queue.map(log => log.group));
+
+		await Promise.all(groups.map(group => this.getCurrentLog(group).then(log => update(log))));
 	}
 
 	@autobind
@@ -367,39 +400,30 @@ export default abstract class Chart<T extends Record<string, any>> {
 				.execute();
 		};
 
-		return Promise.all([
-			this.getCurrentLog('day', group).then(log => update(log)),
-			this.getCurrentLog('hour', group).then(log => update(log)),
-		]);
+		return this.getCurrentLog(group).then(log => update(log));
 	}
 
 	@autobind
 	protected async inc(inc: DeepPartial<T>, group: string | null = null): Promise<void> {
-		await this.commit(Chart.convertQuery(inc as any), group);
-	}
-
-	@autobind
-	protected async incIfUnique(inc: DeepPartial<T>, key: string, value: string, group: string | null = null): Promise<void> {
-		await this.commit(Chart.convertQuery(inc as any), group, key, value);
+		await this.commit(inc, group);
 	}
 
 	@autobind
-	public async getChart(span: Span, amount: number, begin: Date | null, group: string | null = null): Promise<ArrayValue<T>> {
-		const [y, m, d, h, _m, _s, _ms] = begin ? Chart.parseDate(subtractTime(addTime(begin, 1, span), 1)) : Chart.getCurrentDate();
-		const [y2, m2, d2, h2] = begin ? Chart.parseDate(addTime(begin, 1, span)) : [] as never;
+	public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<ArrayValue<T>> {
+		const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate();
+		const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [] as never;
 
 		const lt = dateUTC([y, m, d, h, _m, _s, _ms]);
 
 		const gt =
-			span === 'day' ? subtractTime(begin ? dateUTC([y2, m2, d2, 0]) : dateUTC([y, m, d, 0]), amount - 1, 'day') :
-			span === 'hour' ? subtractTime(begin ? dateUTC([y2, m2, d2, h2]) : dateUTC([y, m, d, h]), amount - 1, 'hour') :
+			span === 'day' ? subtractTime(cursor ? dateUTC([y2, m2, d2, 0]) : dateUTC([y, m, d, 0]), amount - 1, 'day') :
+			span === 'hour' ? subtractTime(cursor ? dateUTC([y2, m2, d2, h2]) : dateUTC([y, m, d, h]), amount - 1, 'hour') :
 			null as never;
 
 		// ログ取得
 		let logs = await this.repository.find({
 			where: {
 				group: group,
-				span: span,
 				date: Between(Chart.dateToTimestamp(gt), Chart.dateToTimestamp(lt))
 			},
 			order: {
@@ -413,7 +437,6 @@ export default abstract class Chart<T extends Record<string, any>> {
 			// (すくなくともひとつログが無いと隙間埋めできないため)
 			const recentLog = await this.repository.findOne({
 				group: group,
-				span: span
 			}, {
 				order: {
 					date: -1
@@ -430,7 +453,6 @@ export default abstract class Chart<T extends Record<string, any>> {
 			// (隙間埋めできないため)
 			const outdatedLog = await this.repository.findOne({
 				group: group,
-				span: span,
 				date: LessThan(Chart.dateToTimestamp(gt))
 			}, {
 				order: {
@@ -445,23 +467,56 @@ export default abstract class Chart<T extends Record<string, any>> {
 
 		const chart: T[] = [];
 
-		// 整形
-		for (let i = (amount - 1); i >= 0; i--) {
-			const current =
-				span === 'day' ? subtractTime(dateUTC([y, m, d, 0]), i, 'day') :
-				span === 'hour' ? subtractTime(dateUTC([y, m, d, h]), i, 'hour') :
-				null as never;
+		if (span === 'hour') {
+			for (let i = (amount - 1); i >= 0; i--) {
+				const current = subtractTime(dateUTC([y, m, d, h]), i, 'hour');
 
-			const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current));
+				const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current));
+
+				if (log) {
+					const data = Chart.convertFlattenColumnsToObject(log as Record<string, any>);
+					chart.unshift(Chart.countUniqueFields(data));
+				} else {
+					// 隙間埋め
+					const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current));
+					const data = latest ? Chart.convertFlattenColumnsToObject(latest as Record<string, any>) : null;
+					chart.unshift(Chart.countUniqueFields(this.getNewLog(data)));
+				}
+			}
+		} else if (span === 'day') {
+			const logsForEachDays: T[][] = [];
+			let currentDay = -1;
+			let currentDayIndex = -1;
+			for (let i = ((amount - 1) * 24) + h; i >= 0; i--) {
+				const current = subtractTime(dateUTC([y, m, d, h]), i, 'hour');
+				const _currentDay = Chart.parseDate(current)[2];
+				if (currentDay != _currentDay) currentDayIndex++;
+				currentDay = _currentDay;
+
+				const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current));
+
+				if (log) {
+					if (logsForEachDays[currentDayIndex]) {
+						logsForEachDays[currentDayIndex].unshift(Chart.convertFlattenColumnsToObject(log));
+					} else {
+						logsForEachDays[currentDayIndex] = [Chart.convertFlattenColumnsToObject(log)];
+					}
+				} else {
+					// 隙間埋め
+					const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current));
+					const data = latest ? Chart.convertFlattenColumnsToObject(latest as Record<string, any>) : null;
+					const newLog = this.getNewLog(data);
+					if (logsForEachDays[currentDayIndex]) {
+						logsForEachDays[currentDayIndex].unshift(newLog);
+					} else {
+						logsForEachDays[currentDayIndex] = [newLog];
+					}
+				}
+			}
 
-			if (log) {
-				const data = Chart.convertFlattenColumnsToObject(log as Record<string, any>);
-				chart.unshift(data);
-			} else {
-				// 隙間埋め
-				const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current));
-				const data = latest ? Chart.convertFlattenColumnsToObject(latest as Record<string, any>) : null;
-				chart.unshift(this.getNewLog(data));
+			for (const logs of logsForEachDays) {
+				const log = this.aggregate(logs);
+				chart.unshift(Chart.countUniqueFields(log));
 			}
 		}
 
@@ -473,20 +528,19 @@ export default abstract class Chart<T extends Record<string, any>> {
 		 * { foo: [1, 2, 3], bar: [5, 6, 7] }
 		 * にする
 		 */
-		const dive = (x: Obj, path?: string) => {
+		const compact = (x: Obj, path?: string) => {
 			for (const [k, v] of Object.entries(x)) {
 				const p = path ? `${path}.${k}` : k;
-				if (typeof v == 'object') {
-					dive(v, p);
+				if (typeof v === 'object' && !Array.isArray(v)) {
+					compact(v, p);
 				} else {
-					const values = chart.map(s => nestedProperty.get(s, p))
-						.map(v => parseInt(v, 10)); // TypeORMのバグ(？)で何故か数値カラムの値が文字列型になっているので数値に戻す
+					const values = chart.map(s => nestedProperty.get(s, p));
 					nestedProperty.set(res, p, values);
 				}
 			}
 		};
 
-		dive(chart[0]);
+		compact(chart[0]);
 
 		return res;
 	}