diff options
| author | syuilo <syuilotan@yahoo.co.jp> | 2018-10-21 07:10:35 +0900 |
|---|---|---|
| committer | syuilo <syuilotan@yahoo.co.jp> | 2018-10-21 07:10:35 +0900 |
| commit | ba0e57396d5ab609e18dcaf4ba8235376cf429ee (patch) | |
| tree | f73fed603024739483d3edd51fc2191a9d818ebb /src/services | |
| parent | Fix: Remove duplicated key (diff) | |
| download | sharkey-ba0e57396d5ab609e18dcaf4ba8235376cf429ee.tar.gz sharkey-ba0e57396d5ab609e18dcaf4ba8235376cf429ee.tar.bz2 sharkey-ba0e57396d5ab609e18dcaf4ba8235376cf429ee.zip | |
Refactoring
Diffstat (limited to 'src/services')
| -rw-r--r-- | src/services/drive/add-file.ts | 4 | ||||
| -rw-r--r-- | src/services/drive/delete-file.ts | 4 | ||||
| -rw-r--r-- | src/services/note/create.ts | 4 | ||||
| -rw-r--r-- | src/services/note/delete.ts | 4 | ||||
| -rw-r--r-- | src/services/stats.ts | 628 | ||||
| -rw-r--r-- | src/services/update-chart.ts | 267 |
6 files changed, 636 insertions, 275 deletions
diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index 3e689bec6a..a50f518c2b 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -17,7 +17,7 @@ import { isLocalUser, IUser, IRemoteUser } from '../../models/user'; import delFile from './delete-file'; import config from '../../config'; import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail'; -import { updateDriveStats } from '../update-chart'; +import { coreChart } from '../stats'; const log = debug('misskey:drive:add-file'); @@ -389,7 +389,7 @@ export default async function( }); // 統計を更新 - updateDriveStats(driveFile, true); + coreChart.updateDriveStats(driveFile, true); return driveFile; } diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index 73532a2953..1829705bcd 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -2,7 +2,7 @@ import * as Minio from 'minio'; import DriveFile, { DriveFileChunk, IDriveFile } from '../../models/drive-file'; import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-file-thumbnail'; import config from '../../config'; -import { updateDriveStats } from '../update-chart'; +import { coreChart } from '../stats'; export default async function(file: IDriveFile, isExpired = false) { if (file.metadata.storage == 'minio') { @@ -48,5 +48,5 @@ export default async function(file: IDriveFile, isExpired = false) { //#endregion // 統計を更新 - updateDriveStats(file, false); + coreChart.updateDriveStats(file, false); } diff --git a/src/services/note/create.ts b/src/services/note/create.ts index 3dc411d434..0f844d4e35 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -23,7 +23,7 @@ import registerHashtag from '../register-hashtag'; import isQuote from '../../misc/is-quote'; import { TextElementMention } from '../../mfm/parse/elements/mention'; import { TextElementHashtag } from '../../mfm/parse/elements/hashtag'; -import { updateNoteStats } from '../update-chart'; +import { coreChart } from '../stats'; import { erase, unique } from '../../prelude/array'; import insertNoteUnread from './unread'; @@ -165,7 +165,7 @@ export default async (user: IUser, data: Option, silent = false) => new Promise< } // 統計を更新 - updateNoteStats(note, true); + coreChart.updateNoteStats(note, true); // ハッシュタグ登録 tags.map(tag => registerHashtag(user, tag)); diff --git a/src/services/note/delete.ts b/src/services/note/delete.ts index 4b0f482c08..a0a86cad25 100644 --- a/src/services/note/delete.ts +++ b/src/services/note/delete.ts @@ -6,7 +6,7 @@ import pack from '../../remote/activitypub/renderer'; import { deliver } from '../../queue'; import Following from '../../models/following'; import renderTombstone from '../../remote/activitypub/renderer/tombstone'; -import { updateNoteStats } from '../update-chart'; +import { coreChart } from '../stats'; import config from '../../config'; import NoteUnread from '../../models/note-unread'; import read from './read'; @@ -63,5 +63,5 @@ export default async function(user: IUser, note: INote) { //#endregion // 統計を更新 - updateNoteStats(note, false); + coreChart.updateNoteStats(note, false); } diff --git a/src/services/stats.ts b/src/services/stats.ts new file mode 100644 index 0000000000..9e471a02e9 --- /dev/null +++ b/src/services/stats.ts @@ -0,0 +1,628 @@ +import * as mongo from 'mongodb'; +import db from '../db/mongodb'; +import { INote } from '../models/note'; +import { isLocalUser, IUser } from '../models/user'; +import { IDriveFile } from '../models/drive-file'; +import { ICollection } from 'monk'; + +type Obj = { [key: string]: any }; + +type Partial<T> = { + [P in keyof T]?: Partial<T[P]>; +}; + +type Span = 'day' | 'hour'; + +type ChartDocument<T extends Obj> = { + _id: mongo.ObjectID; + + /** + * 集計日時 + */ + date: Date; + + /** + * 集計期間 + */ + span: Span; + + /** + * データ + */ + data: T; +}; + +abstract class Chart<T> { + protected collection: ICollection<ChartDocument<T>>; + protected abstract generateInitialStats(): T; + protected abstract generateEmptyStats(mostRecentStats: T): T; + + constructor(dbCollectionName: string) { + this.collection = db.get<ChartDocument<T>>(dbCollectionName); + this.collection.createIndex({ span: -1, date: -1 }, { unique: true }); + } + + protected async getCurrentStats(span: Span, group?: Obj): Promise<ChartDocument<T>> { + const now = new Date(); + const y = now.getFullYear(); + const m = now.getMonth(); + const d = now.getDate(); + const h = now.getHours(); + + const current = + span == 'day' ? new Date(y, m, d) : + span == 'hour' ? new Date(y, m, d, h) : + null; + + // 現在(今日または今のHour)の統計 + const currentStats = await this.collection.findOne(Object.assign({}, { + span: span, + date: current + }, group)); + + if (currentStats) { + return currentStats; + } else { + // 集計期間が変わってから、初めてのチャート更新なら + // 最も最近の統計を持ってくる + // * 例えば集計期間が「日」である場合で考えると、 + // * 昨日何もチャートを更新するような出来事がなかった場合は、 + // * 統計がそもそも作られずドキュメントが存在しないということがあり得るため、 + // * 「昨日の」と決め打ちせずに「もっとも最近の」とします + const mostRecentStats = await this.collection.findOne(Object.assign({}, { + span: span + }, group), { + sort: { + date: -1 + } + }); + + if (mostRecentStats) { + // 現在の統計を初期挿入 + const data = this.generateEmptyStats(mostRecentStats.data); + + const stats = await this.collection.insert(Object.assign({}, { + span: span, + date: current, + data: data + }, group)); + + return stats; + } else { + // 統計が存在しなかったら + // * Misskeyインスタンスを建てて初めてのチャート更新時など + + // 空の統計を作成 + const data = this.generateInitialStats(); + + const stats = await this.collection.insert(Object.assign({}, { + span: span, + date: current, + data: data + }, group)); + + return stats; + } + } + } + + protected update(inc: Partial<T>, group?: Obj): void { + const query: Obj = {}; + + const dive = (path: string, x: Obj) => { + Object.entries(x).forEach(([k, v]) => { + if (typeof v === 'number') { + query[path == null ? `data.${k}` : `data.${path}.${k}`] = v; + } else { + dive(path == null ? k : `${path}.${k}`, v); + } + }); + }; + + dive(null, inc); + + this.getCurrentStats('day', group).then(stats => { + this.collection.findOneAndUpdate({ + _id: stats._id + }, { + $inc: query + }); + }); + + this.getCurrentStats('hour', group).then(stats => { + this.collection.findOneAndUpdate({ + _id: stats._id + }, { + $inc: query + }); + }); + } + + public async getStats(span: Span, range: number, group?: Obj): Promise<T[]> { + const chart: T[] = []; + + const now = new Date(); + const y = now.getFullYear(); + const m = now.getMonth(); + const d = now.getDate(); + const h = now.getHours(); + + const gt = + span == 'day' ? new Date(y, m, d - range) : + span == 'hour' ? new Date(y, m, d, h - range) : null; + + const stats = await this.collection.find(Object.assign({ + span: span, + date: { + $gt: gt + } + }, group), { + sort: { + date: -1 + }, + fields: { + _id: 0 + } + }); + + for (let i = (range - 1); i >= 0; i--) { + const current = + span == 'day' ? new Date(y, m, d - i) : + span == 'hour' ? new Date(y, m, d, h - i) : + null; + + const stat = stats.find(s => s.date.getTime() == current.getTime()); + + if (stat) { + chart.unshift(stat.data); + } else { // 隙間埋め + const mostRecent = stats.find(s => s.date.getTime() < current.getTime()); + if (mostRecent) { + chart.unshift(this.generateEmptyStats(mostRecent.data)); + } else { + chart.unshift(this.generateInitialStats()); + } + } + } + + return chart; + } +} + +type CoreStats = { + /** + * ユーザーに関する統計 + */ + users: { + local: { + /** + * 集計期間時点での、全ユーザー数 (ローカル) + */ + total: number; + + /** + * 増加したユーザー数 (ローカル) + */ + inc: number; + + /** + * 減少したユーザー数 (ローカル) + */ + dec: number; + }; + + remote: { + /** + * 集計期間時点での、全ユーザー数 (リモート) + */ + total: number; + + /** + * 増加したユーザー数 (リモート) + */ + inc: number; + + /** + * 減少したユーザー数 (リモート) + */ + dec: number; + }; + }; + + /** + * 投稿に関する統計 + */ + notes: { + local: { + /** + * 集計期間時点での、全投稿数 (ローカル) + */ + total: number; + + /** + * 増加した投稿数 (ローカル) + */ + inc: number; + + /** + * 減少した投稿数 (ローカル) + */ + dec: number; + + diffs: { + /** + * 通常の投稿数の差分 (ローカル) + */ + normal: number; + + /** + * リプライの投稿数の差分 (ローカル) + */ + reply: number; + + /** + * Renoteの投稿数の差分 (ローカル) + */ + renote: number; + }; + }; + + remote: { + /** + * 集計期間時点での、全投稿数 (リモート) + */ + total: number; + + /** + * 増加した投稿数 (リモート) + */ + inc: number; + + /** + * 減少した投稿数 (リモート) + */ + dec: number; + + diffs: { + /** + * 通常の投稿数の差分 (リモート) + */ + normal: number; + + /** + * リプライの投稿数の差分 (リモート) + */ + reply: number; + + /** + * Renoteの投稿数の差分 (リモート) + */ + renote: number; + }; + }; + }; + + /** + * ドライブ(のファイル)に関する統計 + */ + drive: { + local: { + /** + * 集計期間時点での、全ドライブファイル数 (ローカル) + */ + totalCount: number; + + /** + * 集計期間時点での、全ドライブファイルの合計サイズ (ローカル) + */ + totalSize: number; + + /** + * 増加したドライブファイル数 (ローカル) + */ + incCount: number; + + /** + * 増加したドライブ使用量 (ローカル) + */ + incSize: number; + + /** + * 減少したドライブファイル数 (ローカル) + */ + decCount: number; + + /** + * 減少したドライブ使用量 (ローカル) + */ + decSize: number; + }; + + remote: { + /** + * 集計期間時点での、全ドライブファイル数 (リモート) + */ + totalCount: number; + + /** + * 集計期間時点での、全ドライブファイルの合計サイズ (リモート) + */ + totalSize: number; + + /** + * 増加したドライブファイル数 (リモート) + */ + incCount: number; + + /** + * 増加したドライブ使用量 (リモート) + */ + incSize: number; + + /** + * 減少したドライブファイル数 (リモート) + */ + decCount: number; + + /** + * 減少したドライブ使用量 (リモート) + */ + decSize: number; + }; + }; + + /** + * ネットワークに関する統計 + */ + network: { + /** + * 受信したリクエスト数 + */ + incomingRequests: number; + + /** + * 送信したリクエスト数 + */ + outgoingRequests: number; + + /** + * 応答時間の合計 + * TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる + */ + totalTime: number; + + /** + * 合計受信データ量 + */ + incomingBytes: number; + + /** + * 合計送信データ量 + */ + outgoingBytes: number; + }; +}; + +class CoreChart extends Chart<CoreStats> { + constructor() { + super('coreStats'); + } + + protected generateInitialStats(): CoreStats { + return { + users: { + local: { + total: 0, + inc: 0, + dec: 0 + }, + remote: { + total: 0, + inc: 0, + dec: 0 + } + }, + notes: { + local: { + total: 0, + inc: 0, + dec: 0, + diffs: { + normal: 0, + reply: 0, + renote: 0 + } + }, + remote: { + total: 0, + inc: 0, + dec: 0, + diffs: { + normal: 0, + reply: 0, + renote: 0 + } + } + }, + drive: { + local: { + totalCount: 0, + totalSize: 0, + incCount: 0, + incSize: 0, + decCount: 0, + decSize: 0 + }, + remote: { + totalCount: 0, + totalSize: 0, + incCount: 0, + incSize: 0, + decCount: 0, + decSize: 0 + } + }, + network: { + incomingRequests: 0, + outgoingRequests: 0, + totalTime: 0, + incomingBytes: 0, + outgoingBytes: 0 + } + }; + } + + protected generateEmptyStats(mostRecentStats: CoreStats): CoreStats { + return { + users: { + local: { + total: mostRecentStats.users.local.total, + inc: 0, + dec: 0 + }, + remote: { + total: mostRecentStats.users.remote.total, + inc: 0, + dec: 0 + } + }, + notes: { + local: { + total: mostRecentStats.notes.local.total, + inc: 0, + dec: 0, + diffs: { + normal: 0, + reply: 0, + renote: 0 + } + }, + remote: { + total: mostRecentStats.notes.remote.total, + inc: 0, + dec: 0, + diffs: { + normal: 0, + reply: 0, + renote: 0 + } + } + }, + drive: { + local: { + totalCount: mostRecentStats.drive.local.totalCount, + totalSize: mostRecentStats.drive.local.totalSize, + incCount: 0, + incSize: 0, + decCount: 0, + decSize: 0 + }, + remote: { + totalCount: mostRecentStats.drive.remote.totalCount, + totalSize: mostRecentStats.drive.remote.totalSize, + incCount: 0, + incSize: 0, + decCount: 0, + decSize: 0 + } + }, + network: { + incomingRequests: 0, + outgoingRequests: 0, + totalTime: 0, + incomingBytes: 0, + outgoingBytes: 0 + } + }; + } + + public async updateUserStats(user: IUser, isAdditional: boolean) { + const origin = isLocalUser(user) ? 'local' : 'remote'; + + const update: Obj = {}; + + update.total = isAdditional ? 1 : -1; + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + const inc: Obj = { + users: {} + }; + + inc.users[origin] = update; + + await this.update(inc); + } + + public async updateNoteStats(note: INote, isAdditional: boolean) { + const origin = isLocalUser(note._user) ? 'local' : 'remote'; + + const update: Obj = {}; + + update.total = isAdditional ? 1 : -1; + + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + if (note.replyId != null) { + update.diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + update.diffs.renote = isAdditional ? 1 : -1; + } else { + update.diffs.normal = isAdditional ? 1 : -1; + } + + const inc: Obj = { + notes: {} + }; + + inc.notes[origin] = update; + + await this.update(inc); + } + + public async updateDriveStats(file: IDriveFile, isAdditional: boolean) { + const origin = isLocalUser(file.metadata._user) ? 'local' : 'remote'; + + const update: Obj = {}; + + update.totalCount = isAdditional ? 1 : -1; + update.totalSize = isAdditional ? file.length : -file.length; + if (isAdditional) { + update.incCount = 1; + update.incSize = file.length; + } else { + update.decCount = 1; + update.decSize = file.length; + } + + const inc: Obj = { + drive: {} + }; + + inc.drive[origin] = update; + + await this.update(inc); + } + + public async updateNetworkStats(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number) { + const inc: Partial<CoreStats> = { + network: { + incomingRequests: incomingRequests, + totalTime: time, + incomingBytes: incomingBytes, + outgoingBytes: outgoingBytes + } + }; + + await this.update(inc); + } +} + +export const coreChart = new CoreChart(); diff --git a/src/services/update-chart.ts b/src/services/update-chart.ts deleted file mode 100644 index 78834ba601..0000000000 --- a/src/services/update-chart.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { INote } from '../models/note'; -import Stats, { IStats } from '../models/stats'; -import { isLocalUser, IUser } from '../models/user'; -import { IDriveFile } from '../models/drive-file'; - -type Omit<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>; - -async function getCurrentStats(span: 'day' | 'hour'): Promise<IStats> { - const now = new Date(); - const y = now.getFullYear(); - const m = now.getMonth(); - const d = now.getDate(); - const h = now.getHours(); - - const current = - span == 'day' ? new Date(y, m, d) : - span == 'hour' ? new Date(y, m, d, h) : - null; - - // 現在(今日または今のHour)の統計 - const currentStats = await Stats.findOne({ - span: span, - date: current - }); - - if (currentStats) { - return currentStats; - } else { - // 集計期間が変わってから、初めてのチャート更新なら - // 最も最近の統計を持ってくる - // * 例えば集計期間が「日」である場合で考えると、 - // * 昨日何もチャートを更新するような出来事がなかった場合は、 - // * 統計がそもそも作られずドキュメントが存在しないということがあり得るため、 - // * 「昨日の」と決め打ちせずに「もっとも最近の」とします - const mostRecentStats = await Stats.findOne({ - span: span - }, { - sort: { - date: -1 - } - }); - - if (mostRecentStats) { - // 現在の統計を初期挿入 - const data: Omit<IStats, '_id'> = { - span: span, - date: current, - users: { - local: { - total: mostRecentStats.users.local.total, - inc: 0, - dec: 0 - }, - remote: { - total: mostRecentStats.users.remote.total, - inc: 0, - dec: 0 - } - }, - notes: { - local: { - total: mostRecentStats.notes.local.total, - inc: 0, - dec: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - }, - remote: { - total: mostRecentStats.notes.remote.total, - inc: 0, - dec: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - } - }, - drive: { - local: { - totalCount: mostRecentStats.drive.local.totalCount, - totalSize: mostRecentStats.drive.local.totalSize, - incCount: 0, - incSize: 0, - decCount: 0, - decSize: 0 - }, - remote: { - totalCount: mostRecentStats.drive.remote.totalCount, - totalSize: mostRecentStats.drive.remote.totalSize, - incCount: 0, - incSize: 0, - decCount: 0, - decSize: 0 - } - }, - network: { - requests: 0, - totalTime: 0, - incomingBytes: 0, - outgoingBytes: 0 - } - }; - - const stats = await Stats.insert(data); - - return stats; - } else { - // 統計が存在しなかったら - // * Misskeyインスタンスを建てて初めてのチャート更新時など - - // 空の統計を作成 - const emptyStat: Omit<IStats, '_id'> = { - span: span, - date: current, - users: { - local: { - total: 0, - inc: 0, - dec: 0 - }, - remote: { - total: 0, - inc: 0, - dec: 0 - } - }, - notes: { - local: { - total: 0, - inc: 0, - dec: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - }, - remote: { - total: 0, - inc: 0, - dec: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - } - }, - drive: { - local: { - totalCount: 0, - totalSize: 0, - incCount: 0, - incSize: 0, - decCount: 0, - decSize: 0 - }, - remote: { - totalCount: 0, - totalSize: 0, - incCount: 0, - incSize: 0, - decCount: 0, - decSize: 0 - } - }, - network: { - requests: 0, - totalTime: 0, - incomingBytes: 0, - outgoingBytes: 0 - } - }; - - const stats = await Stats.insert(emptyStat); - - return stats; - } - } -} - -function update(inc: any) { - getCurrentStats('day').then(stats => { - Stats.findOneAndUpdate({ - _id: stats._id - }, { - $inc: inc - }); - }); - - getCurrentStats('hour').then(stats => { - Stats.findOneAndUpdate({ - _id: stats._id - }, { - $inc: inc - }); - }); -} - -export async function updateUserStats(user: IUser, isAdditional: boolean) { - const origin = isLocalUser(user) ? 'local' : 'remote'; - - const inc = {} as any; - inc[`users.${origin}.total`] = isAdditional ? 1 : -1; - if (isAdditional) { - inc[`users.${origin}.inc`] = 1; - } else { - inc[`users.${origin}.dec`] = 1; - } - - await update(inc); -} - -export async function updateNoteStats(note: INote, isAdditional: boolean) { - const origin = isLocalUser(note._user) ? 'local' : 'remote'; - - const inc = {} as any; - - inc[`notes.${origin}.total`] = isAdditional ? 1 : -1; - - if (isAdditional) { - inc[`notes.${origin}.inc`] = 1; - } else { - inc[`notes.${origin}.dec`] = 1; - } - - if (note.replyId != null) { - inc[`notes.${origin}.diffs.reply`] = isAdditional ? 1 : -1; - } else if (note.renoteId != null) { - inc[`notes.${origin}.diffs.renote`] = isAdditional ? 1 : -1; - } else { - inc[`notes.${origin}.diffs.normal`] = isAdditional ? 1 : -1; - } - - await update(inc); -} - -export async function updateDriveStats(file: IDriveFile, isAdditional: boolean) { - const origin = isLocalUser(file.metadata._user) ? 'local' : 'remote'; - - const inc = {} as any; - inc[`drive.${origin}.totalCount`] = isAdditional ? 1 : -1; - inc[`drive.${origin}.totalSize`] = isAdditional ? file.length : -file.length; - if (isAdditional) { - inc[`drive.${origin}.incCount`] = 1; - inc[`drive.${origin}.incSize`] = file.length; - } else { - inc[`drive.${origin}.decCount`] = 1; - inc[`drive.${origin}.decSize`] = file.length; - } - - await update(inc); -} - -export async function updateNetworkStats(requests: number, time: number, incomingBytes: number, outgoingBytes: number) { - const inc = {} as any; - inc['network.requests'] = requests; - inc['network.totalTime'] = time; - inc['network.incomingBytes'] = incomingBytes; - inc['network.outgoingBytes'] = outgoingBytes; - - await update(inc); -} |