From 5db5bbd1cd25f83640d4dd01de14e7774d9370db Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 5 Feb 2019 19:50:14 +0900 Subject: 自分の投稿情報をエクスポートできるように (#4144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * 正しいJSONを生成するように * データを整形 --- src/queue/processors/export-notes.ts | 128 +++++++++++++++++++++++++++++++++++ src/queue/processors/http/deliver.ts | 2 +- src/queue/processors/http/index.ts | 19 ------ src/queue/processors/index.ts | 21 ++++++ 4 files changed, 150 insertions(+), 20 deletions(-) create mode 100644 src/queue/processors/export-notes.ts delete mode 100644 src/queue/processors/http/index.ts create mode 100644 src/queue/processors/index.ts (limited to 'src/queue/processors') diff --git a/src/queue/processors/export-notes.ts b/src/queue/processors/export-notes.ts new file mode 100644 index 0000000000..52845a5a9c --- /dev/null +++ b/src/queue/processors/export-notes.ts @@ -0,0 +1,128 @@ +import * as bq from 'bee-queue'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +import * as mongo from 'mongodb'; + +import { queueLogger } from '../logger'; +import Note, { INote } from '../../models/note'; +import addFile from '../../services/drive/add-file'; +import User from '../../models/user'; +import dateFormat = require('dateformat'); + +const logger = queueLogger.createSubLogger('export-notes'); + +export async function exportNotes(job: bq.Job, done: any): Promise { + logger.info(`Exporting notes of ${job.data.user._id} ...`); + + const user = await User.findOne({ + _id: new mongo.ObjectID(job.data.user._id.toString()) + }); + + // Create temp file + const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + logger.info(`Temp file is ${path}`); + + const stream = fs.createWriteStream(path, { flags: 'a' }); + + await new Promise((res, rej) => { + stream.write('[', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + + let exportedNotesCount = 0; + let ended = false; + let cursor: any = null; + + while (!ended) { + const notes = await Note.find({ + userId: user._id, + ...(cursor ? { _id: { $gt: cursor } } : {}) + }, { + limit: 100, + sort: { + _id: 1 + } + }); + + if (notes.length === 0) { + ended = true; + job.reportProgress(100); + break; + } + + cursor = notes[notes.length - 1]._id; + + for (const note of notes) { + const content = JSON.stringify(serialize(note)); + await new Promise((res, rej) => { + stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + exportedNotesCount++; + } + + const total = await Note.count({ + userId: user._id, + }); + + job.reportProgress(exportedNotesCount / total); + } + + await new Promise((res, rej) => { + stream.write(']', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + + stream.end(); + logger.succ(`Exported to: ${path}`); + + const fileName = dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json'; + const driveFile = await addFile(user, path, fileName); + + logger.succ(`Exported to: ${driveFile._id}`); + cleanup(); + done(); +} + +function serialize(note: INote): any { + return { + id: note._id, + text: note.text, + createdAt: note.createdAt, + fileIds: note.fileIds, + replyId: note.replyId, + renoteId: note.renoteId, + poll: note.poll, + cw: note.cw, + viaMobile: note.viaMobile, + visibility: note.visibility, + visibleUserIds: note.visibleUserIds, + appId: note.appId, + geo: note.geo, + localOnly: note.localOnly + }; +} diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index d8d90a2773..d1dad55cd7 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -1,7 +1,7 @@ import * as bq from 'bee-queue'; import request from '../../../remote/activitypub/request'; -import { queueLogger } from '../..'; +import { queueLogger } from '../../logger'; export default async (job: bq.Job, done: any): Promise => { try { diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/http/index.ts deleted file mode 100644 index 74ed723bd3..0000000000 --- a/src/queue/processors/http/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -import deliver from './deliver'; -import processInbox from './process-inbox'; -import { queueLogger } from '../..'; - -const handlers: any = { - deliver, - processInbox, -}; - -export default (job: any, done: any) => { - const handler = handlers[job.data.type]; - - if (handler) { - handler(job, done); - } else { - queueLogger.error(`Unknown job: ${job.data.type}`); - done(); - } -}; diff --git a/src/queue/processors/index.ts b/src/queue/processors/index.ts new file mode 100644 index 0000000000..3f08fe29fb --- /dev/null +++ b/src/queue/processors/index.ts @@ -0,0 +1,21 @@ +import deliver from './http/deliver'; +import processInbox from './http/process-inbox'; +import { exportNotes } from './export-notes'; +import { queueLogger } from '../logger'; + +const handlers: any = { + deliver, + processInbox, + exportNotes, +}; + +export default (job: any, done: any) => { + const handler = handlers[job.data.type]; + + if (handler) { + handler(job, done); + } else { + queueLogger.error(`Unknown job: ${job.data.type}`); + done(); + } +}; -- cgit v1.2.3-freya