1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
import * as bq from 'bee-queue';
import * as tmp from 'tmp';
import * as fs from 'fs';
import * as mongo from 'mongodb';
import { queueLogger } from '../logger';
import Note, { INote } from '../../models/note';
import addFile from '../../services/drive/add-file';
import User from '../../models/user';
import dateFormat = require('dateformat');
const logger = queueLogger.createSubLogger('export-notes');
export async function exportNotes(job: bq.Job, done: any): Promise<void> {
logger.info(`Exporting notes of ${job.data.user._id} ...`);
const user = await User.findOne({
_id: new mongo.ObjectID(job.data.user._id.toString())
});
// Create temp file
const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
logger.info(`Temp file is ${path}`);
const stream = fs.createWriteStream(path, { flags: 'a' });
await new Promise((res, rej) => {
stream.write('[', err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
let exportedNotesCount = 0;
let ended = false;
let cursor: any = null;
while (!ended) {
const notes = await Note.find({
userId: user._id,
...(cursor ? { _id: { $gt: cursor } } : {})
}, {
limit: 100,
sort: {
_id: 1
}
});
if (notes.length === 0) {
ended = true;
if (job.reportProgress) job.reportProgress(100);
break;
}
cursor = notes[notes.length - 1]._id;
for (const note of notes) {
const content = JSON.stringify(serialize(note));
await new Promise((res, rej) => {
stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
exportedNotesCount++;
}
const total = await Note.count({
userId: user._id,
});
if (job.reportProgress) job.reportProgress(exportedNotesCount / total);
}
await new Promise((res, rej) => {
stream.write(']', err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
stream.end();
logger.succ(`Exported to: ${path}`);
const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json';
const driveFile = await addFile(user, path, fileName);
logger.succ(`Exported to: ${driveFile._id}`);
cleanup();
done();
}
function serialize(note: INote): any {
return {
id: note._id,
text: note.text,
createdAt: note.createdAt,
fileIds: note.fileIds,
replyId: note.replyId,
renoteId: note.renoteId,
poll: note.poll,
cw: note.cw,
viaMobile: note.viaMobile,
visibility: note.visibility,
visibleUserIds: note.visibleUserIds,
appId: note.appId,
geo: note.geo,
localOnly: note.localOnly
};
}
|
