1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
|
import * as Bull from 'bull';
import * as tmp from 'tmp';
import * as fs from 'fs';
import { queueLogger } from '../../logger';
import addFile from '../../../services/drive/add-file';
import dateFormat = require('dateformat');
import { Users, Notes, Polls } from '../../../models';
import { MoreThan } from 'typeorm';
import { Note } from '../../../models/entities/note';
import { Poll } from '../../../models/entities/poll';
import { ensure } from '../../../prelude/ensure';
const logger = queueLogger.createSubLogger('export-notes');
export async function exportNotes(job: Bull.Job, done: any): Promise<void> {
logger.info(`Exporting notes of ${job.data.user.id} ...`);
const user = await Users.findOne(job.data.user.id);
if (user == null) {
done();
return;
}
// Create temp file
const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
logger.info(`Temp file is ${path}`);
const stream = fs.createWriteStream(path, { flags: 'a' });
await new Promise((res, rej) => {
stream.write('[', err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
let exportedNotesCount = 0;
let cursor: any = null;
while (true) {
const notes = await Notes.find({
where: {
userId: user.id,
...(cursor ? { id: MoreThan(cursor) } : {})
},
take: 100,
order: {
id: 1
}
});
if (notes.length === 0) {
job.progress(100);
break;
}
cursor = notes[notes.length - 1].id;
for (const note of notes) {
let poll: Poll | undefined;
if (note.hasPoll) {
poll = await Polls.findOne({ noteId: note.id }).then(ensure);
}
const content = JSON.stringify(serialize(note, poll));
await new Promise((res, rej) => {
stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
exportedNotesCount++;
}
const total = await Notes.count({
userId: user.id,
});
job.progress(exportedNotesCount / total);
}
await new Promise((res, rej) => {
stream.write(']', err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
stream.end();
logger.succ(`Exported to: ${path}`);
const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json';
const driveFile = await addFile(user, path, fileName);
logger.succ(`Exported to: ${driveFile.id}`);
cleanup();
done();
}
function serialize(note: Note, poll: Poll | null = null): any {
return {
id: note.id,
text: note.text,
createdAt: note.createdAt,
fileIds: note.fileIds,
replyId: note.replyId,
renoteId: note.renoteId,
poll: poll,
cw: note.cw,
viaMobile: note.viaMobile,
visibility: note.visibility,
visibleUserIds: note.visibleUserIds,
appId: note.appId,
geo: note.geo,
localOnly: note.localOnly
};
}
|
