diff options
Diffstat (limited to 'packages/backend/src/queue')
4 files changed, 144 insertions, 1 deletions
diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index c5a07673bd..2fbc1b1c01 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -117,6 +117,15 @@ export function createDeleteDriveFilesJob(user: ThinUser) { }); } +export function createExportCustomEmojisJob(user: ThinUser) { + return dbQueue.add('exportCustomEmojis', { + user: user, + }, { + removeOnComplete: true, + removeOnFail: true, + }); +} + export function createExportNotesJob(user: ThinUser) { return dbQueue.add('exportNotes', { user: user, diff --git a/packages/backend/src/queue/processors/db/export-custom-emojis.ts b/packages/backend/src/queue/processors/db/export-custom-emojis.ts new file mode 100644 index 0000000000..ed0ad249a5 --- /dev/null +++ b/packages/backend/src/queue/processors/db/export-custom-emojis.ts @@ -0,0 +1,131 @@ +import * as Bull from 'bull'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; + +import { ulid } from 'ulid'; +const mime = require('mime-types'); +const archiver = require('archiver'); +import { queueLogger } from '../../logger'; +import addFile from '@/services/drive/add-file'; +import * as dateFormat from 'dateformat'; +import { Users, Emojis } from '@/models/index'; +import { } from '@/queue/types'; +import { downloadUrl } from '@/misc/download-url'; + +const logger = queueLogger.createSubLogger('export-custom-emojis'); + +export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promise<void> { + logger.info(`Exporting custom emojis ...`); + + const user = await Users.findOne(job.data.user.id); + if (user == null) { + done(); + return; + } + + // Create temp dir + const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => { + tmp.dir((e, path, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + logger.info(`Temp dir is ${path}`); + + const metaPath = path + '/meta.json'; + + fs.writeFileSync(metaPath, '', 'utf-8'); + + const metaStream = fs.createWriteStream(metaPath, { flags: 'a' }); + + await new Promise<void>((res, rej) => { + metaStream.write('[', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + + const customEmojis = await Emojis.find({ + where: { + host: null, + }, + order: { + id: 'ASC', + }, + }); + + for (const emoji of customEmojis) { + const exportId = ulid().toLowerCase(); + const emojiPath = path + '/' + exportId + '.' + mime.extension(emoji.type); + fs.writeFileSync(emojiPath, '', 'binary'); + let downloaded = false; + + try { + await downloadUrl(emoji.url, emojiPath); + downloaded = true; + } catch (e) { // TODO: 何度か再試行 + logger.error(e); + } + + await new Promise<void>((res, rej) => { + const content = JSON.stringify({ + id: exportId, + downloaded: downloaded, + emoji: emoji, + }); + const isFirst = customEmojis.indexOf(emoji) === 0; + metaStream.write(isFirst ? content : ',\n' + content, err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + } + + await new Promise<void>((res, rej) => { + metaStream.write(']', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + + metaStream.end(); + + // Create archive + const [archivePath, archiveCleanup] = await new Promise<[string, () => void]>((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + const archiveStream = fs.createWriteStream(archivePath); + const archive = archiver('zip', { + zlib: { level: 0 }, + }); + archiveStream.on('close', async () => { + logger.succ(`Exported to: ${archivePath}`); + + const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.zip'; + const driveFile = await addFile(user, archivePath, fileName, null, null, true); + + logger.succ(`Exported to: ${driveFile.id}`); + cleanup(); + archiveCleanup(); + done(); + }); + archive.pipe(archiveStream); + archive.directory(path, false); + archive.finalize(); +} diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts index 761f4d827b..7220455edc 100644 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ b/packages/backend/src/queue/processors/db/export-notes.ts @@ -74,7 +74,8 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom } const content = JSON.stringify(serialize(note, poll)); await new Promise<void>((res, rej) => { - stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => { + const isFirst = exportedNotesCount === 0; + stream.write(isFirst ? content : ',\n' + content, err => { if (err) { logger.error(err); rej(err); diff --git a/packages/backend/src/queue/processors/db/index.ts b/packages/backend/src/queue/processors/db/index.ts index 97087642b7..1542f401ef 100644 --- a/packages/backend/src/queue/processors/db/index.ts +++ b/packages/backend/src/queue/processors/db/index.ts @@ -1,6 +1,7 @@ import * as Bull from 'bull'; import { DbJobData } from '@/queue/types'; import { deleteDriveFiles } from './delete-drive-files'; +import { exportCustomEmojis } from './export-custom-emojis'; import { exportNotes } from './export-notes'; import { exportFollowing } from './export-following'; import { exportMute } from './export-mute'; @@ -14,6 +15,7 @@ import { importBlocking } from './import-blocking'; const jobs = { deleteDriveFiles, + exportCustomEmojis, exportNotes, exportFollowing, exportMute, |