diff options
Diffstat (limited to 'packages/backend/src/queue/processors')
9 files changed, 111 insertions, 19 deletions
diff --git a/packages/backend/src/queue/processors/db/export-blocking.ts b/packages/backend/src/queue/processors/db/export-blocking.ts index 8c886d3b4e..01edaaeb63 100644 --- a/packages/backend/src/queue/processors/db/export-blocking.ts +++ b/packages/backend/src/queue/processors/db/export-blocking.ts @@ -3,7 +3,7 @@ import * as tmp from 'tmp'; import * as fs from 'fs'; import { queueLogger } from '../../logger'; -import addFile from '@/services/drive/add-file'; +import { addFile } from '@/services/drive/add-file'; import * as dateFormat from 'dateformat'; import { getFullApAccount } from '@/misc/convert-host'; import { Users, Blockings } from '@/models/index'; @@ -86,7 +86,7 @@ export async function exportBlocking(job: Bull.Job<DbUserJobData>, done: any): P logger.succ(`Exported to: ${path}`); const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; - const driveFile = await addFile(user, path, fileName, null, null, true); + const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); cleanup(); diff --git a/packages/backend/src/queue/processors/db/export-custom-emojis.ts b/packages/backend/src/queue/processors/db/export-custom-emojis.ts index 3930b9d6d4..240a542fec 100644 --- a/packages/backend/src/queue/processors/db/export-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/export-custom-emojis.ts @@ -6,11 +6,12 @@ import { ulid } from 'ulid'; const mime = require('mime-types'); const archiver = require('archiver'); import { queueLogger } from '../../logger'; -import addFile from '@/services/drive/add-file'; +import { addFile } from '@/services/drive/add-file'; import * as dateFormat from 'dateformat'; import { Users, Emojis } from '@/models/index'; import { } from '@/queue/types'; import { downloadUrl } from '@/misc/download-url'; +import config from '@/config/index'; const logger = queueLogger.createSubLogger('export-custom-emojis'); @@ -52,7 +53,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi }); }; - await writeMeta(`{"metaVersion":1,"emojis":[`); + await writeMeta(`{"metaVersion":2,"host":"${config.host}","exportedAt":"${new Date().toString()}","emojis":[`); const customEmojis = await Emojis.find({ where: { @@ -64,21 +65,25 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi }); for (const emoji of customEmojis) { - const exportId = ulid().toLowerCase(); const ext = mime.extension(emoji.type); - const emojiPath = path + '/' + exportId + (ext ? '.' + ext : ''); + const fileName = emoji.name + (ext ? '.' + ext : ''); + const emojiPath = path + '/' + fileName; fs.writeFileSync(emojiPath, '', 'binary'); let downloaded = false; try { - await downloadUrl(emoji.url, emojiPath); + await downloadUrl(emoji.originalUrl, emojiPath); downloaded = true; } catch (e) { // TODO: 何度か再試行 logger.error(e); } + if (!downloaded) { + fs.unlinkSync(emojiPath); + } + const content = JSON.stringify({ - id: exportId, + fileName: fileName, downloaded: downloaded, emoji: emoji, }); @@ -106,7 +111,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi logger.succ(`Exported to: ${archivePath}`); const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.zip'; - const driveFile = await addFile(user, archivePath, fileName, null, null, true); + const driveFile = await addFile({ user, path: archivePath, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); cleanup(); diff --git a/packages/backend/src/queue/processors/db/export-following.ts b/packages/backend/src/queue/processors/db/export-following.ts index fbb9e25247..06572acec1 100644 --- a/packages/backend/src/queue/processors/db/export-following.ts +++ b/packages/backend/src/queue/processors/db/export-following.ts @@ -3,7 +3,7 @@ import * as tmp from 'tmp'; import * as fs from 'fs'; import { queueLogger } from '../../logger'; -import addFile from '@/services/drive/add-file'; +import { addFile } from '@/services/drive/add-file'; import * as dateFormat from 'dateformat'; import { getFullApAccount } from '@/misc/convert-host'; import { Users, Followings, Mutings } from '@/models/index'; @@ -87,7 +87,7 @@ export async function exportFollowing(job: Bull.Job<DbUserJobData>, done: () => logger.succ(`Exported to: ${path}`); const fileName = 'following-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; - const driveFile = await addFile(user, path, fileName, null, null, true); + const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); cleanup(); diff --git a/packages/backend/src/queue/processors/db/export-mute.ts b/packages/backend/src/queue/processors/db/export-mute.ts index 0b1fd24fe0..4a856f8ef9 100644 --- a/packages/backend/src/queue/processors/db/export-mute.ts +++ b/packages/backend/src/queue/processors/db/export-mute.ts @@ -3,7 +3,7 @@ import * as tmp from 'tmp'; import * as fs from 'fs'; import { queueLogger } from '../../logger'; -import addFile from '@/services/drive/add-file'; +import { addFile } from '@/services/drive/add-file'; import * as dateFormat from 'dateformat'; import { getFullApAccount } from '@/misc/convert-host'; import { Users, Mutings } from '@/models/index'; @@ -86,7 +86,7 @@ export async function exportMute(job: Bull.Job<DbUserJobData>, done: any): Promi logger.succ(`Exported to: ${path}`); const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; - const driveFile = await addFile(user, path, fileName, null, null, true); + const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); cleanup(); diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts index e64e763513..305abf44cf 100644 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ b/packages/backend/src/queue/processors/db/export-notes.ts @@ -3,7 +3,7 @@ import * as tmp from 'tmp'; import * as fs from 'fs'; import { queueLogger } from '../../logger'; -import addFile from '@/services/drive/add-file'; +import { addFile } from '@/services/drive/add-file'; import * as dateFormat from 'dateformat'; import { Users, Notes, Polls } from '@/models/index'; import { MoreThan } from 'typeorm'; @@ -95,7 +95,7 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom logger.succ(`Exported to: ${path}`); const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json'; - const driveFile = await addFile(user, path, fileName, null, null, true); + const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); cleanup(); diff --git a/packages/backend/src/queue/processors/db/export-user-lists.ts b/packages/backend/src/queue/processors/db/export-user-lists.ts index 44a8f9f671..f907cf9526 100644 --- a/packages/backend/src/queue/processors/db/export-user-lists.ts +++ b/packages/backend/src/queue/processors/db/export-user-lists.ts @@ -3,7 +3,7 @@ import * as tmp from 'tmp'; import * as fs from 'fs'; import { queueLogger } from '../../logger'; -import addFile from '@/services/drive/add-file'; +import { addFile } from '@/services/drive/add-file'; import * as dateFormat from 'dateformat'; import { getFullApAccount } from '@/misc/convert-host'; import { Users, UserLists, UserListJoinings } from '@/models/index'; @@ -63,7 +63,7 @@ export async function exportUserLists(job: Bull.Job<DbUserJobData>, done: any): logger.succ(`Exported to: ${path}`); const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; - const driveFile = await addFile(user, path, fileName, null, null, true); + const driveFile = await addFile({ user, path, name: fileName, force: true }); logger.succ(`Exported to: ${driveFile.id}`); cleanup(); diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts new file mode 100644 index 0000000000..04e93671ed --- /dev/null +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -0,0 +1,85 @@ +import * as Bull from 'bull'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +const unzipper = require('unzipper'); +import { getConnection } from 'typeorm'; + +import { queueLogger } from '../../logger'; +import { downloadUrl } from '@/misc/download-url'; +import { DriveFiles, Emojis } from '@/models/index'; +import { DbUserImportJobData } from '@/queue/types'; +import { addFile } from '@/services/drive/add-file'; +import { genId } from '@/misc/gen-id'; + +const logger = queueLogger.createSubLogger('import-custom-emojis'); + +// TODO: 名前衝突時の動作を選べるようにする +export async function importCustomEmojis(job: Bull.Job<DbUserImportJobData>, done: any): Promise<void> { + logger.info(`Importing custom emojis ...`); + + const file = await DriveFiles.findOne({ + id: job.data.fileId, + }); + if (file == null) { + done(); + return; + } + + // Create temp dir + const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => { + tmp.dir((e, path, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + logger.info(`Temp dir is ${path}`); + + const destPath = path + '/emojis.zip'; + + try { + fs.writeFileSync(destPath, '', 'binary'); + await downloadUrl(file.url, destPath); + } catch (e) { // TODO: 何度か再試行 + logger.error(e); + throw e; + } + + const outputPath = path + '/emojis'; + const unzipStream = fs.createReadStream(destPath); + const extractor = unzipper.Extract({ path: outputPath }); + extractor.on('close', async () => { + const metaRaw = fs.readFileSync(outputPath + '/meta.json', 'utf-8'); + const meta = JSON.parse(metaRaw); + + for (const record of meta.emojis) { + if (!record.downloaded) continue; + const emojiInfo = record.emoji; + const emojiPath = outputPath + '/' + record.fileName; + await Emojis.delete({ + name: emojiInfo.name, + }); + const driveFile = await addFile({ user: null, path: emojiPath, name: record.fileName, force: true }); + const emoji = await Emojis.insert({ + id: genId(), + updatedAt: new Date(), + name: emojiInfo.name, + category: emojiInfo.category, + host: null, + aliases: emojiInfo.aliases, + originalUrl: driveFile.url, + publicUrl: driveFile.webpublicUrl ?? driveFile.url, + type: driveFile.webpublicType ?? driveFile.type, + }).then(x => Emojis.findOneOrFail(x.identifiers[0])); + } + + await getConnection().queryResultCache!.remove(['meta_emojis']); + + cleanup(); + + logger.succ('Imported'); + done(); + }); + unzipStream.pipe(extractor); + logger.succ(`Unzipping to ${outputPath}`); +} diff --git a/packages/backend/src/queue/processors/db/import-user-lists.ts b/packages/backend/src/queue/processors/db/import-user-lists.ts index 8245010de0..e060e86dd8 100644 --- a/packages/backend/src/queue/processors/db/import-user-lists.ts +++ b/packages/backend/src/queue/processors/db/import-user-lists.ts @@ -46,13 +46,13 @@ export async function importUserLists(job: Bull.Job<DbUserImportJobData>, done: }); if (list == null) { - list = await UserLists.save({ + list = await UserLists.insert({ id: genId(), createdAt: new Date(), userId: user.id, name: listName, userIds: [], - }); + }).then(x => UserLists.findOneOrFail(x.identifiers[0])); } let target = isSelfHost(host!) ? await Users.findOne({ diff --git a/packages/backend/src/queue/processors/db/index.ts b/packages/backend/src/queue/processors/db/index.ts index 1542f401ef..5fffa378f5 100644 --- a/packages/backend/src/queue/processors/db/index.ts +++ b/packages/backend/src/queue/processors/db/index.ts @@ -12,6 +12,7 @@ import { importUserLists } from './import-user-lists'; import { deleteAccount } from './delete-account'; import { importMuting } from './import-muting'; import { importBlocking } from './import-blocking'; +import { importCustomEmojis } from './import-custom-emojis'; const jobs = { deleteDriveFiles, @@ -25,6 +26,7 @@ const jobs = { importMuting, importBlocking, importUserLists, + importCustomEmojis, deleteAccount, } as Record<string, Bull.ProcessCallbackFunction<DbJobData> | Bull.ProcessPromiseFunction<DbJobData>>; |