summaryrefslogtreecommitdiff
path: root/packages/backend
diff options
context:
space:
mode:
authorMar0xy <marie@kaifa.ch>2023-11-07 03:31:47 +0100
committerMar0xy <marie@kaifa.ch>2023-11-07 03:31:47 +0100
commite9b6ed941bdfb3b94a97a4a8ee55ddf62ff9abef (patch)
treea074e5eca671c04a74aee760cb4a012e28525f45 /packages/backend
parentadd: locales for mfm play button and dialog (diff)
downloadsharkey-e9b6ed941bdfb3b94a97a4a8ee55ddf62ff9abef.tar.gz
sharkey-e9b6ed941bdfb3b94a97a4a8ee55ddf62ff9abef.tar.bz2
sharkey-e9b6ed941bdfb3b94a97a4a8ee55ddf62ff9abef.zip
add: endpoint and processor for account data export
Diffstat (limited to 'packages/backend')
-rw-r--r--packages/backend/src/core/QueueService.ts10
-rw-r--r--packages/backend/src/queue/QueueProcessorModule.ts2
-rw-r--r--packages/backend/src/queue/QueueProcessorService.ts3
-rw-r--r--packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts767
-rw-r--r--packages/backend/src/queue/types.ts1
-rw-r--r--packages/backend/src/server/api/EndpointsModule.ts4
-rw-r--r--packages/backend/src/server/api/endpoints.ts2
-rw-r--r--packages/backend/src/server/api/endpoints/i/export-data.ts35
8 files changed, 824 insertions, 0 deletions
diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts
index be378a899b..c5830168b8 100644
--- a/packages/backend/src/core/QueueService.ts
+++ b/packages/backend/src/core/QueueService.ts
@@ -165,6 +165,16 @@ export class QueueService {
}
@bindThis
+ public createExportAccountDataJob(user: ThinUser) {
+ return this.dbQueue.add('exportAccountData', {
+ user: { id: user.id },
+ }, {
+ removeOnComplete: true,
+ removeOnFail: true,
+ });
+ }
+
+ @bindThis
public createExportNotesJob(user: ThinUser) {
return this.dbQueue.add('exportNotes', {
user: { id: user.id },
diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts
index e6327002c5..5c61eb9e98 100644
--- a/packages/backend/src/queue/QueueProcessorModule.ts
+++ b/packages/backend/src/queue/QueueProcessorModule.ts
@@ -19,6 +19,7 @@ import { CleanRemoteFilesProcessorService } from './processors/CleanRemoteFilesP
import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js';
import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
import { DeleteFileProcessorService } from './processors/DeleteFileProcessorService.js';
+import { ExportAccountDataProcessorService } from './processors/ExportAccountDataProcessorService.js';
import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js';
import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
@@ -51,6 +52,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor
CheckExpiredMutingsProcessorService,
CleanProcessorService,
DeleteDriveFilesProcessorService,
+ ExportAccountDataProcessorService,
ExportCustomEmojisProcessorService,
ExportNotesProcessorService,
ExportFavoritesProcessorService,
diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts
index 5201bfed8e..7e45509fbf 100644
--- a/packages/backend/src/queue/QueueProcessorService.ts
+++ b/packages/backend/src/queue/QueueProcessorService.ts
@@ -14,6 +14,7 @@ import { EndedPollNotificationProcessorService } from './processors/EndedPollNot
import { DeliverProcessorService } from './processors/DeliverProcessorService.js';
import { InboxProcessorService } from './processors/InboxProcessorService.js';
import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
+import { ExportAccountDataProcessorService } from './processors/ExportAccountDataProcessorService.js';
import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
@@ -89,6 +90,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
private deliverProcessorService: DeliverProcessorService,
private inboxProcessorService: InboxProcessorService,
private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService,
+ private exportAccountDataProcessorService: ExportAccountDataProcessorService,
private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService,
private exportNotesProcessorService: ExportNotesProcessorService,
private exportFavoritesProcessorService: ExportFavoritesProcessorService,
@@ -162,6 +164,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => {
switch (job.name) {
case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job);
+ case 'exportAccountData': return this.exportAccountDataProcessorService.process(job);
case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job);
case 'exportNotes': return this.exportNotesProcessorService.process(job);
case 'exportFavorites': return this.exportFavoritesProcessorService.process(job);
diff --git a/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts b/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts
new file mode 100644
index 0000000000..f85f757fa1
--- /dev/null
+++ b/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts
@@ -0,0 +1,767 @@
+/* eslint-disable no-constant-condition */
+/* eslint-disable @typescript-eslint/no-unnecessary-condition */
+
+import * as fs from 'node:fs';
+import { Inject, Injectable } from '@nestjs/common';
+import { In, IsNull, MoreThan, Not } from 'typeorm';
+import { format as dateFormat } from 'date-fns';
+import mime from 'mime-types';
+import archiver from 'archiver';
+import { DI } from '@/di-symbols.js';
+import type { AntennasRepository, BlockingsRepository, DriveFilesRepository, FollowingsRepository, MiBlocking, MiFollowing, MiMuting, MiNote, MiNoteFavorite, MiPoll, MiUser, MutingsRepository, NoteFavoritesRepository, NotesRepository, PollsRepository, UserListMembershipsRepository, UserListsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js';
+import type { Config } from '@/config.js';
+import type Logger from '@/logger.js';
+import { DriveService } from '@/core/DriveService.js';
+import { IdService } from '@/core/IdService.js';
+import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
+import { createTemp, createTempDir } from '@/misc/create-temp.js';
+import { bindThis } from '@/decorators.js';
+import { Packed } from '@/misc/json-schema.js';
+import { UtilityService } from '@/core/UtilityService.js';
+import { DownloadService } from '@/core/DownloadService.js';
+import { QueueLoggerService } from '../QueueLoggerService.js';
+import type * as Bull from 'bullmq';
+
+@Injectable()
+export class ExportAccountDataProcessorService {
+ private logger: Logger;
+
+ constructor(
+ @Inject(DI.config)
+ private config: Config,
+
+ @Inject(DI.usersRepository)
+ private usersRepository: UsersRepository,
+
+ @Inject(DI.userProfilesRepository)
+ private userProfilesRepository: UserProfilesRepository,
+
+ @Inject(DI.notesRepository)
+ private notesRepository: NotesRepository,
+
+ @Inject(DI.noteFavoritesRepository)
+ private noteFavoritesRepository: NoteFavoritesRepository,
+
+ @Inject(DI.pollsRepository)
+ private pollsRepository: PollsRepository,
+
+ @Inject(DI.followingsRepository)
+ private followingsRepository: FollowingsRepository,
+
+ @Inject(DI.mutingsRepository)
+ private mutingsRepository: MutingsRepository,
+
+ @Inject(DI.blockingsRepository)
+ private blockingsRepository: BlockingsRepository,
+
+ @Inject(DI.driveFilesRepository)
+ private driveFilesRepository: DriveFilesRepository,
+
+ @Inject(DI.antennasRepository)
+ private antennasRepository: AntennasRepository,
+
+ @Inject(DI.userListsRepository)
+ private userListsRepository: UserListsRepository,
+
+ @Inject(DI.userListMembershipsRepository)
+ private userListMembershipsRepository: UserListMembershipsRepository,
+
+ private utilityService: UtilityService,
+ private driveService: DriveService,
+ private idService: IdService,
+ private driveFileEntityService: DriveFileEntityService,
+ private downloadService: DownloadService,
+ private queueLoggerService: QueueLoggerService,
+ ) {
+ this.logger = this.queueLoggerService.logger.createSubLogger('export-account-data');
+ }
+
+ @bindThis
+ public async process(job: Bull.Job): Promise<void> {
+ this.logger.info('Exporting Account Data...');
+
+ const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
+ if (user == null) {
+ return;
+ }
+
+ const profile = await this.userProfilesRepository.findOneBy({ userId: job.data.user.id });
+ if (profile == null) {
+ return;
+ }
+
+ const [path, cleanup] = await createTempDir();
+
+ this.logger.info(`Temp dir is ${path}`);
+
+ // User Export
+
+ const userPath = path + '/user.json';
+
+ fs.writeFileSync(userPath, '', 'utf-8');
+
+ const userStream = fs.createWriteStream(userPath, { flags: 'a' });
+
+ const writeUser = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ userStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ await writeUser(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","user":[`);
+
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+ const { host, uri, sharedInbox, followersUri, lastFetchedAt, inbox, ...userTrimmed } = user;
+
+ await writeUser(JSON.stringify(userTrimmed));
+
+ await writeUser(']}');
+
+ userStream.end();
+
+ // Profile Export
+
+ const profilePath = path + '/profile.json';
+
+ fs.writeFileSync(profilePath, '', 'utf-8');
+
+ const profileStream = fs.createWriteStream(profilePath, { flags: 'a' });
+
+ const writeProfile = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ profileStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+ const { emailVerifyCode, twoFactorBackupSecret, twoFactorSecret, password, twoFactorTempSecret, userHost, ...profileTrimmed } = profile;
+
+ await writeProfile(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","profile":[`);
+
+ await writeProfile(JSON.stringify(profileTrimmed));
+
+ await writeProfile(']}');
+
+ profileStream.end();
+
+ // Note Export
+
+ const notesPath = path + '/notes.json';
+
+ fs.writeFileSync(notesPath, '', 'utf-8');
+
+ const notesStream = fs.createWriteStream(notesPath, { flags: 'a' });
+
+ const writeNotes = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ notesStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ await writeNotes(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","notes":[`);
+
+ let noteCursor: MiNote['id'] | null = null;
+ let exportedNotesCount = 0;
+
+ while (true) {
+ const notes = await this.notesRepository.find({
+ where: {
+ userId: user.id,
+ ...(noteCursor ? { id: MoreThan(noteCursor) } : {}),
+ },
+ take: 100,
+ order: {
+ id: 1,
+ },
+ }) as MiNote[];
+
+ if (notes.length === 0) {
+ break;
+ }
+
+ noteCursor = notes.at(-1)?.id ?? null;
+
+ for (const note of notes) {
+ let poll: MiPoll | undefined;
+ if (note.hasPoll) {
+ poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id });
+ }
+ const files = await this.driveFileEntityService.packManyByIds(note.fileIds);
+ const content = JSON.stringify(this.noteSerialize(note, poll, files));
+ const isFirst = exportedNotesCount === 0;
+ await writeNotes(isFirst ? content : ',\n' + content);
+ exportedNotesCount++;
+ }
+ }
+
+ await writeNotes(']}');
+
+ notesStream.end();
+
+ // Following Export
+
+ const followingsPath = path + '/followings.json';
+
+ fs.writeFileSync(followingsPath, '', 'utf-8');
+
+ const followingStream = fs.createWriteStream(followingsPath, { flags: 'a' });
+
+ const writeFollowing = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ followingStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ await writeFollowing(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","followings":[`);
+
+ let followingsCursor: MiFollowing['id'] | null = null;
+ let exportedFollowingsCount = 0;
+
+ const mutings = await this.mutingsRepository.findBy({
+ muterId: user.id,
+ });
+
+ while (true) {
+ const followings = await this.followingsRepository.find({
+ where: {
+ followerId: user.id,
+ ...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}),
+ ...(followingsCursor ? { id: MoreThan(followingsCursor) } : {}),
+ },
+ take: 100,
+ order: {
+ id: 1,
+ },
+ }) as MiFollowing[];
+
+ if (followings.length === 0) {
+ break;
+ }
+
+ followingsCursor = followings.at(-1)?.id ?? null;
+
+ for (const following of followings) {
+ const u = await this.usersRepository.findOneBy({ id: following.followeeId });
+ if (u == null) {
+ continue;
+ }
+
+ if (u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) {
+ continue;
+ }
+
+ const isFirst = exportedFollowingsCount === 0;
+ const content = this.utilityService.getFullApAccount(u.username, u.host);
+ await writeFollowing(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
+ exportedFollowingsCount++;
+ }
+ }
+
+ await writeFollowing(']}');
+
+ followingStream.end();
+
+ // Followers Export
+
+ const followersPath = path + '/followers.json';
+
+ fs.writeFileSync(followersPath, '', 'utf-8');
+
+ const followerStream = fs.createWriteStream(followersPath, { flags: 'a' });
+
+ const writeFollowers = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ followerStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ await writeFollowers(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","followers":[`);
+
+ let followersCursor: MiFollowing['id'] | null = null;
+ let exportedFollowersCount = 0;
+
+ while (true) {
+ const followers = await this.followingsRepository.find({
+ where: {
+ followeeId: user.id,
+ ...(followersCursor ? { id: MoreThan(followersCursor) } : {}),
+ },
+ take: 100,
+ order: {
+ id: 1,
+ },
+ }) as MiFollowing[];
+
+ if (followers.length === 0) {
+ break;
+ }
+
+ followersCursor = followers.at(-1)?.id ?? null;
+
+ for (const follower of followers) {
+ const u = await this.usersRepository.findOneBy({ id: follower.followerId });
+ if (u == null) {
+ continue;
+ }
+
+ const isFirst = exportedFollowersCount === 0;
+ const content = this.utilityService.getFullApAccount(u.username, u.host);
+ await writeFollowers(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
+ exportedFollowersCount++;
+ }
+ }
+
+ await writeFollowers(']}');
+
+ followerStream.end();
+
+ // Drive Export
+
+ const filesPath = path + '/drive.json';
+
+ fs.writeFileSync(filesPath, '', 'utf-8');
+
+ const filesStream = fs.createWriteStream(filesPath, { flags: 'a' });
+
+ const writeDrive = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ filesStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ fs.mkdirSync(`${path}/files`);
+
+ await writeDrive(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","drive":[`);
+
+ const driveFiles = await this.driveFilesRepository.find({ where: { userId: user.id } });
+
+ for (const file of driveFiles) {
+ const ext = mime.extension(file.type);
+ const fileName = file.name + '.' + ext;
+ const filePath = path + '/files/' + fileName;
+ fs.writeFileSync(filePath, '', 'binary');
+ let downloaded = false;
+
+ try {
+ await this.downloadService.downloadUrl(file.url, filePath);
+ downloaded = true;
+ } catch (e) {
+ this.logger.error(e instanceof Error ? e : new Error(e as string));
+ }
+
+ if (!downloaded) {
+ fs.unlinkSync(filePath);
+ }
+
+ const content = JSON.stringify({
+ fileName: fileName,
+ file: file,
+ });
+ const isFirst = driveFiles.indexOf(file) === 0;
+
+ await writeDrive(isFirst ? content : ',\n' + content);
+ }
+
+ await writeDrive(']}');
+
+ filesStream.end();
+
+ // Muting Export
+
+ const mutingPath = path + '/mutings.json';
+
+ fs.writeFileSync(mutingPath, '', 'utf-8');
+
+ const mutingStream = fs.createWriteStream(mutingPath, { flags: 'a' });
+
+ const writeMuting = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ mutingStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ await writeMuting(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","mutings":[`);
+
+ let exportedMutingCount = 0;
+ let mutingCursor: MiMuting['id'] | null = null;
+
+ while (true) {
+ const mutes = await this.mutingsRepository.find({
+ where: {
+ muterId: user.id,
+ expiresAt: IsNull(),
+ ...(mutingCursor ? { id: MoreThan(mutingCursor) } : {}),
+ },
+ take: 100,
+ order: {
+ id: 1,
+ },
+ });
+
+ if (mutes.length === 0) {
+ break;
+ }
+
+ mutingCursor = mutes.at(-1)?.id ?? null;
+
+ for (const mute of mutes) {
+ const u = await this.usersRepository.findOneBy({ id: mute.muteeId });
+
+ if (u == null) {
+ exportedMutingCount++; continue;
+ }
+
+ const content = this.utilityService.getFullApAccount(u.username, u.host);
+ const isFirst = exportedMutingCount === 0;
+ await writeMuting(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
+ exportedMutingCount++;
+ }
+ }
+
+ await writeMuting(']}');
+
+ mutingStream.end();
+
+ // Blockings Export
+
+ const blockingPath = path + '/blockings.json';
+
+ fs.writeFileSync(blockingPath, '', 'utf-8');
+
+ const blockingStream = fs.createWriteStream(blockingPath, { flags: 'a' });
+
+ const writeBlocking = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ blockingStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ await writeBlocking(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","blockings":[`);
+
+ let exportedBlockingCount = 0;
+ let blockingCursor: MiBlocking['id'] | null = null;
+
+ while (true) {
+ const blockings = await this.blockingsRepository.find({
+ where: {
+ blockerId: user.id,
+ ...(blockingCursor ? { id: MoreThan(blockingCursor) } : {}),
+ },
+ take: 100,
+ order: {
+ id: 1,
+ },
+ });
+
+ if (blockings.length === 0) {
+ break;
+ }
+
+ blockingCursor = blockings.at(-1)?.id ?? null;
+
+ for (const block of blockings) {
+ const u = await this.usersRepository.findOneBy({ id: block.blockeeId });
+
+ if (u == null) {
+ exportedBlockingCount++; continue;
+ }
+
+ const content = this.utilityService.getFullApAccount(u.username, u.host);
+ const isFirst = exportedBlockingCount === 0;
+ await writeBlocking(isFirst ? `"${content}"` : ',\n' + `"${content}"`);
+ exportedBlockingCount++;
+ }
+ }
+
+ await writeBlocking(']}');
+
+ blockingStream.end();
+
+ // Favorites export
+
+ const favoritePath = path + '/favorites.json';
+
+ fs.writeFileSync(favoritePath, '', 'utf-8');
+
+ const favoriteStream = fs.createWriteStream(favoritePath, { flags: 'a' });
+
+ const writeFavorite = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ favoriteStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ await writeFavorite(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","favorites":[`);
+
+ let exportedFavoritesCount = 0;
+ let favoriteCursor: MiNoteFavorite['id'] | null = null;
+
+ while (true) {
+ const favorites = await this.noteFavoritesRepository.find({
+ where: {
+ userId: user.id,
+ ...(favoriteCursor ? { id: MoreThan(favoriteCursor) } : {}),
+ },
+ take: 100,
+ order: {
+ id: 1,
+ },
+ relations: ['note', 'note.user'],
+ }) as (MiNoteFavorite & { note: MiNote & { user: MiUser } })[];
+
+ if (favorites.length === 0) {
+ break;
+ }
+
+ favoriteCursor = favorites.at(-1)?.id ?? null;
+
+ for (const favorite of favorites) {
+ let poll: MiPoll | undefined;
+ if (favorite.note.hasPoll) {
+ poll = await this.pollsRepository.findOneByOrFail({ noteId: favorite.note.id });
+ }
+ const content = JSON.stringify(this.favoriteSerialize(favorite, poll));
+ const isFirst = exportedFavoritesCount === 0;
+ await writeFavorite(isFirst ? content : ',\n' + content);
+ exportedFavoritesCount++;
+ }
+ }
+
+ await writeFavorite(']}');
+
+ favoriteStream.end();
+
+ // Antennas export
+
+ const antennaPath = path + '/antennas.json';
+
+ fs.writeFileSync(antennaPath, '', 'utf-8');
+
+ const antennaStream = fs.createWriteStream(antennaPath, { flags: 'a' });
+
+ const writeAntenna = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ antennaStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ await writeAntenna(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","antennas":[`);
+
+ const antennas = await this.antennasRepository.findBy({ userId: user.id });
+
+ for (const [index, antenna] of antennas.entries()) {
+ let users: MiUser[] | undefined;
+ if (antenna.userListId !== null) {
+ const memberships = await this.userListMembershipsRepository.findBy({ userListId: antenna.userListId });
+ users = await this.usersRepository.findBy({
+ id: In(memberships.map(j => j.userId)),
+ });
+ }
+
+ await writeAntenna(JSON.stringify({
+ name: antenna.name,
+ src: antenna.src,
+ keywords: antenna.keywords,
+ excludeKeywords: antenna.excludeKeywords,
+ users: antenna.users,
+ userListAccts: typeof users !== 'undefined' ? users.map((u) => {
+ return this.utilityService.getFullApAccount(u.username, u.host); // acct
+ }) : null,
+ caseSensitive: antenna.caseSensitive,
+ localOnly: antenna.localOnly,
+ withReplies: antenna.withReplies,
+ withFile: antenna.withFile,
+ notify: antenna.notify,
+ }));
+
+ if (antennas.length - 1 !== index) {
+ await writeAntenna(', ');
+ }
+ }
+
+ await writeAntenna(']}');
+
+ antennaStream.end();
+
+ // Lists export
+
+ const listPath = path + '/lists.csv';
+
+ fs.writeFileSync(listPath, '', 'utf-8');
+
+ const listStream = fs.createWriteStream(listPath, { flags: 'a' });
+
+ const writeList = (text: string): Promise<void> => {
+ return new Promise<void>((res, rej) => {
+ listStream.write(text, err => {
+ if (err) {
+ this.logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ };
+
+ const lists = await this.userListsRepository.findBy({
+ userId: user.id,
+ });
+
+ for (const list of lists) {
+ const memberships = await this.userListMembershipsRepository.findBy({ userListId: list.id });
+ const users = await this.usersRepository.findBy({
+ id: In(memberships.map(j => j.userId)),
+ });
+
+ for (const u of users) {
+ const acct = this.utilityService.getFullApAccount(u.username, u.host);
+ const content = `${list.name},${acct}`;
+ await writeList(content + '\n');
+ }
+ }
+
+ listStream.end();
+
+ // Create archive
+ await new Promise<void>(async (resolve) => {
+ const [archivePath, archiveCleanup] = await createTemp();
+ const archiveStream = fs.createWriteStream(archivePath);
+ const archive = archiver('zip', {
+ zlib: { level: 0 },
+ });
+ archiveStream.on('close', async () => {
+ this.logger.succ(`Exported to: ${archivePath}`);
+
+ const fileName = 'data-request-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip';
+ const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true });
+
+ this.logger.succ(`Exported to: ${driveFile.id}`);
+ cleanup();
+ archiveCleanup();
+ resolve();
+ });
+ archive.pipe(archiveStream);
+ archive.directory(path, false);
+ archive.finalize();
+ });
+ }
+
+ private noteSerialize(note: MiNote, poll: MiPoll | null = null, files: Packed<'DriveFile'>[]): Record<string, unknown> {
+ return {
+ id: note.id,
+ text: note.text,
+ createdAt: this.idService.parse(note.id).date.toISOString(),
+ fileIds: note.fileIds,
+ files: files,
+ replyId: note.replyId,
+ renoteId: note.renoteId,
+ poll: poll,
+ cw: note.cw,
+ visibility: note.visibility,
+ visibleUserIds: note.visibleUserIds,
+ localOnly: note.localOnly,
+ reactionAcceptance: note.reactionAcceptance,
+ };
+ }
+
+ private favoriteSerialize(favorite: MiNoteFavorite & { note: MiNote & { user: MiUser } }, poll: MiPoll | null = null): Record<string, unknown> {
+ return {
+ id: favorite.id,
+ createdAt: this.idService.parse(favorite.id).date.toISOString(),
+ note: {
+ id: favorite.note.id,
+ text: favorite.note.text,
+ createdAt: this.idService.parse(favorite.note.id).date.toISOString(),
+ fileIds: favorite.note.fileIds,
+ replyId: favorite.note.replyId,
+ renoteId: favorite.note.renoteId,
+ poll: poll,
+ cw: favorite.note.cw,
+ visibility: favorite.note.visibility,
+ visibleUserIds: favorite.note.visibleUserIds,
+ localOnly: favorite.note.localOnly,
+ reactionAcceptance: favorite.note.reactionAcceptance,
+ uri: favorite.note.uri,
+ url: favorite.note.url,
+ user: {
+ id: favorite.note.user.id,
+ name: favorite.note.user.name,
+ username: favorite.note.user.username,
+ host: favorite.note.user.host,
+ uri: favorite.note.user.uri,
+ },
+ },
+ };
+ }
+}
diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts
index 9330c01528..94a95d8b90 100644
--- a/packages/backend/src/queue/types.ts
+++ b/packages/backend/src/queue/types.ts
@@ -39,6 +39,7 @@ export type DbJobData<T extends keyof DbJobMap> = DbJobMap[T];
export type DbJobMap = {
deleteDriveFiles: DbJobDataWithUser;
+ exportAccountData: DbJobDataWithUser;
exportCustomEmojis: DbJobDataWithUser;
exportAntennas: DBExportAntennasData;
exportNotes: DbJobDataWithUser;
diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts
index fde35ffd32..09a8d8c37d 100644
--- a/packages/backend/src/server/api/EndpointsModule.ts
+++ b/packages/backend/src/server/api/EndpointsModule.ts
@@ -206,6 +206,7 @@ import * as ep___i_authorizedApps from './endpoints/i/authorized-apps.js';
import * as ep___i_claimAchievement from './endpoints/i/claim-achievement.js';
import * as ep___i_changePassword from './endpoints/i/change-password.js';
import * as ep___i_deleteAccount from './endpoints/i/delete-account.js';
+import * as ep___i_exportData from './endpoints/i/export-data.js';
import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
import * as ep___i_exportMute from './endpoints/i/export-mute.js';
@@ -573,6 +574,7 @@ const $i_authorizedApps: Provider = { provide: 'ep:i/authorized-apps', useClass:
const $i_claimAchievement: Provider = { provide: 'ep:i/claim-achievement', useClass: ep___i_claimAchievement.default };
const $i_changePassword: Provider = { provide: 'ep:i/change-password', useClass: ep___i_changePassword.default };
const $i_deleteAccount: Provider = { provide: 'ep:i/delete-account', useClass: ep___i_deleteAccount.default };
+const $i_exportData: Provider = { provide: 'ep:i/export-data', useClass: ep___i_exportData.default };
const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass: ep___i_exportBlocking.default };
const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default };
const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default };
@@ -944,6 +946,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de
$i_claimAchievement,
$i_changePassword,
$i_deleteAccount,
+ $i_exportData,
$i_exportBlocking,
$i_exportFollowing,
$i_exportMute,
@@ -1309,6 +1312,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de
$i_claimAchievement,
$i_changePassword,
$i_deleteAccount,
+ $i_exportData,
$i_exportBlocking,
$i_exportFollowing,
$i_exportMute,
diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts
index c1cabd33e9..527235264c 100644
--- a/packages/backend/src/server/api/endpoints.ts
+++ b/packages/backend/src/server/api/endpoints.ts
@@ -206,6 +206,7 @@ import * as ep___i_authorizedApps from './endpoints/i/authorized-apps.js';
import * as ep___i_claimAchievement from './endpoints/i/claim-achievement.js';
import * as ep___i_changePassword from './endpoints/i/change-password.js';
import * as ep___i_deleteAccount from './endpoints/i/delete-account.js';
+import * as ep___i_exportData from './endpoints/i/export-data.js';
import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
import * as ep___i_exportMute from './endpoints/i/export-mute.js';
@@ -571,6 +572,7 @@ const eps = [
['i/claim-achievement', ep___i_claimAchievement],
['i/change-password', ep___i_changePassword],
['i/delete-account', ep___i_deleteAccount],
+ ['i/export-data', ep___i_exportData],
['i/export-blocking', ep___i_exportBlocking],
['i/export-following', ep___i_exportFollowing],
['i/export-mute', ep___i_exportMute],
diff --git a/packages/backend/src/server/api/endpoints/i/export-data.ts b/packages/backend/src/server/api/endpoints/i/export-data.ts
new file mode 100644
index 0000000000..d9a1e087b9
--- /dev/null
+++ b/packages/backend/src/server/api/endpoints/i/export-data.ts
@@ -0,0 +1,35 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { Injectable } from '@nestjs/common';
+import ms from 'ms';
+import { Endpoint } from '@/server/api/endpoint-base.js';
+import { QueueService } from '@/core/QueueService.js';
+
+export const meta = {
+ secure: true,
+ requireCredential: true,
+ limit: {
+ duration: ms('3days'),
+ max: 1,
+ },
+} as const;
+
+export const paramDef = {
+ type: 'object',
+ properties: {},
+ required: [],
+} as const;
+
+@Injectable()
+export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
+ constructor(
+ private queueService: QueueService,
+ ) {
+ super(meta, paramDef, async (ps, me) => {
+ this.queueService.createExportAccountDataJob(me);
+ });
+ }
+}