From d415fd29a3b0fb7f337423fff8318ec393928875 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Sat, 6 Jan 2024 09:07:48 +0900 Subject: enhance(backend): ActivityPub Deliver queueでBodyを事前処理するように (#12916) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Pre-processing deliver body * CHANGELOG * ループ内で計算されると意味がないので * 同じ処理を同じ形に --------- Co-authored-by: まっちゃとーにゅ <17376330+u1-liquid@users.noreply.github.com> --- packages/backend/src/queue/processors/DeliverProcessorService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/backend/src/queue/processors') diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 4a1d9f28b4..675eccbe34 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -72,7 +72,7 @@ export class DeliverProcessorService { } try { - await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content); + await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, job.data.digest); // Update stats this.federatedInstanceService.fetch(host).then(i => { -- cgit v1.2.3-freya From 24645e3d3dbcdde5bda02c19d0358353e843734a Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Sat, 6 Jan 2024 09:40:08 +0900 Subject: enhance(backend): ActivityPub 周りで連合先から HTTP 429 Too Many Requests を受け取った際にジョブをリトライするように (#12917) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enhance(backend): ActivityPub 周りで HTTP 429 Too Many Requests を受け取った際にリトライするように * add to changelog --------- Co-authored-by: syuilo --- CHANGELOG.md | 1 + packages/backend/src/core/activitypub/ApInboxService.ts | 10 ++++++---- packages/backend/src/core/activitypub/models/ApNoteService.ts | 2 +- packages/backend/src/misc/status-error.ts | 2 ++ .../backend/src/queue/processors/DeliverProcessorService.ts | 2 +- packages/backend/src/queue/processors/InboxProcessorService.ts | 2 +- .../src/queue/processors/WebhookDeliverProcessorService.ts | 2 +- 7 files changed, 13 insertions(+), 8 deletions(-) (limited to 'packages/backend/src/queue/processors') diff --git a/CHANGELOG.md b/CHANGELOG.md index f4fe8de755..f7e1ac6a78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Enhance: チャンネルノートのピン留めをノートのメニューからできるよ ### Server +- Enhance: 連合先のレートリミットに引っかかった際にリトライするようになりました - Enhance: ActivityPub Deliver queueでBodyを事前処理するように (#12916) ## 2023.12.2 diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index baaab67e48..a0c63bdbf7 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -97,6 +97,8 @@ export class ApInboxService { } catch (err) { if (err instanceof Error || typeof err === 'string') { this.logger.error(err); + } else { + throw err; } } } @@ -256,7 +258,7 @@ export class ApInboxService { const targetUri = getApId(activity.object); - this.announceNote(actor, activity, targetUri); + await this.announceNote(actor, activity, targetUri); } @bindThis @@ -288,7 +290,7 @@ export class ApInboxService { } catch (err) { // 対象が4xxならスキップ if (err instanceof StatusError) { - if (err.isClientError) { + if (!err.isRetryable) { this.logger.warn(`Ignored announce target ${targetUri} - ${err.statusCode}`); return; } @@ -373,7 +375,7 @@ export class ApInboxService { }); if (isPost(object)) { - this.createNote(resolver, actor, object, false, activity); + await this.createNote(resolver, actor, object, false, activity); } else { this.logger.warn(`Unknown type: ${getApType(object)}`); } @@ -404,7 +406,7 @@ export class ApInboxService { await this.apNoteService.createNote(note, resolver, silent); return 'ok'; } catch (err) { - if (err instanceof StatusError && err.isClientError) { + if (err instanceof StatusError && !err.isRetryable) { return `skip ${err.statusCode}`; } else { throw err; diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 05d5ca15db..e3eccd5405 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -216,7 +216,7 @@ export class ApNoteService { return { status: 'ok', res }; } catch (e) { return { - status: (e instanceof StatusError && e.isClientError) ? 'permerror' : 'temperror', + status: (e instanceof StatusError && !e.isRetryable) ? 'permerror' : 'temperror', }; } }; diff --git a/packages/backend/src/misc/status-error.ts b/packages/backend/src/misc/status-error.ts index 4285685d24..be213088a8 100644 --- a/packages/backend/src/misc/status-error.ts +++ b/packages/backend/src/misc/status-error.ts @@ -7,6 +7,7 @@ export class StatusError extends Error { public statusCode: number; public statusMessage?: string; public isClientError: boolean; + public isRetryable: boolean; constructor(message: string, statusCode: number, statusMessage?: string) { super(message); @@ -14,5 +15,6 @@ export class StatusError extends Error { this.statusCode = statusCode; this.statusMessage = statusMessage; this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500; + this.isRetryable = !this.isClientError || this.statusCode === 429; } } diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 675eccbe34..64c3445552 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -111,7 +111,7 @@ export class DeliverProcessorService { if (res instanceof StatusError) { // 4xx - if (res.isClientError) { + if (!res.isRetryable) { // 相手が閉鎖していることを明示しているため、配送停止する if (job.data.isSharedInbox && res.statusCode === 410) { this.federatedInstanceService.fetch(host).then(i => { diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 89d4ea503e..50e7a1631e 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -85,7 +85,7 @@ export class InboxProcessorService { } catch (err) { // 対象が4xxならスキップ if (err instanceof StatusError) { - if (err.isClientError) { + if (!err.isRetryable) { throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); } throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`); diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts index a41f5565c8..7a0d533846 100644 --- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts @@ -71,7 +71,7 @@ export class WebhookDeliverProcessorService { if (res instanceof StatusError) { // 4xx - if (res.isClientError) { + if (!res.isRetryable) { throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); } -- cgit v1.2.3-freya From 2a9db983fcd79e1993d5ea5b03e4979c1a578d7d Mon Sep 17 00:00:00 2001 From: Kagami Sascha Rosylight Date: Sun, 7 Jan 2024 02:35:58 +0100 Subject: feat: export clips (#12931) * feat: export clips * Update CHANGELOG.md --- CHANGELOG.md | 1 + locales/index.d.ts | 1 + locales/ja-JP.yml | 1 + packages/backend/src/core/QueueService.ts | 10 + packages/backend/src/queue/QueueProcessorModule.ts | 2 + .../backend/src/queue/QueueProcessorService.ts | 3 + .../processors/ExportClipsProcessorService.ts | 206 +++++++++++++++++++++ packages/backend/src/server/api/EndpointsModule.ts | 4 + packages/backend/src/server/api/endpoints.ts | 4 +- .../src/server/api/endpoints/i/export-clips.ts | 35 ++++ packages/backend/test/e2e/exports.ts | 194 +++++++++++++++++++ packages/backend/test/utils.ts | 2 +- .../frontend/src/pages/settings/import-export.vue | 12 ++ 13 files changed, 473 insertions(+), 2 deletions(-) create mode 100644 packages/backend/src/queue/processors/ExportClipsProcessorService.ts create mode 100644 packages/backend/src/server/api/endpoints/i/export-clips.ts create mode 100644 packages/backend/test/e2e/exports.ts (limited to 'packages/backend/src/queue/processors') diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c27349f61..0d2fb4ccd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ ### Server - Enhance: 連合先のレートリミットに引っかかった際にリトライするようになりました - Enhance: ActivityPub Deliver queueでBodyを事前処理するように (#12916) +- Enhance: クリップをエクスポートできるように ## 2023.12.2 diff --git a/locales/index.d.ts b/locales/index.d.ts index 99bc0fc04f..75517fa2ad 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -2256,6 +2256,7 @@ export interface Locale { "_exportOrImport": { "allNotes": string; "favoritedNotes": string; + "clips": string; "followingList": string; "muteList": string; "blockingList": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 7cf5663a72..8b6b119d7e 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -2159,6 +2159,7 @@ _profile: _exportOrImport: allNotes: "全てのノート" favoritedNotes: "お気に入りにしたノート" + clips: "クリップ" followingList: "フォロー" muteList: "ミュート" blockingList: "ブロック" diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 4f99dee64e..dc3f248da4 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -182,6 +182,16 @@ export class QueueService { }); } + @bindThis + public createExportClipsJob(user: ThinUser) { + return this.dbQueue.add('exportClips', { + user: { id: user.id }, + }, { + removeOnComplete: true, + removeOnFail: true, + }); + } + @bindThis public createExportFavoritesJob(user: ThinUser) { return this.dbQueue.add('exportFavorites', { diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index e6327002c5..9c52c7d76a 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -24,6 +24,7 @@ import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmo import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js'; import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js'; import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js'; +import { ExportClipsProcessorService } from './processors/ExportClipsProcessorService.js'; import { ExportUserListsProcessorService } from './processors/ExportUserListsProcessorService.js'; import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js'; import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js'; @@ -53,6 +54,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor DeleteDriveFilesProcessorService, ExportCustomEmojisProcessorService, ExportNotesProcessorService, + ExportClipsProcessorService, ExportFavoritesProcessorService, ExportFollowingProcessorService, ExportMutingProcessorService, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index b872dd65f7..bcc1a69f80 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -16,6 +16,7 @@ import { InboxProcessorService } from './processors/InboxProcessorService.js'; import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js'; import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js'; import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js'; +import { ExportClipsProcessorService } from './processors/ExportClipsProcessorService.js'; import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js'; import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js'; import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js'; @@ -91,6 +92,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService, private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService, private exportNotesProcessorService: ExportNotesProcessorService, + private exportClipsProcessorService: ExportClipsProcessorService, private exportFavoritesProcessorService: ExportFavoritesProcessorService, private exportFollowingProcessorService: ExportFollowingProcessorService, private exportMutingProcessorService: ExportMutingProcessorService, @@ -164,6 +166,7 @@ export class QueueProcessorService implements OnApplicationShutdown { case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job); case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job); case 'exportNotes': return this.exportNotesProcessorService.process(job); + case 'exportClips': return this.exportClipsProcessorService.process(job); case 'exportFavorites': return this.exportFavoritesProcessorService.process(job); case 'exportFollowing': return this.exportFollowingProcessorService.process(job); case 'exportMuting': return this.exportMutingProcessorService.process(job); diff --git a/packages/backend/src/queue/processors/ExportClipsProcessorService.ts b/packages/backend/src/queue/processors/ExportClipsProcessorService.ts new file mode 100644 index 0000000000..5221497bd3 --- /dev/null +++ b/packages/backend/src/queue/processors/ExportClipsProcessorService.ts @@ -0,0 +1,206 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import * as fs from 'node:fs'; +import { Writable } from 'node:stream'; +import { Inject, Injectable, StreamableFile } from '@nestjs/common'; +import { MoreThan } from 'typeorm'; +import { format as dateFormat } from 'date-fns'; +import { DI } from '@/di-symbols.js'; +import type { ClipNotesRepository, ClipsRepository, MiClip, MiClipNote, MiUser, NotesRepository, PollsRepository, UsersRepository } from '@/models/_.js'; +import type Logger from '@/logger.js'; +import { DriveService } from '@/core/DriveService.js'; +import { createTemp } from '@/misc/create-temp.js'; +import type { MiPoll } from '@/models/Poll.js'; +import type { MiNote } from '@/models/Note.js'; +import { bindThis } from '@/decorators.js'; +import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; +import { Packed } from '@/misc/json-schema.js'; +import { IdService } from '@/core/IdService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbJobDataWithUser } from '../types.js'; + +@Injectable() +export class ExportClipsProcessorService { + private logger: Logger; + + constructor( + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + + @Inject(DI.pollsRepository) + private pollsRepository: PollsRepository, + + @Inject(DI.clipsRepository) + private clipsRepository: ClipsRepository, + + @Inject(DI.clipNotesRepository) + private clipNotesRepository: ClipNotesRepository, + + private driveService: DriveService, + private queueLoggerService: QueueLoggerService, + private idService: IdService, + ) { + this.logger = this.queueLoggerService.logger.createSubLogger('export-clips'); + } + + @bindThis + public async process(job: Bull.Job): Promise { + this.logger.info(`Exporting clips of ${job.data.user.id} ...`); + + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + // Create temp file + const [path, cleanup] = await createTemp(); + + this.logger.info(`Temp file is ${path}`); + + try { + const stream = Writable.toWeb(fs.createWriteStream(path, { flags: 'a' })); + const writer = stream.getWriter(); + writer.closed.catch(this.logger.error); + + await writer.write('['); + + await this.processClips(writer, user, job); + + await writer.write(']'); + await writer.close(); + + this.logger.succ(`Exported to: ${path}`); + + const fileName = 'clips-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; + const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' }); + + this.logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } + } + + async processClips(writer: WritableStreamDefaultWriter, user: MiUser, job: Bull.Job) { + let exportedClipsCount = 0; + let cursor: MiClip['id'] | null = null; + + while (true) { + const clips = await this.clipsRepository.find({ + where: { + userId: user.id, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }); + + if (clips.length === 0) { + job.updateProgress(100); + break; + } + + cursor = clips.at(-1)?.id ?? null; + + for (const clip of clips) { + // Stringify but remove the last `]}` + const content = JSON.stringify(this.serializeClip(clip)).slice(0, -2); + const isFirst = exportedClipsCount === 0; + await writer.write(isFirst ? content : ',\n' + content); + + await this.processClipNotes(writer, clip.id); + + await writer.write(']}'); + exportedClipsCount++; + } + + const total = await this.clipsRepository.countBy({ + userId: user.id, + }); + + job.updateProgress(exportedClipsCount / total); + } + } + + async processClipNotes(writer: WritableStreamDefaultWriter, clipId: string): Promise { + let exportedClipNotesCount = 0; + let cursor: MiClipNote['id'] | null = null; + + while (true) { + const clipNotes = await this.clipNotesRepository.find({ + where: { + clipId, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + relations: ['note', 'note.user'], + }) as (MiClipNote & { note: MiNote & { user: MiUser } })[]; + + if (clipNotes.length === 0) { + break; + } + + cursor = clipNotes.at(-1)?.id ?? null; + + for (const clipNote of clipNotes) { + let poll: MiPoll | undefined; + if (clipNote.note.hasPoll) { + poll = await this.pollsRepository.findOneByOrFail({ noteId: clipNote.note.id }); + } + const content = JSON.stringify(this.serializeClipNote(clipNote, poll)); + const isFirst = exportedClipNotesCount === 0; + await writer.write(isFirst ? content : ',\n' + content); + + exportedClipNotesCount++; + } + } + } + + private serializeClip(clip: MiClip): Record { + return { + id: clip.id, + name: clip.name, + description: clip.description, + lastClippedAt: clip.lastClippedAt?.toISOString(), + clipNotes: [], + }; + } + + private serializeClipNote(clip: MiClipNote & { note: MiNote & { user: MiUser } }, poll: MiPoll | undefined): Record { + return { + id: clip.id, + createdAt: this.idService.parse(clip.id).date.toISOString(), + note: { + id: clip.note.id, + text: clip.note.text, + createdAt: this.idService.parse(clip.note.id).date.toISOString(), + fileIds: clip.note.fileIds, + replyId: clip.note.replyId, + renoteId: clip.note.renoteId, + poll: poll, + cw: clip.note.cw, + visibility: clip.note.visibility, + visibleUserIds: clip.note.visibleUserIds, + localOnly: clip.note.localOnly, + reactionAcceptance: clip.note.reactionAcceptance, + uri: clip.note.uri, + url: clip.note.url, + user: { + id: clip.note.user.id, + name: clip.note.user.name, + username: clip.note.user.username, + host: clip.note.user.host, + uri: clip.note.user.uri, + }, + }, + }; + } +} diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index 86a64d7121..a3a9805444 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -208,6 +208,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js'; import * as ep___i_exportNotes from './endpoints/i/export-notes.js'; +import * as ep___i_exportClips from './endpoints/i/export-clips.js'; import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js'; import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js'; import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js'; @@ -569,6 +570,7 @@ const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass: const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default }; const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default }; const $i_exportNotes: Provider = { provide: 'ep:i/export-notes', useClass: ep___i_exportNotes.default }; +const $i_exportClips: Provider = { provide: 'ep:i/export-clips', useClass: ep___i_exportClips.default }; const $i_exportFavorites: Provider = { provide: 'ep:i/export-favorites', useClass: ep___i_exportFavorites.default }; const $i_exportUserLists: Provider = { provide: 'ep:i/export-user-lists', useClass: ep___i_exportUserLists.default }; const $i_exportAntennas: Provider = { provide: 'ep:i/export-antennas', useClass: ep___i_exportAntennas.default }; @@ -934,6 +936,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $i_exportFollowing, $i_exportMute, $i_exportNotes, + $i_exportClips, $i_exportFavorites, $i_exportUserLists, $i_exportAntennas, @@ -1293,6 +1296,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $i_exportFollowing, $i_exportMute, $i_exportNotes, + $i_exportClips, $i_exportFavorites, $i_exportUserLists, $i_exportAntennas, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 41232091c6..bd8aa4af72 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -3,8 +3,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import type { Schema } from '@/misc/json-schema.js'; import { permissions } from 'misskey-js'; +import type { Schema } from '@/misc/json-schema.js'; import { RolePolicies } from '@/core/RoleService.js'; import * as ep___admin_meta from './endpoints/admin/meta.js'; @@ -209,6 +209,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js'; import * as ep___i_exportNotes from './endpoints/i/export-notes.js'; +import * as ep___i_exportClips from './endpoints/i/export-clips.js'; import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js'; import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js'; import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js'; @@ -568,6 +569,7 @@ const eps = [ ['i/export-following', ep___i_exportFollowing], ['i/export-mute', ep___i_exportMute], ['i/export-notes', ep___i_exportNotes], + ['i/export-clips', ep___i_exportClips], ['i/export-favorites', ep___i_exportFavorites], ['i/export-user-lists', ep___i_exportUserLists], ['i/export-antennas', ep___i_exportAntennas], diff --git a/packages/backend/src/server/api/endpoints/i/export-clips.ts b/packages/backend/src/server/api/endpoints/i/export-clips.ts new file mode 100644 index 0000000000..9435a2b23c --- /dev/null +++ b/packages/backend/src/server/api/endpoints/i/export-clips.ts @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import ms from 'ms'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { QueueService } from '@/core/QueueService.js'; + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1day'), + max: 1, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: {}, + required: [], +} as const; + +@Injectable() +export default class extends Endpoint { // eslint-disable-line import/no-default-export + constructor( + private queueService: QueueService, + ) { + super(meta, paramDef, async (ps, me) => { + this.queueService.createExportClipsJob(me); + }); + } +} diff --git a/packages/backend/test/e2e/exports.ts b/packages/backend/test/e2e/exports.ts new file mode 100644 index 0000000000..9686f2b7fd --- /dev/null +++ b/packages/backend/test/e2e/exports.ts @@ -0,0 +1,194 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +process.env.NODE_ENV = 'test'; + +import * as assert from 'assert'; +import { signup, api, startServer, startJobQueue, port, post } from '../utils.js'; +import type { INestApplicationContext } from '@nestjs/common'; +import type * as misskey from 'misskey-js'; + +describe('export-clips', () => { + let app: INestApplicationContext; + let alice: misskey.entities.SignupResponse; + let bob: misskey.entities.SignupResponse; + + // XXX: Any better way to get the result? + async function pollFirstDriveFile() { + while (true) { + const files = (await api('/drive/files', {}, alice)).body; + if (!files.length) { + await new Promise(r => setTimeout(r, 100)); + continue; + } + if (files.length > 1) { + throw new Error('Too many files?'); + } + const file = (await api('/drive/files/show', { fileId: files[0].id }, alice)).body; + const res = await fetch(new URL(new URL(file.url).pathname, `http://127.0.0.1:${port}`)); + return await res.json(); + } + } + + beforeAll(async () => { + app = await startServer(); + await startJobQueue(); + alice = await signup({ username: 'alice' }); + bob = await signup({ username: 'bob' }); + }, 1000 * 60 * 2); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + // Clean all clips and files of alice + const clips = (await api('/clips/list', {}, alice)).body; + for (const clip of clips) { + const res = await api('/clips/delete', { clipId: clip.id }, alice); + if (res.status !== 204) { + throw new Error('Failed to delete clip'); + } + } + const files = (await api('/drive/files', {}, alice)).body; + for (const file of files) { + const res = await api('/drive/files/delete', { fileId: file.id }, alice); + if (res.status !== 204) { + throw new Error('Failed to delete file'); + } + } + }); + + test('basic export', async () => { + let res = await api('/clips/create', { + name: 'foo', + description: 'bar', + }, alice); + assert.strictEqual(res.status, 200); + + res = await api('/i/export-clips', {}, alice); + assert.strictEqual(res.status, 204); + + const exported = await pollFirstDriveFile(); + assert.strictEqual(exported[0].name, 'foo'); + assert.strictEqual(exported[0].description, 'bar'); + assert.strictEqual(exported[0].clipNotes.length, 0); + }); + + test('export with notes', async () => { + let res = await api('/clips/create', { + name: 'foo', + description: 'bar', + }, alice); + assert.strictEqual(res.status, 200); + const clip = res.body; + + const note1 = await post(alice, { + text: 'baz1', + }); + + const note2 = await post(alice, { + text: 'baz2', + poll: { + choices: ['sakura', 'izumi', 'ako'], + }, + }); + + for (const note of [note1, note2]) { + res = await api('/clips/add-note', { + clipId: clip.id, + noteId: note.id, + }, alice); + assert.strictEqual(res.status, 204); + } + + res = await api('/i/export-clips', {}, alice); + assert.strictEqual(res.status, 204); + + const exported = await pollFirstDriveFile(); + assert.strictEqual(exported[0].name, 'foo'); + assert.strictEqual(exported[0].description, 'bar'); + assert.strictEqual(exported[0].clipNotes.length, 2); + assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz1'); + assert.strictEqual(exported[0].clipNotes[1].note.text, 'baz2'); + assert.deepStrictEqual(exported[0].clipNotes[1].note.poll.choices[0], 'sakura'); + }); + + test('multiple clips', async () => { + let res = await api('/clips/create', { + name: 'kawaii', + description: 'kawaii', + }, alice); + assert.strictEqual(res.status, 200); + const clip1 = res.body; + + res = await api('/clips/create', { + name: 'yuri', + description: 'yuri', + }, alice); + assert.strictEqual(res.status, 200); + const clip2 = res.body; + + const note1 = await post(alice, { + text: 'baz1', + }); + + const note2 = await post(alice, { + text: 'baz2', + }); + + res = await api('/clips/add-note', { + clipId: clip1.id, + noteId: note1.id, + }, alice); + assert.strictEqual(res.status, 204); + + res = await api('/clips/add-note', { + clipId: clip2.id, + noteId: note2.id, + }, alice); + assert.strictEqual(res.status, 204); + + res = await api('/i/export-clips', {}, alice); + assert.strictEqual(res.status, 204); + + const exported = await pollFirstDriveFile(); + assert.strictEqual(exported[0].name, 'kawaii'); + assert.strictEqual(exported[0].clipNotes.length, 1); + assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz1'); + assert.strictEqual(exported[1].name, 'yuri'); + assert.strictEqual(exported[1].clipNotes.length, 1); + assert.strictEqual(exported[1].clipNotes[0].note.text, 'baz2'); + }); + + test('Clipping other user\'s note', async () => { + let res = await api('/clips/create', { + name: 'kawaii', + description: 'kawaii', + }, alice); + assert.strictEqual(res.status, 200); + const clip = res.body; + + const note = await post(bob, { + text: 'baz', + visibility: 'followers', + }); + + res = await api('/clips/add-note', { + clipId: clip.id, + noteId: note.id, + }, alice); + assert.strictEqual(res.status, 204); + + res = await api('/i/export-clips', {}, alice); + assert.strictEqual(res.status, 204); + + const exported = await pollFirstDriveFile(); + assert.strictEqual(exported[0].name, 'kawaii'); + assert.strictEqual(exported[0].clipNotes.length, 1); + assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz'); + assert.strictEqual(exported[0].clipNotes[0].note.user.username, 'bob'); + }); +}); diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 46b8ea9cdd..7c9428d476 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -17,7 +17,7 @@ import { entities } from '../src/postgres.js'; import { loadConfig } from '../src/config.js'; import type * as misskey from 'misskey-js'; -export { server as startServer } from '@/boot/common.js'; +export { server as startServer, jobQueue as startJobQueue } from '@/boot/common.js'; interface UserToken { token: string; diff --git a/packages/frontend/src/pages/settings/import-export.vue b/packages/frontend/src/pages/settings/import-export.vue index 990eff99c1..70d718f1ab 100644 --- a/packages/frontend/src/pages/settings/import-export.vue +++ b/packages/frontend/src/pages/settings/import-export.vue @@ -21,6 +21,14 @@ SPDX-License-Identifier: AGPL-3.0-only {{ i18n.ts.export }} + + + + + + {{ i18n.ts.export }} + +
@@ -157,6 +165,10 @@ const exportFavorites = () => { misskeyApi('i/export-favorites', {}).then(onExportSuccess).catch(onError); }; +const exportClips = () => { + misskeyApi('i/export-clips', {}).then(onExportSuccess).catch(onError); +}; + const exportFollowing = () => { misskeyApi('i/export-following', { excludeMuting: excludeMutingUsers.value, -- cgit v1.2.3-freya