From 83f328de8a1536c9fbae0605e97ec4af51bd84a4 Mon Sep 17 00:00:00 2001 From: Mar0xy Date: Sun, 12 Nov 2023 15:07:32 +0100 Subject: add: Importing of Posts - Supports Instagram, Mastodon/Pleroma/Akkoma, Twitter and *key --- packages/backend/src/core/NoteCreateService.ts | 268 +++++++++++++++++++++++++ packages/backend/src/core/QueueService.ts | 44 +++- packages/backend/src/core/RoleService.ts | 3 + 3 files changed, 314 insertions(+), 1 deletion(-) (limited to 'packages/backend/src/core') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 3e7bba9e1e..9398b5c2c0 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -378,6 +378,167 @@ export class NoteCreateService implements OnApplicationShutdown { return note; } + @bindThis + public async import(user: { + id: MiUser['id']; + username: MiUser['username']; + host: MiUser['host']; + isBot: MiUser['isBot']; + isIndexable: MiUser['isIndexable']; + }, data: Option, silent = false): Promise { + // チャンネル外にリプライしたら対象のスコープに合わせる + // (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで) + if (data.reply && data.channel && data.reply.channelId !== data.channel.id) { + if (data.reply.channelId) { + data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId }); + } else { + data.channel = null; + } + } + + // チャンネル内にリプライしたら対象のスコープに合わせる + // (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで) + if (data.reply && (data.channel == null) && data.reply.channelId) { + data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId }); + } + + if (data.createdAt == null) data.createdAt = new Date(); + if (data.visibility == null) data.visibility = 'public'; + if (data.localOnly == null) data.localOnly = false; + if (data.channel != null) data.visibility = 'public'; + if (data.channel != null) data.visibleUsers = []; + if (data.channel != null) data.localOnly = true; + + const meta = await this.metaService.fetch(); + + if (data.visibility === 'public' && data.channel == null) { + const sensitiveWords = meta.sensitiveWords; + if (this.isSensitive(data, sensitiveWords)) { + data.visibility = 'home'; + } else if ((await this.roleService.getUserPolicies(user.id)).canPublicNote === false) { + data.visibility = 'home'; + } + } + + const inSilencedInstance = this.utilityService.isSilencedHost(meta.silencedHosts, user.host); + + if (data.visibility === 'public' && inSilencedInstance && user.host !== null) { + data.visibility = 'home'; + } + + if (data.renote) { + switch (data.renote.visibility) { + case 'public': + // public noteは無条件にrenote可能 + break; + case 'home': + // home noteはhome以下にrenote可能 + if (data.visibility === 'public') { + data.visibility = 'home'; + } + break; + case 'followers': + // 他人のfollowers noteはreject + if (data.renote.userId !== user.id) { + throw new Error('Renote target is not public or home'); + } + + // Renote対象がfollowersならfollowersにする + data.visibility = 'followers'; + break; + case 'specified': + // specified / direct noteはreject + throw new Error('Renote target is not public or home'); + } + } + + // Check blocking + if (data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)) { + if (data.renote.userHost === null) { + if (data.renote.userId !== user.id) { + const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id); + if (blocked) { + throw new Error('blocked'); + } + } + } + } + + // 返信対象がpublicではないならhomeにする + if (data.reply && data.reply.visibility !== 'public' && data.visibility === 'public') { + data.visibility = 'home'; + } + + // ローカルのみをRenoteしたらローカルのみにする + if (data.renote && data.renote.localOnly && data.channel == null) { + data.localOnly = true; + } + + // ローカルのみにリプライしたらローカルのみにする + if (data.reply && data.reply.localOnly && data.channel == null) { + data.localOnly = true; + } + + if (data.text) { + if (data.text.length > DB_MAX_NOTE_TEXT_LENGTH) { + data.text = data.text.slice(0, DB_MAX_NOTE_TEXT_LENGTH); + } + data.text = data.text.trim(); + } else { + data.text = null; + } + + let tags = data.apHashtags; + let emojis = data.apEmojis; + let mentionedUsers = data.apMentions; + + // Parse MFM if needed + if (!tags || !emojis || !mentionedUsers) { + const tokens = (data.text ? mfm.parse(data.text)! : []); + const cwTokens = data.cw ? mfm.parse(data.cw)! : []; + const choiceTokens = data.poll && data.poll.choices + ? concat(data.poll.choices.map(choice => mfm.parse(choice)!)) + : []; + + const combinedTokens = tokens.concat(cwTokens).concat(choiceTokens); + + tags = data.apHashtags ?? extractHashtags(combinedTokens); + + emojis = data.apEmojis ?? extractCustomEmojisFromMfm(combinedTokens); + + mentionedUsers = data.apMentions ?? await this.extractMentionedUsers(user, combinedTokens); + } + + tags = tags.filter(tag => Array.from(tag).length <= 128).splice(0, 32); + + if (data.reply && (user.id !== data.reply.userId) && !mentionedUsers.some(u => u.id === data.reply!.userId)) { + mentionedUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId })); + } + + if (data.visibility === 'specified') { + if (data.visibleUsers == null) throw new Error('invalid param'); + + for (const u of data.visibleUsers) { + if (!mentionedUsers.some(x => x.id === u.id)) { + mentionedUsers.push(u); + } + } + + if (data.reply && !data.visibleUsers.some(x => x.id === data.reply!.userId)) { + data.visibleUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId })); + } + } + + const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); + + setImmediate('post created', { signal: this.#shutdownController.signal }).then( + () => this.postNoteImported(note, user, data, silent, tags!, mentionedUsers!), + () => { /* aborted, ignore this */ }, + ); + + return note; + } + @bindThis private async insertNote(user: { id: MiUser['id']; host: MiUser['host']; }, data: Option, tags: string[], emojis: string[], mentionedUsers: MinimumUser[]) { const insert = new MiNote({ @@ -715,6 +876,113 @@ export class NoteCreateService implements OnApplicationShutdown { if (user.isIndexable) this.index(note); } + @bindThis + private async postNoteImported(note: MiNote, user: { + id: MiUser['id']; + username: MiUser['username']; + host: MiUser['host']; + isBot: MiUser['isBot']; + isIndexable: MiUser['isIndexable']; + }, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) { + const meta = await this.metaService.fetch(); + + this.notesChart.update(note, true); + if (meta.enableChartsForRemoteUser || (user.host == null)) { + this.perUserNotesChart.update(user, note, true); + } + + // Register host + if (this.userEntityService.isRemoteUser(user)) { + this.federatedInstanceService.fetch(user.host).then(async i => { + if (note.renote && note.text) { + this.instancesRepository.increment({ id: i.id }, 'notesCount', 1); + } else if (!note.renote) { + this.instancesRepository.increment({ id: i.id }, 'notesCount', 1); + } + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateNote(i.host, note, true); + } + }); + } + + if (data.renote && data.text) { + // Increment notes count (user) + this.incNotesCountOfUser(user); + } else if (!data.renote) { + // Increment notes count (user) + this.incNotesCountOfUser(user); + } + + this.pushToTl(note, user); + + this.antennaService.addNoteToAntennas(note, user); + + if (data.reply) { + this.saveReply(data.reply, note); + } + + if (data.reply == null) { + // TODO: キャッシュ + this.followingsRepository.findBy({ + followeeId: user.id, + notify: 'normal', + }).then(followings => { + for (const following of followings) { + // TODO: ワードミュート考慮 + this.notificationService.createNotification(following.followerId, 'note', { + noteId: note.id, + }, user.id); + } + }); + } + + if (data.renote && data.text == null && data.renote.userId !== user.id && !user.isBot) { + this.incRenoteCount(data.renote); + } + + if (data.poll && data.poll.expiresAt) { + const delay = data.poll.expiresAt.getTime() - Date.now(); + this.queueService.endedPollNotificationQueue.add(note.id, { + noteId: note.id, + }, { + delay, + removeOnComplete: true, + }); + } + + if (!silent) { + if (this.userEntityService.isLocalUser(user)) this.activeUsersChart.write(user); + + // Pack the note + const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true }); + + this.globalEventService.publishNotesStream(noteObj); + + this.roleService.addNoteToRoleTimeline(noteObj); + } + + if (data.channel) { + this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); + this.channelsRepository.update(data.channel.id, { + lastNotedAt: new Date(), + }); + + this.notesRepository.countBy({ + userId: user.id, + channelId: data.channel.id, + }).then(count => { + // この処理が行われるのはノート作成後なので、ノートが一つしかなかったら最初の投稿だと判断できる + // TODO: とはいえノートを削除して何回も投稿すればその分だけインクリメントされる雑さもあるのでどうにかしたい + if (count === 1) { + this.channelsRepository.increment({ id: data.channel!.id }, 'usersCount', 1); + } + }); + } + + // Register to search database + if (user.isIndexable) this.index(note); + } + @bindThis private isSensitive(note: Option, sensitiveWord: string[]): boolean { if (sensitiveWord.length > 0) { diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index c5830168b8..ed24cfa56a 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -258,6 +258,48 @@ export class QueueService { }); } + @bindThis + public createImportNotesJob(user: ThinUser, fileId: MiDriveFile['id'], type: string | null | undefined) { + return this.dbQueue.add('importNotes', { + user: { id: user.id }, + fileId: fileId, + type: type, + }, { + removeOnComplete: true, + removeOnFail: true, + }); + } + + @bindThis + public createImportTweetsToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importTweetsToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportMastoToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importMastoToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportPleroToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importPleroToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportKeyNotesToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportIGToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importIGToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + @bindThis public createImportFollowingToDbJob(user: ThinUser, targets: string[], withReplies?: boolean) { const jobs = targets.map(rel => this.generateToDbJobData('importFollowingToDb', { user, target: rel, withReplies })); @@ -293,7 +335,7 @@ export class QueueService { } @bindThis - private generateToDbJobData>(name: T, data: D): { + private generateToDbJobData>(name: T, data: D): { name: string, data: D, opts: Bull.JobsOptions, diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index d6a414694a..4c5f883351 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -47,6 +47,7 @@ export type RolePolicies = { userListLimit: number; userEachUserListsLimit: number; rateLimitFactor: number; + canImportNotes: boolean; }; export const DEFAULT_POLICIES: RolePolicies = { @@ -73,6 +74,7 @@ export const DEFAULT_POLICIES: RolePolicies = { userListLimit: 10, userEachUserListsLimit: 50, rateLimitFactor: 1, + canImportNotes: true, }; @Injectable() @@ -323,6 +325,7 @@ export class RoleService implements OnApplicationShutdown { userListLimit: calc('userListLimit', vs => Math.max(...vs)), userEachUserListsLimit: calc('userEachUserListsLimit', vs => Math.max(...vs)), rateLimitFactor: calc('rateLimitFactor', vs => Math.max(...vs)), + canImportNotes: calc('canImportNotes', vs => vs.some(v => v === true)), }; } -- cgit v1.2.3-freya