add: Importing of Posts

- Supports Instagram, Mastodon/Pleroma/Akkoma, Twitter and *key

3 files changed, 314 insertions, 1 deletions

diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts
index 3e7bba9e1e..9398b5c2c0 100644
--- a/packages/backend/src/core/NoteCreateService.ts
+++ b/packages/backend/src/core/NoteCreateService.ts
@@ -379,6 +379,167 @@ export class NoteCreateService implements OnApplicationShutdown {
 	}
 
 	@bindThis
+	public async import(user: {
+		id: MiUser['id'];
+		username: MiUser['username'];
+		host: MiUser['host'];
+		isBot: MiUser['isBot'];
+		isIndexable: MiUser['isIndexable'];
+	}, data: Option, silent = false): Promise<MiNote> {
+		// チャンネル外にリプライしたら対象のスコープに合わせる
+		// (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで)
+		if (data.reply && data.channel && data.reply.channelId !== data.channel.id) {
+			if (data.reply.channelId) {
+				data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId });
+			} else {
+				data.channel = null;
+			}
+		}
+
+		// チャンネル内にリプライしたら対象のスコープに合わせる
+		// (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで)
+		if (data.reply && (data.channel == null) && data.reply.channelId) {
+			data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId });
+		}
+
+		if (data.createdAt == null) data.createdAt = new Date();
+		if (data.visibility == null) data.visibility = 'public';
+		if (data.localOnly == null) data.localOnly = false;
+		if (data.channel != null) data.visibility = 'public';
+		if (data.channel != null) data.visibleUsers = [];
+		if (data.channel != null) data.localOnly = true;
+
+		const meta = await this.metaService.fetch();
+
+		if (data.visibility === 'public' && data.channel == null) {
+			const sensitiveWords = meta.sensitiveWords;
+			if (this.isSensitive(data, sensitiveWords)) {
+				data.visibility = 'home';
+			} else if ((await this.roleService.getUserPolicies(user.id)).canPublicNote === false) {
+				data.visibility = 'home';
+			}
+		}
+
+		const inSilencedInstance = this.utilityService.isSilencedHost(meta.silencedHosts, user.host);
+
+		if (data.visibility === 'public' && inSilencedInstance && user.host !== null) {
+			data.visibility = 'home';
+		}
+
+		if (data.renote) {
+			switch (data.renote.visibility) {
+				case 'public':
+					// public noteは無条件にrenote可能
+					break;
+				case 'home':
+					// home noteはhome以下にrenote可能
+					if (data.visibility === 'public') {
+						data.visibility = 'home';
+					}
+					break;
+				case 'followers':
+					// 他人のfollowers noteはreject
+					if (data.renote.userId !== user.id) {
+						throw new Error('Renote target is not public or home');
+					}
+
+					// Renote対象がfollowersならfollowersにする
+					data.visibility = 'followers';
+					break;
+				case 'specified':
+					// specified / direct noteはreject
+					throw new Error('Renote target is not public or home');
+			}
+		}
+
+		// Check blocking
+		if (data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)) {
+			if (data.renote.userHost === null) {
+				if (data.renote.userId !== user.id) {
+					const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id);
+					if (blocked) {
+						throw new Error('blocked');
+					}
+				}
+			}
+		}
+
+		// 返信対象がpublicではないならhomeにする
+		if (data.reply && data.reply.visibility !== 'public' && data.visibility === 'public') {
+			data.visibility = 'home';
+		}
+
+		// ローカルのみをRenoteしたらローカルのみにする
+		if (data.renote && data.renote.localOnly && data.channel == null) {
+			data.localOnly = true;
+		}
+
+		// ローカルのみにリプライしたらローカルのみにする
+		if (data.reply && data.reply.localOnly && data.channel == null) {
+			data.localOnly = true;
+		}
+
+		if (data.text) {
+			if (data.text.length > DB_MAX_NOTE_TEXT_LENGTH) {
+				data.text = data.text.slice(0, DB_MAX_NOTE_TEXT_LENGTH);
+			}
+			data.text = data.text.trim();
+		} else {
+			data.text = null;
+		}
+
+		let tags = data.apHashtags;
+		let emojis = data.apEmojis;
+		let mentionedUsers = data.apMentions;
+
+		// Parse MFM if needed
+		if (!tags || !emojis || !mentionedUsers) {
+			const tokens = (data.text ? mfm.parse(data.text)! : []);
+			const cwTokens = data.cw ? mfm.parse(data.cw)! : [];
+			const choiceTokens = data.poll && data.poll.choices
+				? concat(data.poll.choices.map(choice => mfm.parse(choice)!))
+				: [];
+
+			const combinedTokens = tokens.concat(cwTokens).concat(choiceTokens);
+
+			tags = data.apHashtags ?? extractHashtags(combinedTokens);
+
+			emojis = data.apEmojis ?? extractCustomEmojisFromMfm(combinedTokens);
+
+			mentionedUsers = data.apMentions ?? await this.extractMentionedUsers(user, combinedTokens);
+		}
+
+		tags = tags.filter(tag => Array.from(tag).length <= 128).splice(0, 32);
+
+		if (data.reply && (user.id !== data.reply.userId) && !mentionedUsers.some(u => u.id === data.reply!.userId)) {
+			mentionedUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId }));
+		}
+
+		if (data.visibility === 'specified') {
+			if (data.visibleUsers == null) throw new Error('invalid param');
+
+			for (const u of data.visibleUsers) {
+				if (!mentionedUsers.some(x => x.id === u.id)) {
+					mentionedUsers.push(u);
+				}
+			}
+
+			if (data.reply && !data.visibleUsers.some(x => x.id === data.reply!.userId)) {
+				data.visibleUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId }));
+			}
+		}
+
+		const note = await this.insertNote(user, data, tags, emojis, mentionedUsers);
+
+		setImmediate('post created', { signal: this.#shutdownController.signal }).then(
+			() => this.postNoteImported(note, user, data, silent, tags!, mentionedUsers!),
+			() => { /* aborted, ignore this */ },
+		);
+
+		return note;
+	}
+
+	@bindThis
 	private async insertNote(user: { id: MiUser['id']; host: MiUser['host']; }, data: Option, tags: string[], emojis: string[], mentionedUsers: MinimumUser[]) {
 		const insert = new MiNote({
 			id: this.idService.gen(data.createdAt?.getTime()),
@@ -716,6 +877,113 @@ export class NoteCreateService implements OnApplicationShutdown {
 	}
 
 	@bindThis
+	private async postNoteImported(note: MiNote, user: {
+		id: MiUser['id'];
+		username: MiUser['username'];
+		host: MiUser['host'];
+		isBot: MiUser['isBot'];
+		isIndexable: MiUser['isIndexable'];
+	}, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) {
+		const meta = await this.metaService.fetch();
+
+		this.notesChart.update(note, true);
+		if (meta.enableChartsForRemoteUser || (user.host == null)) {
+			this.perUserNotesChart.update(user, note, true);
+		}
+
+		// Register host
+		if (this.userEntityService.isRemoteUser(user)) {
+			this.federatedInstanceService.fetch(user.host).then(async i => {
+				if (note.renote && note.text) {
+					this.instancesRepository.increment({ id: i.id }, 'notesCount', 1);
+				} else if (!note.renote) {
+					this.instancesRepository.increment({ id: i.id }, 'notesCount', 1);
+				}
+				if ((await this.metaService.fetch()).enableChartsForFederatedInstances) {
+					this.instanceChart.updateNote(i.host, note, true);
+				}
+			});
+		}
+
+		if (data.renote && data.text) {
+			// Increment notes count (user)
+			this.incNotesCountOfUser(user);
+		} else if (!data.renote) {
+			// Increment notes count (user)
+			this.incNotesCountOfUser(user);
+		}
+
+		this.pushToTl(note, user);
+
+		this.antennaService.addNoteToAntennas(note, user);
+
+		if (data.reply) {
+			this.saveReply(data.reply, note);
+		}
+
+		if (data.reply == null) {
+			// TODO: キャッシュ
+			this.followingsRepository.findBy({
+				followeeId: user.id,
+				notify: 'normal',
+			}).then(followings => {
+				for (const following of followings) {
+					// TODO: ワードミュート考慮
+					this.notificationService.createNotification(following.followerId, 'note', {
+						noteId: note.id,
+					}, user.id);
+				}
+			});
+		}
+
+		if (data.renote && data.text == null && data.renote.userId !== user.id && !user.isBot) {
+			this.incRenoteCount(data.renote);
+		}
+
+		if (data.poll && data.poll.expiresAt) {
+			const delay = data.poll.expiresAt.getTime() - Date.now();
+			this.queueService.endedPollNotificationQueue.add(note.id, {
+				noteId: note.id,
+			}, {
+				delay,
+				removeOnComplete: true,
+			});
+		}
+
+		if (!silent) {
+			if (this.userEntityService.isLocalUser(user)) this.activeUsersChart.write(user);
+
+			// Pack the note
+			const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true });
+
+			this.globalEventService.publishNotesStream(noteObj);
+
+			this.roleService.addNoteToRoleTimeline(noteObj);
+		}
+
+		if (data.channel) {
+			this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1);
+			this.channelsRepository.update(data.channel.id, {
+				lastNotedAt: new Date(),
+			});
+
+			this.notesRepository.countBy({
+				userId: user.id,
+				channelId: data.channel.id,
+			}).then(count => {
+				// この処理が行われるのはノート作成後なので、ノートが一つしかなかったら最初の投稿だと判断できる
+				// TODO: とはいえノートを削除して何回も投稿すればその分だけインクリメントされる雑さもあるのでどうにかしたい
+				if (count === 1) {
+					this.channelsRepository.increment({ id: data.channel!.id }, 'usersCount', 1);
+				}
+			});
+		}
+
+		// Register to search database
+		if (user.isIndexable) this.index(note);
+	}
+
+	@bindThis
 	private isSensitive(note: Option, sensitiveWord: string[]): boolean {
 		if (sensitiveWord.length > 0) {
 			const text = note.cw ?? note.text ?? '';
diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts
index c5830168b8..ed24cfa56a 100644
--- a/packages/backend/src/core/QueueService.ts
+++ b/packages/backend/src/core/QueueService.ts
@@ -259,6 +259,48 @@ export class QueueService {
 	}
 
 	@bindThis
+	public createImportNotesJob(user: ThinUser, fileId: MiDriveFile['id'], type: string | null | undefined) {
+		return this.dbQueue.add('importNotes', {
+			user: { id: user.id },
+			fileId: fileId,
+			type: type,
+		}, {
+			removeOnComplete: true,
+			removeOnFail: true,
+		});
+	}
+
+	@bindThis
+	public createImportTweetsToDbJob(user: ThinUser, targets: string[]) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importTweetsToDb', { user, target: rel }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
+	@bindThis
+	public createImportMastoToDbJob(user: ThinUser, targets: string[]) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importMastoToDb', { user, target: rel }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
+	@bindThis
+	public createImportPleroToDbJob(user: ThinUser, targets: string[]) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importPleroToDb', { user, target: rel }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
+	@bindThis
+	public createImportKeyNotesToDbJob(user: ThinUser, targets: string[]) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
+	@bindThis
+	public createImportIGToDbJob(user: ThinUser, targets: string[]) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importIGToDb', { user, target: rel }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
+	@bindThis
 	public createImportFollowingToDbJob(user: ThinUser, targets: string[], withReplies?: boolean) {
 		const jobs = targets.map(rel => this.generateToDbJobData('importFollowingToDb', { user, target: rel, withReplies }));
 		return this.dbQueue.addBulk(jobs);
@@ -293,7 +335,7 @@ export class QueueService {
 	}
 
 	@bindThis
-	private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb', D extends DbJobData<T>>(name: T, data: D): {
+	private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb' | 'importTweetsToDb' | 'importIGToDb' | 'importMastoToDb' | 'importPleroToDb' | 'importKeyNotesToDb', D extends DbJobData<T>>(name: T, data: D): {
 		name: string,
 		data: D,
 		opts: Bull.JobsOptions,
diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts
index d6a414694a..4c5f883351 100644
--- a/packages/backend/src/core/RoleService.ts
+++ b/packages/backend/src/core/RoleService.ts
@@ -47,6 +47,7 @@ export type RolePolicies = {
 	userListLimit: number;
 	userEachUserListsLimit: number;
 	rateLimitFactor: number;
+	canImportNotes: boolean;
 };
 
 export const DEFAULT_POLICIES: RolePolicies = {
@@ -73,6 +74,7 @@ export const DEFAULT_POLICIES: RolePolicies = {
 	userListLimit: 10,
 	userEachUserListsLimit: 50,
 	rateLimitFactor: 1,
+	canImportNotes: true,
 };
 
 @Injectable()
@@ -323,6 +325,7 @@ export class RoleService implements OnApplicationShutdown {
 			userListLimit: calc('userListLimit', vs => Math.max(...vs)),
 			userEachUserListsLimit: calc('userEachUserListsLimit', vs => Math.max(...vs)),
 			rateLimitFactor: calc('rateLimitFactor', vs => Math.max(...vs)),
+			canImportNotes: calc('canImportNotes', vs => vs.some(v => v === true)),
 		};
 	}