feat: Introduce Meilisearch (#10755)

* wip

* wip

* Update SearchService.ts

* Update SearchService.ts

* wip

* wip

* Update SearchService.ts

* Update CHANGELOG.md

* wip

* Update SearchService.ts

* Update docker-compose.yml.example

9 files changed, 212 insertions, 51 deletions

diff --git a/packages/backend/package.json b/packages/backend/package.json
index 9b20c121eb..08557d415e 100644
--- a/packages/backend/package.json
+++ b/packages/backend/package.json
@@ -91,6 +91,7 @@
 		"jsdom": "21.1.1",
 		"json5": "2.2.3",
 		"jsonld": "8.1.1",
+		"meilisearch": "0.32.3",
 		"jsrsasign": "10.8.6",
 		"mfm-js": "0.23.3",
 		"mime-types": "2.1.35",
diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts
index 4574429c43..2f4862285d 100644
--- a/packages/backend/src/GlobalModule.ts
+++ b/packages/backend/src/GlobalModule.ts
@@ -2,6 +2,7 @@ import { setTimeout } from 'node:timers/promises';
 import { Global, Inject, Module } from '@nestjs/common';
 import * as Redis from 'ioredis';
 import { DataSource } from 'typeorm';
+import { MeiliSearch } from 'meilisearch';
 import { DI } from './di-symbols.js';
 import { loadConfig } from './config.js';
 import { createPostgresDataSource } from './postgres.js';
@@ -22,6 +23,21 @@ const $db: Provider = {
 	inject: [DI.config],
 };
 
+const $meilisearch: Provider = {
+	provide: DI.meilisearch,
+	useFactory: (config) => {
+		if (config.meilisearch) {
+			return new MeiliSearch({
+				host: `http://${config.meilisearch.host}:${config.meilisearch.port}`, 
+				apiKey: config.meilisearch.apiKey,
+			});
+		} else {
+			return null;
+		}
+	},
+	inject: [DI.config],
+};
+
 const $redis: Provider = {
 	provide: DI.redis,
 	useFactory: (config) => {
@@ -73,8 +89,8 @@ const $redisForSub: Provider = {
 @Global()
 @Module({
 	imports: [RepositoryModule],
-	providers: [$config, $db, $redis, $redisForPub, $redisForSub],
-	exports: [$config, $db, $redis, $redisForPub, $redisForSub, RepositoryModule],
+	providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub],
+	exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, RepositoryModule],
 })
 export class GlobalModule implements OnApplicationShutdown {
 	constructor(
diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts
index 4499475ee9..7354268a4d 100644
--- a/packages/backend/src/config.ts
+++ b/packages/backend/src/config.ts
@@ -57,13 +57,10 @@ export type Source = {
 		db?: number;
 		prefix?: string;
 	};
-	elasticsearch: {
+	meilisearch?: {
 		host: string;
-		port: number;
-		ssl?: boolean;
-		user?: string;
-		pass?: string;
-		index?: string;
+		port: string;
+		apiKey: string;
 	};
 
 	proxy?: string;
@@ -139,6 +136,7 @@ const path = process.env.MISSKEY_CONFIG_YML
 	: process.env.NODE_ENV === 'test'
 		? resolve(dir, 'test.yml')
 		: resolve(dir, 'default.yml');
+
 export function loadConfig() {
 	const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8'));
 	const clientManifestExists = fs.existsSync(_dirname + '/../../../built/_vite_/manifest.json');
diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts
index 8775536e4a..d3a1b1b024 100644
--- a/packages/backend/src/core/CoreModule.ts
+++ b/packages/backend/src/core/CoreModule.ts
@@ -50,6 +50,7 @@ import { WebhookService } from './WebhookService.js';
 import { ProxyAccountService } from './ProxyAccountService.js';
 import { UtilityService } from './UtilityService.js';
 import { FileInfoService } from './FileInfoService.js';
+import { SearchService } from './SearchService.js';
 import { ChartLoggerService } from './chart/ChartLoggerService.js';
 import FederationChart from './chart/charts/federation.js';
 import NotesChart from './chart/charts/notes.js';
@@ -171,6 +172,8 @@ const $VideoProcessingService: Provider = { provide: 'VideoProcessingService', u
 const $WebhookService: Provider = { provide: 'WebhookService', useExisting: WebhookService };
 const $UtilityService: Provider = { provide: 'UtilityService', useExisting: UtilityService };
 const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: FileInfoService };
+const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService };
+
 const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService };
 const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart };
 const $NotesChart: Provider = { provide: 'NotesChart', useExisting: NotesChart };
@@ -295,6 +298,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		WebhookService,
 		UtilityService,
 		FileInfoService,
+		SearchService,
 		ChartLoggerService,
 		FederationChart,
 		NotesChart,
@@ -413,6 +417,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		$WebhookService,
 		$UtilityService,
 		$FileInfoService,
+		$SearchService,
 		$ChartLoggerService,
 		$FederationChart,
 		$NotesChart,
@@ -532,6 +537,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		WebhookService,
 		UtilityService,
 		FileInfoService,
+		SearchService,
 		FederationChart,
 		NotesChart,
 		UsersChart,
@@ -649,6 +655,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		$WebhookService,
 		$UtilityService,
 		$FileInfoService,
+		$SearchService,
 		$FederationChart,
 		$NotesChart,
 		$UsersChart,
diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts
index 50081f831b..364976e4a7 100644
--- a/packages/backend/src/core/NoteCreateService.ts
+++ b/packages/backend/src/core/NoteCreateService.ts
@@ -46,6 +46,7 @@ import { bindThis } from '@/decorators.js';
 import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js';
 import { RoleService } from '@/core/RoleService.js';
 import { MetaService } from '@/core/MetaService.js';
+import { SearchService } from '@/core/SearchService.js';
 
 const mutedWordsCache = new MemorySingleCache<{ userId: UserProfile['userId']; mutedWords: UserProfile['mutedWords']; }[]>(1000 * 60 * 5);
 
@@ -198,6 +199,7 @@ export class NoteCreateService implements OnApplicationShutdown {
 		private apRendererService: ApRendererService,
 		private roleService: RoleService,
 		private metaService: MetaService,
+		private searchService: SearchService,
 		private notesChart: NotesChart,
 		private perUserNotesChart: PerUserNotesChart,
 		private activeUsersChart: ActiveUsersChart,
@@ -728,17 +730,9 @@ export class NoteCreateService implements OnApplicationShutdown {
 
 	@bindThis
 	private index(note: Note) {
-		if (note.text == null || this.config.elasticsearch == null) return;
-		/*
-	es!.index({
-		index: this.config.elasticsearch.index ?? 'misskey_note',
-		id: note.id.toString(),
-		body: {
-			text: normalizeForSearch(note.text),
-			userId: note.userId,
-			userHost: note.userHost,
-		},
-	});*/
+		if (note.text == null && note.cw == null) return;
+		
+		this.searchService.indexNote(note);
 	}
 
 	@bindThis
diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts
new file mode 100644
index 0000000000..67332581f7
--- /dev/null
+++ b/packages/backend/src/core/SearchService.ts
@@ -0,0 +1,166 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { In } from 'typeorm';
+import { DI } from '@/di-symbols.js';
+import type { Config } from '@/config.js';
+import { bindThis } from '@/decorators.js';
+import { Note } from '@/models/entities/Note.js';
+import { User } from '@/models/index.js';
+import type { NotesRepository } from '@/models/index.js';
+import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
+import { QueryService } from '@/core/QueryService.js';
+import { IdService } from '@/core/IdService.js';
+import type { Index, MeiliSearch } from 'meilisearch';
+
+type K = string;
+type V = string | number | boolean;
+type Q =
+	{ op: '=', k: K, v: V } |
+	{ op: '!=', k: K, v: V } |
+	{ op: '>', k: K, v: number } |
+	{ op: '<', k: K, v: number } |
+	{ op: '>=', k: K, v: number } |
+	{ op: '<=', k: K, v: number } |
+	{ op: 'and', qs: Q[] } |
+	{ op: 'or', qs: Q[] } |
+	{ op: 'not', q: Q };
+
+function compileValue(value: V): string {
+	if (typeof value === 'string') {
+		return `'${value}'`; // TODO: escape
+	} else if (typeof value === 'number') {
+		return value.toString();
+	} else if (typeof value === 'boolean') {
+		return value.toString();
+	}
+	throw new Error('unrecognized value');
+}
+
+function compileQuery(q: Q): string {
+	switch (q.op) {
+		case '=': return `(${q.k} = ${compileValue(q.v)})`;
+		case '!=': return `(${q.k} != ${compileValue(q.v)})`;
+		case '>': return `(${q.k} > ${compileValue(q.v)})`;
+		case '<': return `(${q.k} < ${compileValue(q.v)})`;
+		case '>=': return `(${q.k} >= ${compileValue(q.v)})`;
+		case '<=': return `(${q.k} <= ${compileValue(q.v)})`;
+		case 'and': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' AND ') })`;
+		case 'or': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' OR ') })`;
+		case 'not': return `(NOT ${compileQuery(q.q)})`;
+		default: throw new Error('unrecognized query operator');
+	}
+}
+
+@Injectable()
+export class SearchService {
+	private meilisearchNoteIndex: Index | null = null;
+
+	constructor(
+		@Inject(DI.config)
+		private config: Config,
+
+		@Inject(DI.meilisearch)
+		private meilisearch: MeiliSearch | null,
+
+		@Inject(DI.notesRepository)
+		private notesRepository: NotesRepository,
+
+		private queryService: QueryService,
+		private idService: IdService,
+	) {
+		if (meilisearch) {
+			this.meilisearchNoteIndex = meilisearch.index('notes');
+			this.meilisearchNoteIndex.updateSettings({
+				searchableAttributes: [
+					'text',
+					'cw',
+				],
+				sortableAttributes: [
+					'createdAt',
+				],
+				filterableAttributes: [
+					'createdAt',
+					'userId',
+					'userHost',
+					'channelId',
+				],
+				typoTolerance: {
+					enabled: false,
+				},
+				pagination: {
+					maxTotalHits: 10000,
+				},
+			});
+		}
+	}
+
+	@bindThis
+	public async indexNote(note: Note): Promise<void> {
+		if (this.meilisearch) {
+			this.meilisearchNoteIndex!.addDocuments([{
+				id: note.id,
+				createdAt: note.createdAt.getTime(),
+				userId: note.userId,
+				userHost: note.userHost,
+				channelId: note.channelId,
+				cw: note.cw,
+				text: note.text,
+			}], {
+				primaryKey: 'id',
+			});
+		}
+	}
+
+	@bindThis
+	public async searchNote(q: string, me: User | null, opts: {
+		userId?: Note['userId'] | null;
+		channelId?: Note['channelId'] | null;
+	}, pagination: {
+		untilId?: Note['id'];
+		sinceId?: Note['id'];
+		limit?: number;
+	}): Promise<Note[]> {
+		if (this.meilisearch) {
+			const filter: Q = {
+				op: 'and',
+				qs: [],
+			};
+			if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() });
+			if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() });
+			if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId });
+			if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId });
+			const res = await this.meilisearchNoteIndex!.search(q, {
+				sort: ['createdAt:desc'],
+				matchingStrategy: 'all',
+				attributesToRetrieve: ['id', 'createdAt'],
+				filter: compileQuery(filter),
+				limit: pagination.limit,
+			});
+			if (res.hits.length === 0) return [];
+			return await this.notesRepository.findBy({
+				id: In(res.hits.map(x => x.id)),
+			});
+		} else {
+			const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId);
+
+			if (opts.userId) {
+				query.andWhere('note.userId = :userId', { userId: opts.userId });
+			} else if (opts.channelId) {
+				query.andWhere('note.channelId = :channelId', { channelId: opts.channelId });
+			}
+
+			query
+				.andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` })
+				.innerJoinAndSelect('note.user', 'user')
+				.leftJoinAndSelect('note.reply', 'reply')
+				.leftJoinAndSelect('note.renote', 'renote')
+				.leftJoinAndSelect('reply.user', 'replyUser')
+				.leftJoinAndSelect('renote.user', 'renoteUser');
+
+			this.queryService.generateVisibilityQuery(query, me);
+			if (me) this.queryService.generateMutedUserQuery(query, me);
+			if (me) this.queryService.generateBlockedUserQuery(query, me);
+
+			return await query.take(pagination.limit).getMany();
+		}
+	}
+}
diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts
index 190d8d65c2..c06c7a7159 100644
--- a/packages/backend/src/di-symbols.ts
+++ b/packages/backend/src/di-symbols.ts
@@ -1,6 +1,7 @@
 export const DI = {
 	config: Symbol('config'),
 	db: Symbol('db'),
+	meilisearch: Symbol('meilisearch'),
 	redis: Symbol('redis'),
 	redisForPub: Symbol('redisForPub'),
 	redisForSub: Symbol('redisForSub'),
diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts
index a5cb3fa7ee..584ea07c3b 100644
--- a/packages/backend/src/server/api/endpoints/meta.ts
+++ b/packages/backend/src/server/api/endpoints/meta.ts
@@ -201,10 +201,6 @@ export const meta = {
 						type: 'boolean',
 						optional: false, nullable: false,
 					},
-					elasticsearch: {
-						type: 'boolean',
-						optional: false, nullable: false,
-					},
 					hcaptcha: {
 						type: 'boolean',
 						optional: false, nullable: false,
@@ -331,7 +327,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 				response.features = {
 					registration: !instance.disableRegistration,
 					emailRequiredForSignup: instance.emailRequiredForSignup,
-					elasticsearch: this.config.elasticsearch ? true : false,
 					hcaptcha: instance.enableHcaptcha,
 					recaptcha: instance.enableRecaptcha,
 					turnstile: instance.enableTurnstile,
diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts
index fb5abd917f..990ba526d9 100644
--- a/packages/backend/src/server/api/endpoints/notes/search.ts
+++ b/packages/backend/src/server/api/endpoints/notes/search.ts
@@ -1,11 +1,10 @@
 import { Inject, Injectable } from '@nestjs/common';
 import type { NotesRepository } from '@/models/index.js';
 import { Endpoint } from '@/server/api/endpoint-base.js';
-import { QueryService } from '@/core/QueryService.js';
+import { SearchService } from '@/core/SearchService.js';
 import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
 import type { Config } from '@/config.js';
 import { DI } from '@/di-symbols.js';
-import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
 import { RoleService } from '@/core/RoleService.js';
 import { ApiError } from '../../error.js';
 
@@ -61,11 +60,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 		@Inject(DI.config)
 		private config: Config,
 	
-		@Inject(DI.notesRepository)
-		private notesRepository: NotesRepository,
-
 		private noteEntityService: NoteEntityService,
-		private queryService: QueryService,
+		private searchService: SearchService,
 		private roleService: RoleService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
@@ -74,27 +70,14 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 				throw new ApiError(meta.errors.unavailable);
 			}
 	
-			const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId);
-
-			if (ps.userId) {
-				query.andWhere('note.userId = :userId', { userId: ps.userId });
-			} else if (ps.channelId) {
-				query.andWhere('note.channelId = :channelId', { channelId: ps.channelId });
-			}
-
-			query
-				.andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(ps.query) }%` })
-				.innerJoinAndSelect('note.user', 'user')
-				.leftJoinAndSelect('note.reply', 'reply')
-				.leftJoinAndSelect('note.renote', 'renote')
-				.leftJoinAndSelect('reply.user', 'replyUser')
-				.leftJoinAndSelect('renote.user', 'renoteUser');
-
-			this.queryService.generateVisibilityQuery(query, me);
-			if (me) this.queryService.generateMutedUserQuery(query, me);
-			if (me) this.queryService.generateBlockedUserQuery(query, me);
-
-			const notes = await query.take(ps.limit).getMany();
+			const notes = await this.searchService.searchNote(ps.query, me, {
+				userId: ps.userId,
+				channelId: ps.channelId,
+			}, {
+				untilId: ps.untilId,
+				sinceId: ps.sinceId,
+				limit: ps.limit,
+			});
 
 			return await this.noteEntityService.packMany(notes, me);
 		});