From 71cecdbcf2d46d00ff58634eafec9227f4264b08 Mon Sep 17 00:00:00 2001 From: おさむのひと <46447427+samunohito@users.noreply.github.com> Date: Tue, 14 Jan 2025 21:01:01 +0900 Subject: feat(backend): pgroongaに対応（configの構成変更あり） (#14978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(backend): pgroongaに対応（configの構成変更あり） * fix CHANGELOG.md * fix CHANGELOG.md * add using provider logging * fix CHANGELOG.md * Update CHANGELOG.md --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- packages/backend/src/core/SearchService.ts | 276 ++++++++++++++++++----------- 1 file changed, 172 insertions(+), 104 deletions(-) (limited to 'packages/backend/src/core/SearchService.ts') diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index f3c11277c5..64e3f2f56a 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -6,16 +6,17 @@ import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; +import { type Config, FulltextSearchProvider } from '@/config.js'; import { bindThis } from '@/decorators.js'; import { MiNote } from '@/models/Note.js'; -import { MiUser } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; +import { MiUser } from '@/models/_.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; +import { LoggerService } from '@/core/LoggerService.js'; import type { Index, MeiliSearch } from 'meilisearch'; type K = string; @@ -27,12 +28,24 @@ type Q = { op: '<', k: K, v: number } | { op: '>=', k: K, v: number } | { op: '<=', k: K, v: number } | - { op: 'is null', k: K} | - { op: 'is not null', k: K} | + { op: 'is null', k: K } | + { op: 'is not null', k: K } | { op: 'and', qs: Q[] } | { op: 'or', qs: Q[] } | { op: 'not', q: Q }; +export type SearchOpts = { + userId?: MiNote['userId'] | null; + channelId?: MiNote['channelId'] | null; + host?: string | null; +}; + +export type SearchPagination = { + untilId?: MiNote['id']; + sinceId?: MiNote['id']; + limit: number; +}; + function compileValue(value: V): string { if (typeof value === 'string') { return `'${value}'`; // TODO: escape @@ -64,7 +77,8 @@ function compileQuery(q: Q): string { @Injectable() export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; - private meilisearchNoteIndex: Index | null = null; + private readonly meilisearchNoteIndex: Index | null = null; + private readonly provider: FulltextSearchProvider; constructor( @Inject(DI.config) @@ -79,6 +93,7 @@ export class SearchService { private cacheService: CacheService, private queryService: QueryService, private idService: IdService, + private loggerService: LoggerService, ) { if (meilisearch) { this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`); @@ -109,132 +124,185 @@ export class SearchService { if (config.meilisearch?.scope) { this.meilisearchIndexScope = config.meilisearch.scope; } + + this.provider = config.fulltextSearch?.provider ?? 'sqlLike'; + this.loggerService.getLogger('SearchService').info(`-- Provider: ${this.provider}`); } @bindThis public async indexNote(note: MiNote): Promise { + if (!this.meilisearch) return; if (note.text == null && note.cw == null) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - switch (this.meilisearchIndexScope) { - case 'global': - break; + switch (this.meilisearchIndexScope) { + case 'global': + break; - case 'local': - if (note.userHost == null) break; - return; + case 'local': + if (note.userHost == null) break; + return; - default: { - if (note.userHost == null) break; - if (this.meilisearchIndexScope.includes(note.userHost)) break; - return; - } + default: { + if (note.userHost == null) break; + if (this.meilisearchIndexScope.includes(note.userHost)) break; + return; } - - await this.meilisearchNoteIndex?.addDocuments([{ - id: note.id, - createdAt: this.idService.parse(note.id).date.getTime(), - userId: note.userId, - userHost: note.userHost, - channelId: note.channelId, - cw: note.cw, - text: note.text, - tags: note.tags, - }], { - primaryKey: 'id', - }); } + + await this.meilisearchNoteIndex?.addDocuments([{ + id: note.id, + createdAt: this.idService.parse(note.id).date.getTime(), + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + cw: note.cw, + text: note.text, + tags: note.tags, + }], { + primaryKey: 'id', + }); } @bindThis public async unindexNote(note: MiNote): Promise { + if (!this.meilisearch) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - this.meilisearchNoteIndex!.deleteDocument(note.id); - } + await this.meilisearchNoteIndex?.deleteDocument(note.id); } @bindThis - public async searchNote(q: string, me: MiUser | null, opts: { - userId?: MiNote['userId'] | null; - channelId?: MiNote['channelId'] | null; - host?: string | null; - }, pagination: { - untilId?: MiNote['id']; - sinceId?: MiNote['id']; - limit?: number; - }): Promise { - if (this.meilisearch) { - const filter: Q = { - op: 'and', - qs: [], - }; - if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() }); - if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() }); - if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); - if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); - if (opts.host) { - if (opts.host === '.') { - filter.qs.push({ op: 'is null', k: 'userHost' }); - } else { - filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); - } + public async searchNote( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise { + switch (this.provider) { + case 'sqlLike': + case 'sqlPgroonga': { + // ほとんど内容に差がないのでsqlLikeとsqlPgroongaを同じ処理にしている. + // 今後の拡張で差が出る用であれば関数を分ける. + return this.searchNoteByLike(q, me, opts, pagination); } - const res = await this.meilisearchNoteIndex!.search(q, { - sort: ['createdAt:desc'], - matchingStrategy: 'all', - attributesToRetrieve: ['id', 'createdAt'], - filter: compileQuery(filter), - limit: pagination.limit, - }); - if (res.hits.length === 0) return []; - const [ - userIdsWhoMeMuting, - userIdsWhoBlockingMe, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - this.cacheService.userBlockedCache.fetch(me.id), - ]) : [new Set(), new Set()]; - const notes = (await this.notesRepository.findBy({ - id: In(res.hits.map(x => x.id)), - })).filter(note => { - if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - return true; - }); - return notes.sort((a, b) => a.id > b.id ? -1 : 1); + case 'meilisearch': { + return this.searchNoteByMeiliSearch(q, me, opts, pagination); + } + default: { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const typeCheck: never = this.provider; + return []; + } + } + } + + @bindThis + private async searchNoteByLike( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise { + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + + if (opts.userId) { + query.andWhere('note.userId = :userId', { userId: opts.userId }); + } else if (opts.channelId) { + query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + } + + query + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (this.config.fulltextSearch?.provider === 'sqlPgroonga') { + query.andWhere('note.text &@ :q', { q }); } else { - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + query.andWhere('LOWER(note.text) LIKE :q', { q: `%${ sqlLikeEscape(q.toLowerCase()) }%` }); + } - if (opts.userId) { - query.andWhere('note.userId = :userId', { userId: opts.userId }); - } else if (opts.channelId) { - query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + query.andWhere('user.host IS NULL'); + } else { + query.andWhere('user.host = :host', { host: opts.host }); } + } - query - .andWhere('LOWER(note.text) LIKE :q', { q: `%${ sqlLikeEscape(q.toLowerCase()) }%` }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); - - if (opts.host) { - if (opts.host === '.') { - query.andWhere('user.host IS NULL'); - } else { - query.andWhere('user.host = :host', { host: opts.host }); - } - } + this.queryService.generateVisibilityQuery(query, me); + if (me) this.queryService.generateMutedUserQuery(query, me); + if (me) this.queryService.generateBlockedUserQuery(query, me); + + return query.limit(pagination.limit).getMany(); + } - this.queryService.generateVisibilityQuery(query, me); - if (me) this.queryService.generateMutedUserQuery(query, me); - if (me) this.queryService.generateBlockedUserQuery(query, me); + @bindThis + private async searchNoteByMeiliSearch( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise { + if (!this.meilisearch || !this.meilisearchNoteIndex) { + throw new Error('MeiliSearch is not available'); + } + + const filter: Q = { + op: 'and', + qs: [], + }; + if (pagination.untilId) filter.qs.push({ + op: '<', + k: 'createdAt', + v: this.idService.parse(pagination.untilId).date.getTime(), + }); + if (pagination.sinceId) filter.qs.push({ + op: '>', + k: 'createdAt', + v: this.idService.parse(pagination.sinceId).date.getTime(), + }); + if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); + if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + filter.qs.push({ op: 'is null', k: 'userHost' }); + } else { + filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); + } + } - return await query.limit(pagination.limit).getMany(); + const res = await this.meilisearchNoteIndex.search(q, { + sort: ['createdAt:desc'], + matchingStrategy: 'all', + attributesToRetrieve: ['id', 'createdAt'], + filter: compileQuery(filter), + limit: pagination.limit, + }); + if (res.hits.length === 0) { + return []; } + + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = me + ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), + ]) + : [new Set(), new Set()]; + const notes = (await this.notesRepository.findBy({ + id: In(res.hits.map(x => x.id)), + })).filter(note => { + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + return true; + }); + + return notes.sort((a, b) => a.id > b.id ? -1 : 1); } } -- cgit v1.2.3-freya