diff options
Diffstat (limited to 'packages/backend/src/core/SearchService.ts')
| -rw-r--r-- | packages/backend/src/core/SearchService.ts | 349 |
1 files changed, 209 insertions, 140 deletions
diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index e8a98ca617..6e46fb798c 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -6,16 +6,17 @@ import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; +import { type Config, FulltextSearchProvider } from '@/config.js'; import { bindThis } from '@/decorators.js'; import { MiNote } from '@/models/Note.js'; -import { MiUser } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; +import { MiUser } from '@/models/_.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; +import { LoggerService } from '@/core/LoggerService.js'; import type { Index, MeiliSearch } from 'meilisearch'; type K = string; @@ -27,40 +28,12 @@ type Q = { op: '<', k: K, v: number } | { op: '>=', k: K, v: number } | { op: '<=', k: K, v: number } | - { op: 'is null', k: K} | - { op: 'is not null', k: K} | + { op: 'is null', k: K } | + { op: 'is not null', k: K } | { op: 'and', qs: Q[] } | { op: 'or', qs: Q[] } | { op: 'not', q: Q }; -function compileValue(value: V): string { - if (typeof value === 'string') { - return `'${value}'`; // TODO: escape - } else if (typeof value === 'number') { - return value.toString(); - } else if (typeof value === 'boolean') { - return value.toString(); - } - throw new Error('unrecognized value'); -} - -function compileQuery(q: Q): string { - switch (q.op) { - case '=': return `(${q.k} = ${compileValue(q.v)})`; - case '!=': return `(${q.k} != ${compileValue(q.v)})`; - case '>': return `(${q.k} > ${compileValue(q.v)})`; - case '<': return `(${q.k} < ${compileValue(q.v)})`; - case '>=': return `(${q.k} >= ${compileValue(q.v)})`; - case '<=': return `(${q.k} <= ${compileValue(q.v)})`; - case 'and': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' AND ') })`; - case 'or': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' OR ') })`; - case 'is null': return `(${q.k} IS NULL)`; - case 'is not null': return `(${q.k} IS NOT NULL)`; - case 'not': return `(NOT ${compileQuery(q.q)})`; - default: throw new Error('unrecognized query operator'); - } -} - const fileTypes = { image: [ 'image/webp', @@ -115,10 +88,54 @@ const fileTypes = { export const fileTypeCategories = ['image', 'video', 'audio', 'module', 'flash', null] as const; export type FileTypeCategory = typeof fileTypeCategories[number]; +export type SearchOpts = { + userId?: MiNote['userId'] | null; + channelId?: MiNote['channelId'] | null; + host?: string | null; + filetype?: FileTypeCategory; + order?: string | null; + disableMeili?: boolean | null; +}; + +export type SearchPagination = { + untilId?: MiNote['id']; + sinceId?: MiNote['id']; + limit: number; +}; + +function compileValue(value: V): string { + if (typeof value === 'string') { + return `'${value}'`; // TODO: escape + } else if (typeof value === 'number') { + return value.toString(); + } else if (typeof value === 'boolean') { + return value.toString(); + } + throw new Error('unrecognized value'); +} + +function compileQuery(q: Q): string { + switch (q.op) { + case '=': return `(${q.k} = ${compileValue(q.v)})`; + case '!=': return `(${q.k} != ${compileValue(q.v)})`; + case '>': return `(${q.k} > ${compileValue(q.v)})`; + case '<': return `(${q.k} < ${compileValue(q.v)})`; + case '>=': return `(${q.k} >= ${compileValue(q.v)})`; + case '<=': return `(${q.k} <= ${compileValue(q.v)})`; + case 'and': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' AND ') })`; + case 'or': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' OR ') })`; + case 'is null': return `(${q.k} IS NULL)`; + case 'is not null': return `(${q.k} IS NOT NULL)`; + case 'not': return `(NOT ${compileQuery(q.q)})`; + default: throw new Error('unrecognized query operator'); + } +} + @Injectable() export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; - private meilisearchNoteIndex: Index | null = null; + private readonly meilisearchNoteIndex: Index | null = null; + private readonly provider: FulltextSearchProvider; constructor( @Inject(DI.config) @@ -133,6 +150,7 @@ export class SearchService { private cacheService: CacheService, private queryService: QueryService, private idService: IdService, + private loggerService: LoggerService, ) { if (meilisearch) { this.meilisearchNoteIndex = meilisearch.index(`${this.config.meilisearch?.index}---notes`); @@ -164,144 +182,195 @@ export class SearchService { if (this.config.meilisearch?.scope) { this.meilisearchIndexScope = this.config.meilisearch.scope; } + + this.provider = config.fulltextSearch?.provider ?? 'sqlLike'; + this.loggerService.getLogger('SearchService').info(`-- Provider: ${this.provider}`); } @bindThis public async indexNote(note: MiNote): Promise<void> { + if (!this.meilisearch) return; if (note.text == null && note.cw == null) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - switch (this.meilisearchIndexScope) { - case 'global': - break; + switch (this.meilisearchIndexScope) { + case 'global': + break; - case 'local': - if (note.userHost == null) break; - return; + case 'local': + if (note.userHost == null) break; + return; - default: { - if (note.userHost == null) break; - if (this.meilisearchIndexScope.includes(note.userHost)) break; - return; - } + default: { + if (note.userHost == null) break; + if (this.meilisearchIndexScope.includes(note.userHost)) break; + return; } - - await this.meilisearchNoteIndex?.addDocuments([{ - id: note.id, - createdAt: this.idService.parse(note.id).date.getTime(), - userId: note.userId, - userHost: note.userHost, - channelId: note.channelId, - cw: note.cw, - text: note.text, - tags: note.tags, - attachedFileTypes: note.attachedFileTypes, - }], { - primaryKey: 'id', - }); } + + await this.meilisearchNoteIndex?.addDocuments([{ + id: note.id, + createdAt: this.idService.parse(note.id).date.getTime(), + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + cw: note.cw, + text: note.text, + tags: note.tags, + attachedFileTypes: note.attachedFileTypes, + }], { + primaryKey: 'id', + }); } @bindThis public async unindexNote(note: MiNote): Promise<void> { + if (!this.meilisearch) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - this.meilisearchNoteIndex!.deleteDocument(note.id); - } + await this.meilisearchNoteIndex?.deleteDocument(note.id); } @bindThis - public async searchNote(q: string, me: MiUser | null, opts: { - userId?: MiNote['userId'] | null; - channelId?: MiNote['channelId'] | null; - host?: string | null; - filetype?: FileTypeCategory; - order?: string | null; - disableMeili?: boolean | null; - }, pagination: { - untilId?: MiNote['id']; - sinceId?: MiNote['id']; - limit?: number; - }): Promise<MiNote[]> { - if (this.meilisearch && !opts.disableMeili) { - const filter: Q = { - op: 'and', - qs: [], - }; - if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() }); - if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() }); - if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); - if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); - if (opts.host) { - if (opts.host === '.') { - filter.qs.push({ op: 'is null', k: 'userHost' }); - } else { - filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); - } + public async searchNote( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + switch (this.provider) { + case 'sqlLike': + case 'sqlPgroonga': { + // ほとんど内容に差がないのでsqlLikeとsqlPgroongaを同じ処理にしている. + // 今後の拡張で差が出る用であれば関数を分ける. + return this.searchNoteByLike(q, me, opts, pagination); } - if (opts.filetype) { - const filters = fileTypes[opts.filetype].map(mime => ({ op: '=' as const, k: 'attachedFileTypes', v: mime })); - filter.qs.push({ op: 'or', qs: filters }); + case 'meilisearch': { + return this.searchNoteByMeiliSearch(q, me, opts, pagination); } - const res = await this.meilisearchNoteIndex!.search(q, { - sort: [`createdAt:${opts.order ? opts.order : 'desc'}`], - matchingStrategy: 'all', - attributesToRetrieve: ['id', 'createdAt'], - filter: compileQuery(filter), - limit: pagination.limit, - }); - if (res.hits.length === 0) return []; - const [ - userIdsWhoMeMuting, - userIdsWhoBlockingMe, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - this.cacheService.userBlockedCache.fetch(me.id), - ]) : [new Set<string>(), new Set<string>()]; - const notes = (await this.notesRepository.findBy({ - id: In(res.hits.map(x => x.id)), - })).filter(note => { - if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - return true; - }); - return notes.sort((a, b) => a.id > b.id ? -1 : 1); + default: { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const typeCheck: never = this.provider; + return []; + } + } + } + + @bindThis + private async searchNoteByLike( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + + if (opts.userId) { + query.andWhere('note.userId = :userId', { userId: opts.userId }); + } else if (opts.channelId) { + query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + } + + query + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (this.config.fulltextSearch?.provider === 'sqlPgroonga') { + query.andWhere('note.text &@~ :q', { q }); } else { - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + query.andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` }); + } - if (opts.userId) { - query.andWhere('note.userId = :userId', { userId: opts.userId }); - } else if (opts.channelId) { - query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + query.andWhere('note.userHost IS NULL'); + } else { + query.andWhere('note.userHost = :host', { host: opts.host }); } + } - query - .andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + if (opts.filetype) { + query.andWhere('note."attachedFileTypes" && :types', { types: fileTypes[opts.filetype] }); + } - if (opts.host) { - if (opts.host === '.') { - query.andWhere('note.userHost IS NULL'); - } else { - query.andWhere('note.userHost = :host', { host: opts.host }); - } - } + this.queryService.generateVisibilityQuery(query, me); + if (me) this.queryService.generateMutedUserQuery(query, me); + if (me) this.queryService.generateBlockedUserQuery(query, me); - if (opts.filetype) { - query.andWhere('note."attachedFileTypes" && :types', { types: fileTypes[opts.filetype] }); + return await query.limit(pagination.limit).getMany(); + } + + @bindThis + private async searchNoteByMeiliSearch( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + if (!this.meilisearch || !this.meilisearchNoteIndex) { + throw new Error('MeiliSearch is not available'); + } + + const filter: Q = { + op: 'and', + qs: [], + }; + if (pagination.untilId) filter.qs.push({ + op: '<', + k: 'createdAt', + v: this.idService.parse(pagination.untilId).date.getTime(), + }); + if (pagination.sinceId) filter.qs.push({ + op: '>', + k: 'createdAt', + v: this.idService.parse(pagination.sinceId).date.getTime(), + }); + if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); + if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + filter.qs.push({ op: 'is null', k: 'userHost' }); + } else { + filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); } + } - this.queryService.generateVisibilityQuery(query, me); - if (me) this.queryService.generateMutedUserQuery(query, me); - if (me) this.queryService.generateBlockedUserQuery(query, me); + if (opts.filetype) { + const filters = fileTypes[opts.filetype].map(mime => ({ op: '=' as const, k: 'attachedFileTypes', v: mime })); + filter.qs.push({ op: 'or', qs: filters }); + } - return await query.limit(pagination.limit).getMany(); + const res = await this.meilisearchNoteIndex.search(q, { + sort: [`createdAt:${opts.order ? opts.order : 'desc'}`], + matchingStrategy: 'all', + attributesToRetrieve: ['id', 'createdAt'], + filter: compileQuery(filter), + limit: pagination.limit, + }); + if (res.hits.length === 0) { + return []; } + + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = me + ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), + ]) + : [new Set<string>(), new Set<string>()]; + const notes = (await this.notesRepository.findBy({ + id: In(res.hits.map(x => x.id)), + })).filter(note => { + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + return true; + }); + + return notes.sort((a, b) => a.id > b.id ? -1 : 1); } } |