diff options
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | elasticsearch/README.md | 6 | ||||
| -rw-r--r-- | elasticsearch/mappings.json | 65 | ||||
| -rw-r--r-- | package.json | 20 | ||||
| -rw-r--r-- | src/client/app/common/views/components/reaction-picker.vue | 2 | ||||
| -rw-r--r-- | src/client/app/desktop/views/pages/search.vue | 61 | ||||
| -rw-r--r-- | src/client/app/mobile/views/pages/search.vue | 58 | ||||
| -rw-r--r-- | src/config/types.ts | 1 | ||||
| -rw-r--r-- | src/db/elasticsearch.ts | 68 | ||||
| -rw-r--r-- | src/index.ts | 4 | ||||
| -rw-r--r-- | src/models/user.ts | 1 | ||||
| -rw-r--r-- | src/server/api/endpoints/notes/search.ts | 63 | ||||
| -rw-r--r-- | src/server/api/private/signup.ts | 1 | ||||
| -rw-r--r-- | src/services/note/create.ts | 16 |
14 files changed, 209 insertions, 159 deletions
@@ -20,7 +20,7 @@ ultimately sophisticated professional microblogging software. ---------------------------------------------------------------- * Reactions * User lists -* Customizable column view (known as MisskeyDeck) +* Customizable column view (called MisskeyDeck) * and widgets! * Private messages * Mute diff --git a/elasticsearch/README.md b/elasticsearch/README.md deleted file mode 100644 index c7fcb245f0..0000000000 --- a/elasticsearch/README.md +++ /dev/null @@ -1,6 +0,0 @@ -How to create indexes -===================== - -``` shell -curl -XPOST localhost:9200/misskey -d @path/to/mappings.json -``` diff --git a/elasticsearch/mappings.json b/elasticsearch/mappings.json deleted file mode 100644 index 654ab17450..0000000000 --- a/elasticsearch/mappings.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "settings": { - "analysis": { - "analyzer": { - "bigram": { - "tokenizer": "bigram_tokenizer" - } - }, - "tokenizer": { - "bigram_tokenizer": { - "type": "nGram", - "min_gram": 2, - "max_gram": 2, - "token_chars": [ - "letter", - "digit" - ] - } - } - } - }, - "mappings": { - "user": { - "properties": { - "username": { - "type": "string", - "index": "analyzed", - "analyzer": "bigram" - }, - "name": { - "type": "string", - "index": "analyzed", - "analyzer": "bigram" - }, - "bio": { - "type": "string", - "index": "analyzed", - "analyzer": "kuromoji" - } - } - }, - "post": { - "properties": { - "text": { - "type": "string", - "index": "analyzed", - "analyzer": "kuromoji" - } - } - }, - "drive_file": { - "properties": { - "name": { - "type": "string", - "index": "analyzed", - "analyzer": "kuromoji" - }, - "user": { - "type": "string", - "index": "not_analyzed" - } - } - } - } -} diff --git a/package.json b/package.json index a9ae2ba591..b5cc2457c7 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "misskey", "author": "syuilo <i@syuilo.com>", - "version": "4.15.0", - "clientVersion": "1.0.6878", + "version": "4.16.1", + "clientVersion": "1.0.6952", "codename": "nighthike", "main": "./built/index.js", "private": true, @@ -46,7 +46,7 @@ "@types/inquirer": "0.0.42", "@types/is-root": "1.0.0", "@types/is-url": "1.2.28", - "@types/js-yaml": "3.11.1", + "@types/js-yaml": "3.11.2", "@types/jsdom": "11.0.6", "@types/koa": "2.0.46", "@types/koa-bodyparser": "5.0.0", @@ -63,7 +63,7 @@ "@types/license-checker": "15.0.0", "@types/mkdirp": "0.5.2", "@types/mocha": "5.2.3", - "@types/mongodb": "3.1.0", + "@types/mongodb": "3.1.1", "@types/ms": "0.7.30", "@types/node": "10.5.1", "@types/nopt": "3.0.29", @@ -80,7 +80,7 @@ "@types/speakeasy": "2.0.2", "@types/tmp": "0.0.33", "@types/uuid": "3.4.3", - "@types/webpack": "4.4.4", + "@types/webpack": "4.4.5", "@types/webpack-stream": "3.2.10", "@types/websocket": "0.0.39", "@types/ws": "5.1.2", @@ -98,8 +98,8 @@ "deepcopy": "0.6.3", "diskusage": "0.2.4", "dompurify": "1.0.5", - "elasticsearch": "15.0.0", - "element-ui": "2.4.2", + "elasticsearch": "15.1.1", + "element-ui": "2.4.3", "emojilib": "2.2.12", "escape-regexp": "0.0.1", "eslint": "5.0.1", @@ -126,7 +126,7 @@ "gulp-util": "3.0.8", "hard-source-webpack-plugin": "0.10.1", "highlight.js": "9.12.0", - "html-minifier": "3.5.17", + "html-minifier": "3.5.18", "http-signature": "1.2.0", "inquirer": "6.0.0", "is-root": "2.0.0", @@ -156,7 +156,7 @@ "monk": "6.0.6", "ms": "2.1.1", "nan": "2.10.0", - "node-sass": "4.9.0", + "node-sass": "4.9.1", "node-sass-json-importer": "3.3.1", "nopt": "4.0.1", "nprogress": "0.2.0", @@ -213,7 +213,7 @@ "vuex-persistedstate": "^2.5.4", "web-push": "3.3.2", "webfinger.js": "2.6.6", - "webpack": "4.14.0", + "webpack": "4.15.0", "webpack-cli": "3.0.8", "websocket": "1.0.26", "ws": "5.2.1", diff --git a/src/client/app/common/views/components/reaction-picker.vue b/src/client/app/common/views/components/reaction-picker.vue index ed7aedb58e..5a149cc4d1 100644 --- a/src/client/app/common/views/components/reaction-picker.vue +++ b/src/client/app/common/views/components/reaction-picker.vue @@ -183,7 +183,7 @@ root(isDark) border-right solid $balloon-size transparent border-bottom solid $balloon-size $bgcolor - &.compact + &.big > div width 280px diff --git a/src/client/app/desktop/views/pages/search.vue b/src/client/app/desktop/views/pages/search.vue index e79ac1c739..6ebb83cac8 100644 --- a/src/client/app/desktop/views/pages/search.vue +++ b/src/client/app/desktop/views/pages/search.vue @@ -7,19 +7,13 @@ <mk-ellipsis-icon/> </div> <p :class="$style.empty" v-if="!fetching && empty">%fa:search%「{{ q }}」に関する投稿は見つかりませんでした。</p> - <mk-notes ref="timeline" :class="$style.notes" :notes="notes"> - <div slot="footer"> - <template v-if="!moreFetching">%fa:search%</template> - <template v-if="moreFetching">%fa:spinner .pulse .fw%</template> - </div> - </mk-notes> + <mk-notes ref="timeline" :class="$style.notes" :more="existMore ? more : null"/> </mk-ui> </template> <script lang="ts"> import Vue from 'vue'; import Progress from '../../../common/scripts/loading'; -import parse from '../../../common/scripts/parse-search-query'; const limit = 20; @@ -30,16 +24,13 @@ export default Vue.extend({ moreFetching: false, existMore: false, offset: 0, - notes: [] + empty: false }; }, watch: { $route: 'fetch' }, computed: { - empty(): boolean { - return this.notes.length == 0; - }, q(): string { return this.$route.query.q; } @@ -66,39 +57,43 @@ export default Vue.extend({ this.fetching = true; Progress.start(); - (this as any).api('notes/search', Object.assign({ - limit: limit + 1, - offset: this.offset - }, parse(this.q))).then(notes => { - if (notes.length == limit + 1) { - notes.pop(); - this.existMore = true; - } - this.notes = notes; - this.fetching = false; - Progress.done(); - }); + (this.$refs.timeline as any).init(() => new Promise((res, rej) => { + (this as any).api('notes/search', { + limit: limit + 1, + offset: this.offset, + query: this.q + }).then(notes => { + if (notes.length == 0) this.empty = true; + if (notes.length == limit + 1) { + notes.pop(); + this.existMore = true; + } + res(notes); + this.fetching = false; + Progress.done(); + }, rej); + })); }, more() { - if (this.moreFetching || this.fetching || this.notes.length == 0 || !this.existMore) return; this.offset += limit; - this.moreFetching = true; - return (this as any).api('notes/search', Object.assign({ + + const promise = (this as any).api('notes/search', { limit: limit + 1, - offset: this.offset - }, parse(this.q))).then(notes => { + offset: this.offset, + query: this.q + }); + + promise.then(notes => { if (notes.length == limit + 1) { notes.pop(); } else { this.existMore = false; } - this.notes = this.notes.concat(notes); + notes.forEach(n => (this.$refs.timeline as any).append(n)); this.moreFetching = false; }); - }, - onScroll() { - const current = window.scrollY + window.innerHeight; - if (current > document.body.offsetHeight - 16) this.more(); + + return promise; } } }); diff --git a/src/client/app/mobile/views/pages/search.vue b/src/client/app/mobile/views/pages/search.vue index 9850fbcfb4..2559922efb 100644 --- a/src/client/app/mobile/views/pages/search.vue +++ b/src/client/app/mobile/views/pages/search.vue @@ -1,14 +1,10 @@ <template> <mk-ui> <span slot="header">%fa:search% {{ q }}</span> - <main v-if="!fetching"> - <mk-notes :class="$style.notes" :notes="notes"> - <span v-if="notes.length == 0">{{ '%i18n:@empty%'.replace('{}', q) }}</span> - <button v-if="existMore" @click="more" :disabled="fetching" slot="tail"> - <span v-if="!fetching">%i18n:@load-more%</span> - <span v-if="fetching">%i18n:common.loading%<mk-ellipsis/></span> - </button> - </mk-notes> + + <main> + <p v-if="!fetching && empty">%fa:search%「{{ q }}」に関する投稿は見つかりませんでした。</p> + <mk-notes ref="timeline" :more="existMore ? more : null"/> </main> </mk-ui> </template> @@ -16,7 +12,6 @@ <script lang="ts"> import Vue from 'vue'; import Progress from '../../../common/scripts/loading'; -import parse from '../../../common/scripts/parse-search-query'; const limit = 20; @@ -24,8 +19,9 @@ export default Vue.extend({ data() { return { fetching: true, + moreFetching: false, existMore: false, - notes: [], + empty: false, offset: 0 }; }, @@ -47,31 +43,43 @@ export default Vue.extend({ this.fetching = true; Progress.start(); - (this as any).api('notes/search', Object.assign({ - limit: limit + 1 - }, parse(this.q))).then(notes => { - if (notes.length == limit + 1) { - notes.pop(); - this.existMore = true; - } - this.notes = notes; - this.fetching = false; - Progress.done(); - }); + (this.$refs.timeline as any).init(() => new Promise((res, rej) => { + (this as any).api('notes/search', { + limit: limit + 1, + offset: this.offset, + query: this.q + }).then(notes => { + if (notes.length == 0) this.empty = true; + if (notes.length == limit + 1) { + notes.pop(); + this.existMore = true; + } + res(notes); + this.fetching = false; + Progress.done(); + }, rej); + })); }, more() { this.offset += limit; - (this as any).api('notes/search', Object.assign({ + + const promise = (this as any).api('notes/search', { limit: limit + 1, - offset: this.offset - }, parse(this.q))).then(notes => { + offset: this.offset, + query: this.q + }); + + promise.then(notes => { if (notes.length == limit + 1) { notes.pop(); } else { this.existMore = false; } - this.notes = this.notes.concat(notes); + notes.forEach(n => (this.$refs.timeline as any).append(n)); + this.moreFetching = false; }); + + return promise; } } }); diff --git a/src/config/types.ts b/src/config/types.ts index 49eeac508b..b0776fd9c1 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -34,7 +34,6 @@ export type Source = { pass: string; }; elasticsearch: { - enable: boolean; host: string; port: number; pass: string; diff --git a/src/db/elasticsearch.ts b/src/db/elasticsearch.ts index 957b7ad97d..4acff40793 100644 --- a/src/db/elasticsearch.ts +++ b/src/db/elasticsearch.ts @@ -1,22 +1,64 @@ import * as elasticsearch from 'elasticsearch'; import config from '../config'; +const index = { + settings: { + analysis: { + analyzer: { + bigram: { + tokenizer: 'bigram_tokenizer' + } + }, + tokenizer: { + bigram_tokenizer: { + type: 'nGram', + min_gram: 2, + max_gram: 2 + } + } + } + }, + mappings: { + note: { + properties: { + text: { + type: 'text', + index: true, + analyzer: 'bigram' + } + } + } + } +}; + // Init ElasticSearch connection -const client = new elasticsearch.Client({ +const client = config.elasticsearch ? new elasticsearch.Client({ host: `${config.elasticsearch.host}:${config.elasticsearch.port}` -}); +}) : null; -// Send a HEAD request -client.ping({ - // Ping usually has a 3000ms timeout - requestTimeout: Infinity, +if (client) { + // Send a HEAD request + client.ping({ + // Ping usually has a 3000ms timeout + requestTimeout: 30000 + }, error => { + if (error) { + console.error('elasticsearch is down!'); + } else { + console.log('elasticsearch is available!'); + } + }); - // Undocumented params are appended to the query string - hello: 'elasticsearch!' -} as any, error => { - if (error) { - console.error('elasticsearch is down!'); - } -}); + client.indices.exists({ + index: 'misskey' + }).then(exist => { + if (exist) return; + + client.indices.create({ + index: 'misskey', + body: index + }); + }); +} export default client; diff --git a/src/index.ts b/src/index.ts index c89252dfc2..1a76044572 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,7 +19,7 @@ import MachineInfo from './utils/machineInfo'; import DependencyInfo from './utils/dependencyInfo'; import serverStats from './daemons/server-stats'; import notesStats from './daemons/notes-stats'; - +import db from './db/mongodb'; import loadConfig from './config/load'; import { Config } from './config/types'; @@ -204,4 +204,6 @@ process.on('uncaughtException', err => { // Dying away... process.on('exit', code => { Logger.info(`The process is going exit (${code})`); + + db.close(); }); diff --git a/src/models/user.ts b/src/models/user.ts index 0d1be439e9..ee95854b95 100644 --- a/src/models/user.ts +++ b/src/models/user.ts @@ -77,7 +77,6 @@ export interface ILocalUser extends IUserBase { host: null; keypair: string; email: string; - links: string[]; password: string; token: string; twitter: { diff --git a/src/server/api/endpoints/notes/search.ts b/src/server/api/endpoints/notes/search.ts new file mode 100644 index 0000000000..20c628b84d --- /dev/null +++ b/src/server/api/endpoints/notes/search.ts @@ -0,0 +1,63 @@ +import $ from 'cafy'; +import * as mongo from 'mongodb'; +import Note from '../../../../models/note'; +import { ILocalUser } from '../../../../models/user'; +import { pack } from '../../../../models/note'; +import es from '../../../../db/elasticsearch'; + +module.exports = (params: any, me: ILocalUser) => new Promise(async (res, rej) => { + // Get 'query' parameter + const [query, queryError] = $.str.get(params.query); + if (queryError) return rej('invalid query param'); + + // Get 'offset' parameter + const [offset = 0, offsetErr] = $.num.optional().min(0).get(params.offset); + if (offsetErr) return rej('invalid offset param'); + + // Get 'limit' parameter + const [limit = 10, limitErr] = $.num.optional().range(1, 30).get(params.limit); + if (limitErr) return rej('invalid limit param'); + + es.search({ + index: 'misskey', + type: 'note', + body: { + size: limit, + from: offset, + query: { + simple_query_string: { + fields: ['text'], + query: query, + default_operator: 'and' + } + }, + sort: [ + { _doc: 'desc' } + ] + } + }, async (error, response) => { + if (error) { + console.error(error); + return res(500); + } + + if (response.hits.total === 0) { + return res([]); + } + + const hits = response.hits.hits.map(hit => new mongo.ObjectID(hit._id)); + + // Fetch found notes + const notes = await Note.find({ + _id: { + $in: hits + } + }, { + sort: { + _id: -1 + } + }); + + res(await Promise.all(notes.map(note => pack(note, me)))); + }); +}); diff --git a/src/server/api/private/signup.ts b/src/server/api/private/signup.ts index cb47d400b0..023a36586f 100644 --- a/src/server/api/private/signup.ts +++ b/src/server/api/private/signup.ts @@ -77,7 +77,6 @@ export default async (ctx: Koa.Context) => { keypair: generateKeypair(), token: secret, email: null, - links: null, password: hash, profile: { bio: null, diff --git a/src/services/note/create.ts b/src/services/note/create.ts index a793c8e580..89f9a91c9b 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -1,3 +1,4 @@ +import es from '../../db/elasticsearch'; import Note, { pack, INote } from '../../models/note'; import User, { isLocalUser, IUser, isRemoteUser, IRemoteUser, ILocalUser } from '../../models/user'; import stream, { publishLocalTimelineStream, publishGlobalTimelineStream, publishUserListStream } from '../../publishers/stream'; @@ -18,6 +19,7 @@ import { IApp } from '../../models/app'; import UserList from '../../models/user-list'; import resolveUser from '../../remote/resolve-user'; import Meta from '../../models/meta'; +import config from '../../config'; type Type = 'reply' | 'renote' | 'quote' | 'mention'; @@ -366,7 +368,7 @@ export default async (user: IUser, data: { watch(user._id, data.reply); } - // (自分自身へのリプライでない限りは)通知を作成 + // 通知 nm.push(data.reply.userId, 'reply'); } @@ -427,4 +429,16 @@ export default async (user: IUser, data: { }); } } + + // Register to search database + if (note.text && config.elasticsearch) { + es.index({ + index: 'misskey', + type: 'note', + id: note._id.toString(), + body: { + text: note.text + } + }); + } }); |