From f0a29721c9fb10f97faf386bc9d6b1b2fad97895 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 7 Apr 2019 21:50:36 +0900 Subject: Use PostgreSQL instead of MongoDB (#4572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * Update note.ts * Update timeline.ts * Update core.ts * wip * Update generate-visibility-query.ts * wip * wip * wip * wip * wip * Update global-timeline.ts * wip * wip * wip * Update vote.ts * wip * wip * Update create.ts * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * Update files.ts * wip * wip * Update CONTRIBUTING.md * wip * wip * wip * wip * wip * wip * wip * wip * Update read-notification.ts * wip * wip * wip * wip * wip * wip * wip * Update cancel.ts * wip * wip * wip * Update show.ts * wip * wip * Update gen-id.ts * Update create.ts * Update id.ts * wip * wip * wip * wip * wip * wip * wip * Docker: Update files about Docker (#4599) * Docker: Use cache if files used by `yarn install` was not updated This patch reduces the number of times to installing node_modules. For example, `yarn install` step will be skipped when only ".config/default.yml" is updated. * Docker: Migrate MongoDB to Postgresql Misskey uses Postgresql as a database instead of Mongodb since version 11. * Docker: Uncomment about data persistence This patch will save a lot of databases. * wip * wip * wip * Update activitypub.ts * wip * wip * wip * Update logs.ts * wip * Update drive-file.ts * Update register.ts * wip * wip * Update mentions.ts * wip * wip * wip * Update recommendation.ts * wip * Update index.ts * wip * Update recommendation.ts * Doc: Update docker.ja.md and docker.en.md (#1) (#4608) Update how to set up misskey. * wip * :v: * wip * Update note.ts * Update postgre.ts * wip * wip * wip * wip * Update add-file.ts * wip * wip * wip * Clean up * Update logs.ts * wip * :pizza: * wip * Ad notes * wip * Update api-visibility.ts * Update note.ts * Update add-file.ts * tests * tests * Update postgre.ts * Update utils.ts * wip * wip * Refactor * wip * Refactor * wip * wip * Update show-users.ts * Update update-instance.ts * wip * Update feed.ts * Update outbox.ts * Update outbox.ts * Update user.ts * wip * Update list.ts * Update update-hashtag.ts * wip * Update update-hashtag.ts * Refactor * Update update.ts * wip * wip * :v: * clean up * docs * Update push.ts * wip * Update api.ts * wip * :v: * Update make-pagination-query.ts * :v: * Delete hashtags.ts * Update instances.ts * Update instances.ts * Update create.ts * Update search.ts * Update reversi-game.ts * Update signup.ts * Update user.ts * id * Update example.yml * :art: * objectid * fix * reversi * reversi * Fix bug of chart engine * Add test of chart engine * Improve test * Better testing * Improve chart engine * Refactor * Add test of chart engine * Refactor * Add chart test * Fix bug * コミットし忘れ * Refactoring * :v: * Add tests * Add test * Extarct note tests * Refactor * 存在しないユーザーにメンションできなくなっていた問題を修正 * Fix bug * Update update-meta.ts * Fix bug * Update mention.vue * Fix bug * Update meta.ts * Update CONTRIBUTING.md * Fix bug * Fix bug * Fix bug * Clean up * Clean up * Update notification.ts * Clean up * Add mute tests * Add test * Refactor * Add test * Fix test * Refactor * Refactor * Add tests * Update utils.ts * Update utils.ts * Fix test * Update package.json * Update update.ts * Update manifest.ts * Fix bug * Fix bug * Add test * :art: * Update endpoint permissions * Updaye permisison * Update person.ts #4299 * データベースと同期しないように * Fix bug * Fix bug * Update reversi-game.ts * Use a feature of Node v11.7.0 to extract a public key (#4644) * wip * wip * :v: * Refactoring #1540 * test * test * test * test * test * test * test * Fix bug * Fix test * :sushi: * wip * #4471 * Add test for #4335 * Refactor * Fix test * Add tests * :clock4: * Fix bug * Add test * Add test * rename * Fix bug --- src/tools/add-emoji.ts | 6 ++- src/tools/clean-remote-files.ts | 21 ++++------ src/tools/move-drive-files.ts | 83 ---------------------------------------- src/tools/show-signin-history.ts | 13 +++---- 4 files changed, 18 insertions(+), 105 deletions(-) delete mode 100644 src/tools/move-drive-files.ts (limited to 'src/tools') diff --git a/src/tools/add-emoji.ts b/src/tools/add-emoji.ts index 2aa99e37ae..a75798bdad 100644 --- a/src/tools/add-emoji.ts +++ b/src/tools/add-emoji.ts @@ -1,9 +1,11 @@ -import Emoji from '../models/emoji'; +import { Emojis } from '../models'; +import { genId } from '../misc/gen-id'; async function main(name: string, url: string, alias?: string): Promise { const aliases = alias != null ? [ alias ] : []; - await Emoji.insert({ + await Emojis.save({ + id: genId(), host: null, name, url, diff --git a/src/tools/clean-remote-files.ts b/src/tools/clean-remote-files.ts index 28c76345c7..f64affea97 100644 --- a/src/tools/clean-remote-files.ts +++ b/src/tools/clean-remote-files.ts @@ -1,18 +1,13 @@ import * as promiseLimit from 'promise-limit'; -import DriveFile, { IDriveFile } from '../models/drive-file'; import del from '../services/drive/delete-file'; +import { DriveFiles } from '../models'; +import { Not } from 'typeorm'; +import { DriveFile } from '../models/entities/drive-file'; const limit = promiseLimit(16); -DriveFile.find({ - 'metadata._user.host': { - $ne: null - }, - 'metadata.deletedAt': { $exists: false } -}, { - fields: { - _id: true - } +DriveFiles.find({ + userHost: Not(null) }).then(async files => { console.log(`there is ${files.length} files`); @@ -21,10 +16,10 @@ DriveFile.find({ console.log('ALL DONE'); }); -async function job(file: IDriveFile): Promise { - file = await DriveFile.findOne({ _id: file._id }); +async function job(file: DriveFile): Promise { + file = await DriveFiles.findOne(file.id); await del(file, true); - console.log('done', file._id); + console.log('done', file.id); } diff --git a/src/tools/move-drive-files.ts b/src/tools/move-drive-files.ts deleted file mode 100644 index 8a1e944503..0000000000 --- a/src/tools/move-drive-files.ts +++ /dev/null @@ -1,83 +0,0 @@ -import * as Minio from 'minio'; -import * as uuid from 'uuid'; -import * as promiseLimit from 'promise-limit'; -import DriveFile, { DriveFileChunk, getDriveFileBucket, IDriveFile } from '../models/drive-file'; -import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../models/drive-file-thumbnail'; -import config from '../config'; - -const limit = promiseLimit(16); - -DriveFile.find({ - $or: [{ - 'metadata.withoutChunks': { $exists: false } - }, { - 'metadata.withoutChunks': false - }], - 'metadata.deletedAt': { $exists: false } -}, { - fields: { - _id: true - } -}).then(async files => { - console.log(`there is ${files.length} files`); - - await Promise.all(files.map(file => limit(() => job(file)))); - - console.log('ALL DONE'); -}); - -async function job(file: IDriveFile): Promise { - file = await DriveFile.findOne({ _id: file._id }); - - const minio = new Minio.Client(config.drive.config); - - const name = file.filename.substr(0, 50); - const keyDir = `${config.drive.prefix}/${uuid.v4()}`; - const key = `${keyDir}/${name}`; - const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`; - const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`; - - const baseUrl = config.drive.baseUrl - || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`; - - const bucket = await getDriveFileBucket(); - const readable = bucket.openDownloadStream(file._id); - - await minio.putObject(config.drive.bucket, key, readable, file.length, { - 'Content-Type': file.contentType, - 'Cache-Control': 'max-age=31536000, immutable' - }); - - await DriveFile.findOneAndUpdate({ _id: file._id }, { - $set: { - 'metadata.withoutChunks': true, - 'metadata.storage': 'minio', - 'metadata.storageProps': { - key: key, - thumbnailKey: thumbnailKey - }, - 'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`, - } - }); - - // チャンクをすべて削除 - await DriveFileChunk.remove({ - files_id: file._id - }); - - //#region サムネイルもあれば削除 - const thumbnail = await DriveFileThumbnail.findOne({ - 'metadata.originalId': file._id - }); - - if (thumbnail) { - await DriveFileThumbnailChunk.remove({ - files_id: thumbnail._id - }); - - await DriveFileThumbnail.remove({ _id: thumbnail._id }); - } - //#endregion - - console.log('done', file._id); -} diff --git a/src/tools/show-signin-history.ts b/src/tools/show-signin-history.ts index e770710322..584bece6bb 100644 --- a/src/tools/show-signin-history.ts +++ b/src/tools/show-signin-history.ts @@ -1,3 +1,5 @@ +import { Users, Signins } from '../models'; + // node built/tools/show-signin-history username // => {Success} {Date} {IPAddrsss} @@ -7,19 +9,16 @@ // node built/tools/show-signin-history username all // with full request headers -import User from '../models/user'; -import Signin from '../models/signin'; - async function main(username: string, headers: string[]) { - const user = await User.findOne({ + const user = await Users.findOne({ host: null, usernameLower: username.toLowerCase(), }); - if (user === null) throw 'User not found'; + if (user == null) throw 'User not found'; - const history = await Signin.find({ - userId: user._id + const history = await Signins.find({ + userId: user.id }); for (const signin of history) { -- cgit v1.2.3-freya