From f0a29721c9fb10f97faf386bc9d6b1b2fad97895 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 7 Apr 2019 21:50:36 +0900 Subject: Use PostgreSQL instead of MongoDB (#4572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * Update note.ts * Update timeline.ts * Update core.ts * wip * Update generate-visibility-query.ts * wip * wip * wip * wip * wip * Update global-timeline.ts * wip * wip * wip * Update vote.ts * wip * wip * Update create.ts * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * Update files.ts * wip * wip * Update CONTRIBUTING.md * wip * wip * wip * wip * wip * wip * wip * wip * Update read-notification.ts * wip * wip * wip * wip * wip * wip * wip * Update cancel.ts * wip * wip * wip * Update show.ts * wip * wip * Update gen-id.ts * Update create.ts * Update id.ts * wip * wip * wip * wip * wip * wip * wip * Docker: Update files about Docker (#4599) * Docker: Use cache if files used by `yarn install` was not updated This patch reduces the number of times to installing node_modules. For example, `yarn install` step will be skipped when only ".config/default.yml" is updated. * Docker: Migrate MongoDB to Postgresql Misskey uses Postgresql as a database instead of Mongodb since version 11. * Docker: Uncomment about data persistence This patch will save a lot of databases. * wip * wip * wip * Update activitypub.ts * wip * wip * wip * Update logs.ts * wip * Update drive-file.ts * Update register.ts * wip * wip * Update mentions.ts * wip * wip * wip * Update recommendation.ts * wip * Update index.ts * wip * Update recommendation.ts * Doc: Update docker.ja.md and docker.en.md (#1) (#4608) Update how to set up misskey. * wip * :v: * wip * Update note.ts * Update postgre.ts * wip * wip * wip * wip * Update add-file.ts * wip * wip * wip * Clean up * Update logs.ts * wip * :pizza: * wip * Ad notes * wip * Update api-visibility.ts * Update note.ts * Update add-file.ts * tests * tests * Update postgre.ts * Update utils.ts * wip * wip * Refactor * wip * Refactor * wip * wip * Update show-users.ts * Update update-instance.ts * wip * Update feed.ts * Update outbox.ts * Update outbox.ts * Update user.ts * wip * Update list.ts * Update update-hashtag.ts * wip * Update update-hashtag.ts * Refactor * Update update.ts * wip * wip * :v: * clean up * docs * Update push.ts * wip * Update api.ts * wip * :v: * Update make-pagination-query.ts * :v: * Delete hashtags.ts * Update instances.ts * Update instances.ts * Update create.ts * Update search.ts * Update reversi-game.ts * Update signup.ts * Update user.ts * id * Update example.yml * :art: * objectid * fix * reversi * reversi * Fix bug of chart engine * Add test of chart engine * Improve test * Better testing * Improve chart engine * Refactor * Add test of chart engine * Refactor * Add chart test * Fix bug * コミットし忘れ * Refactoring * :v: * Add tests * Add test * Extarct note tests * Refactor * 存在しないユーザーにメンションできなくなっていた問題を修正 * Fix bug * Update update-meta.ts * Fix bug * Update mention.vue * Fix bug * Update meta.ts * Update CONTRIBUTING.md * Fix bug * Fix bug * Fix bug * Clean up * Clean up * Update notification.ts * Clean up * Add mute tests * Add test * Refactor * Add test * Fix test * Refactor * Refactor * Add tests * Update utils.ts * Update utils.ts * Fix test * Update package.json * Update update.ts * Update manifest.ts * Fix bug * Fix bug * Add test * :art: * Update endpoint permissions * Updaye permisison * Update person.ts #4299 * データベースと同期しないように * Fix bug * Fix bug * Update reversi-game.ts * Use a feature of Node v11.7.0 to extract a public key (#4644) * wip * wip * :v: * Refactoring #1540 * test * test * test * test * test * test * test * Fix bug * Fix test * :sushi: * wip * #4471 * Add test for #4335 * Refactor * Fix test * Add tests * :clock4: * Fix bug * Add test * Add test * rename * Fix bug --- cli/migration/2.0.0.js | 57 -------------------- cli/migration/2.4.0.js | 71 ------------------------ cli/migration/5.0.0.js | 9 ---- cli/migration/7.0.0.js | 134 --------------------------------------------- cli/migration/8.0.0.js | 144 ------------------------------------------------- 5 files changed, 415 deletions(-) delete mode 100644 cli/migration/2.0.0.js delete mode 100644 cli/migration/2.4.0.js delete mode 100644 cli/migration/5.0.0.js delete mode 100644 cli/migration/7.0.0.js delete mode 100644 cli/migration/8.0.0.js (limited to 'cli') diff --git a/cli/migration/2.0.0.js b/cli/migration/2.0.0.js deleted file mode 100644 index f7298972e5..0000000000 --- a/cli/migration/2.0.0.js +++ /dev/null @@ -1,57 +0,0 @@ -// for Node.js interpret - -const chalk = require('chalk'); -const sequential = require('promise-sequential'); - -const { default: User } = require('../../built/models/user'); -const { default: DriveFile } = require('../../built/models/drive-file'); - -async function main() { - const promiseGens = []; - - const count = await DriveFile.count({}); - - let prev; - - for (let i = 0; i < count; i++) { - promiseGens.push(() => { - const promise = new Promise(async (res, rej) => { - const file = await DriveFile.findOne(prev ? { - _id: { $gt: prev._id } - } : {}, { - sort: { - _id: 1 - } - }); - - prev = file; - - const user = await User.findOne({ _id: file.metadata.userId }); - - DriveFile.update({ - _id: file._id - }, { - $set: { - 'metadata._user': { - host: user.host - } - } - }).then(() => { - res([i, file]); - }).catch(rej); - }); - - promise.then(([i, file]) => { - console.log(chalk`{gray ${i}} {green done: {bold ${file._id}} ${file.filename}}`); - }); - - return promise; - }); - } - - return await sequential(promiseGens); -} - -main().then(() => { - console.log('ALL DONE'); -}).catch(console.error); diff --git a/cli/migration/2.4.0.js b/cli/migration/2.4.0.js deleted file mode 100644 index aa37849aa1..0000000000 --- a/cli/migration/2.4.0.js +++ /dev/null @@ -1,71 +0,0 @@ -// for Node.js interpret - -const chalk = require('chalk'); -const sequential = require('promise-sequential'); - -const { default: User } = require('../../built/models/user'); -const { default: DriveFile } = require('../../built/models/drive-file'); - -async function main() { - const promiseGens = []; - - const count = await User.count({}); - - let prev; - - for (let i = 0; i < count; i++) { - promiseGens.push(() => { - const promise = new Promise(async (res, rej) => { - const user = await User.findOne(prev ? { - _id: { $gt: prev._id } - } : {}, { - sort: { - _id: 1 - } - }); - - prev = user; - - const set = {}; - - if (user.avatarId != null) { - const file = await DriveFile.findOne({ _id: user.avatarId }); - - if (file && file.metadata.properties.avgColor) { - set.avatarColor = file.metadata.properties.avgColor; - } - } - - if (user.bannerId != null) { - const file = await DriveFile.findOne({ _id: user.bannerId }); - - if (file && file.metadata.properties.avgColor) { - set.bannerColor = file.metadata.properties.avgColor; - } - } - - if (Object.keys(set).length === 0) return res([i, user]); - - User.update({ - _id: user._id - }, { - $set: set - }).then(() => { - res([i, user]); - }).catch(rej); - }); - - promise.then(([i, user]) => { - console.log(chalk`{gray ${i}} {green done: {bold ${user._id}} @${user.username}}`); - }); - - return promise; - }); - } - - return await sequential(promiseGens); -} - -main().then(() => { - console.log('ALL DONE'); -}).catch(console.error); diff --git a/cli/migration/5.0.0.js b/cli/migration/5.0.0.js deleted file mode 100644 index bef103fe4a..0000000000 --- a/cli/migration/5.0.0.js +++ /dev/null @@ -1,9 +0,0 @@ -const { default: DriveFile } = require('../../built/models/drive-file'); - -DriveFile.update({}, { - $rename: { - 'metadata.isMetaOnly': 'metadata.withoutChunks' - } -}, { - multi: true -}); diff --git a/cli/migration/7.0.0.js b/cli/migration/7.0.0.js deleted file mode 100644 index fa5e363db8..0000000000 --- a/cli/migration/7.0.0.js +++ /dev/null @@ -1,134 +0,0 @@ -const { default: Stats } = require('../../built/models/stats'); -const { default: User } = require('../../built/models/user'); -const { default: Note } = require('../../built/models/note'); -const { default: DriveFile } = require('../../built/models/drive-file'); - -const now = new Date(); -const y = now.getFullYear(); -const m = now.getMonth(); -const d = now.getDate(); -const today = new Date(y, m, d); - -async function main() { - const localUsersCount = await User.count({ - host: null - }); - - const remoteUsersCount = await User.count({ - host: { $ne: null } - }); - - const localNotesCount = await Note.count({ - '_user.host': null - }); - - const remoteNotesCount = await Note.count({ - '_user.host': { $ne: null } - }); - - const localDriveFilesCount = await DriveFile.count({ - 'metadata._user.host': null - }); - - const remoteDriveFilesCount = await DriveFile.count({ - 'metadata._user.host': { $ne: null } - }); - - const localDriveFilesSize = await DriveFile - .aggregate([{ - $match: { - 'metadata._user.host': null, - 'metadata.deletedAt': { $exists: false } - } - }, { - $project: { - length: true - } - }, { - $group: { - _id: null, - usage: { $sum: '$length' } - } - }]) - .then(aggregates => { - if (aggregates.length > 0) { - return aggregates[0].usage; - } - return 0; - }); - - const remoteDriveFilesSize = await DriveFile - .aggregate([{ - $match: { - 'metadata._user.host': { $ne: null }, - 'metadata.deletedAt': { $exists: false } - } - }, { - $project: { - length: true - } - }, { - $group: { - _id: null, - usage: { $sum: '$length' } - } - }]) - .then(aggregates => { - if (aggregates.length > 0) { - return aggregates[0].usage; - } - return 0; - }); - - await Stats.insert({ - date: today, - users: { - local: { - total: localUsersCount, - diff: 0 - }, - remote: { - total: remoteUsersCount, - diff: 0 - } - }, - notes: { - local: { - total: localNotesCount, - diff: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - }, - remote: { - total: remoteNotesCount, - diff: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - } - }, - drive: { - local: { - totalCount: localDriveFilesCount, - totalSize: localDriveFilesSize, - diffCount: 0, - diffSize: 0 - }, - remote: { - totalCount: remoteDriveFilesCount, - totalSize: remoteDriveFilesSize, - diffCount: 0, - diffSize: 0 - } - } - }); - - console.log('done'); -} - -main(); diff --git a/cli/migration/8.0.0.js b/cli/migration/8.0.0.js deleted file mode 100644 index fd6cb24525..0000000000 --- a/cli/migration/8.0.0.js +++ /dev/null @@ -1,144 +0,0 @@ -const { default: Stats } = require('../../built/models/stats'); -const { default: User } = require('../../built/models/user'); -const { default: Note } = require('../../built/models/note'); -const { default: DriveFile } = require('../../built/models/drive-file'); - -const now = new Date(); -const y = now.getFullYear(); -const m = now.getMonth(); -const d = now.getDate(); -const h = now.getHours(); -const date = new Date(y, m, d, h); - -async function main() { - await Stats.update({}, { - $set: { - span: 'day' - } - }, { - multi: true - }); - - const localUsersCount = await User.count({ - host: null - }); - - const remoteUsersCount = await User.count({ - host: { $ne: null } - }); - - const localNotesCount = await Note.count({ - '_user.host': null - }); - - const remoteNotesCount = await Note.count({ - '_user.host': { $ne: null } - }); - - const localDriveFilesCount = await DriveFile.count({ - 'metadata._user.host': null - }); - - const remoteDriveFilesCount = await DriveFile.count({ - 'metadata._user.host': { $ne: null } - }); - - const localDriveFilesSize = await DriveFile - .aggregate([{ - $match: { - 'metadata._user.host': null, - 'metadata.deletedAt': { $exists: false } - } - }, { - $project: { - length: true - } - }, { - $group: { - _id: null, - usage: { $sum: '$length' } - } - }]) - .then(aggregates => { - if (aggregates.length > 0) { - return aggregates[0].usage; - } - return 0; - }); - - const remoteDriveFilesSize = await DriveFile - .aggregate([{ - $match: { - 'metadata._user.host': { $ne: null }, - 'metadata.deletedAt': { $exists: false } - } - }, { - $project: { - length: true - } - }, { - $group: { - _id: null, - usage: { $sum: '$length' } - } - }]) - .then(aggregates => { - if (aggregates.length > 0) { - return aggregates[0].usage; - } - return 0; - }); - - await Stats.insert({ - date: date, - span: 'hour', - users: { - local: { - total: localUsersCount, - diff: 0 - }, - remote: { - total: remoteUsersCount, - diff: 0 - } - }, - notes: { - local: { - total: localNotesCount, - diff: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - }, - remote: { - total: remoteNotesCount, - diff: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - } - }, - drive: { - local: { - totalCount: localDriveFilesCount, - totalSize: localDriveFilesSize, - diffCount: 0, - diffSize: 0 - }, - remote: { - totalCount: remoteDriveFilesCount, - totalSize: remoteDriveFilesSize, - diffCount: 0, - diffSize: 0 - } - } - }); - - console.log('done'); -} - -main(); -- cgit v1.2.3-freya