From 04dd2cb12bdc5a72ad004802be68c9867efe05a6 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 7 Nov 2017 23:38:18 +0900 Subject: rename with execution order, add desc & initial creation script --- tools/migration/README.md | 3 + .../change-gridfs-metadata-name-to-filename.js | 50 --------------- tools/migration/issue_882.js | 47 -------------- tools/migration/like-to-reactions.js | 22 ------- tools/migration/node.1509958623.use-gridfs.js | 71 ++++++++++++++++++++++ ...6282.change-gridfs-metadata-name-to-filename.js | 50 +++++++++++++++ tools/migration/node.1510056272.issue_882.js | 47 ++++++++++++++ tools/migration/reply_to-to-reply.js | 5 -- tools/migration/shell.1487734995.user-profile.js | 18 ++++++ .../shell.1489951459.like-to-reactions.js | 22 +++++++ .../shell.1509507382.reply_to-to-reply.js | 5 ++ tools/migration/use-gridfs.js | 71 ---------------------- tools/migration/user-profile.js | 18 ------ 13 files changed, 216 insertions(+), 213 deletions(-) delete mode 100644 tools/migration/change-gridfs-metadata-name-to-filename.js delete mode 100644 tools/migration/issue_882.js delete mode 100644 tools/migration/like-to-reactions.js create mode 100644 tools/migration/node.1509958623.use-gridfs.js create mode 100644 tools/migration/node.1510016282.change-gridfs-metadata-name-to-filename.js create mode 100644 tools/migration/node.1510056272.issue_882.js delete mode 100644 tools/migration/reply_to-to-reply.js create mode 100644 tools/migration/shell.1487734995.user-profile.js create mode 100644 tools/migration/shell.1489951459.like-to-reactions.js create mode 100644 tools/migration/shell.1509507382.reply_to-to-reply.js delete mode 100644 tools/migration/use-gridfs.js delete mode 100644 tools/migration/user-profile.js (limited to 'tools/migration') diff --git a/tools/migration/README.md b/tools/migration/README.md index cd553efb94..d52e84b35a 100644 --- a/tools/migration/README.md +++ b/tools/migration/README.md @@ -1,8 +1,11 @@ Misskeyの破壊的変更に対応するいくつかのスニペットがあります。 MongoDBシェルで実行する必要のあるものとnodeで直接実行する必要のあるものがあります。 +ファイル名が `shell.` から始まるものは前者、 `node.` から始まるものは後者です。 MongoDBシェルで実行する場合、`use`でデータベースを選択しておく必要があります。 nodeで実行するいくつかのスニペットは、並列処理させる数を引数で設定できるものがあります。 処理中にエラーで落ちる場合は、メモリが足りていない可能性があるので、少ない数に設定してみてください。 ※デフォルトは`5`です。 + +ファイルを作成する際は `../init-migration-file.sh -t _type_ -n _name_` を実行すると _type_._unixtime_._name_.js が生成されます diff --git a/tools/migration/change-gridfs-metadata-name-to-filename.js b/tools/migration/change-gridfs-metadata-name-to-filename.js deleted file mode 100644 index 9128d852cf..0000000000 --- a/tools/migration/change-gridfs-metadata-name-to-filename.js +++ /dev/null @@ -1,50 +0,0 @@ -// for Node.js interpret -/** - * change usage of GridFS filename - * see commit fb422b4d603c53a70712caba55b35a48a8c2e619 - */ - -const { default: DriveFile } = require('../../built/api/models/drive-file') - -async function applyNewChange (doc) { - const result = await DriveFile.update(doc._id, { - $set: { - filename: doc.metadata.name - }, - $unset: { - 'metadata.name': '' - } - }) - return result.ok === 1 -} - -async function main () { - const query = { - 'metadata.name': { - $exists: true - } - } - - const count = await DriveFile.count(query) - - const dop = Number.parseInt(process.argv[2]) || 5 - const idop = ((count - (count % dop)) / dop) + 1 - - return zip( - 1, - async (time) => { - console.log(`${time} / ${idop}`) - const doc = await db.get('drive_files').find(query, { - limit: dop, skip: time * dop - }) - return Promise.all(doc.map(applyNewChange)) - }, - idop - ).then(a => { - const rv = [] - a.forEach(e => rv.push(...e)) - return rv - }) -} - -main().then(console.dir).catch(console.error) diff --git a/tools/migration/issue_882.js b/tools/migration/issue_882.js deleted file mode 100644 index aa11413255..0000000000 --- a/tools/migration/issue_882.js +++ /dev/null @@ -1,47 +0,0 @@ -// for Node.js interpret - -const { default: DriveFile } = require('../../built/api/models/drive-file') -const { default: zip } = require('@prezzemolo/zip') - -const migrate = async (doc) => { - const result = await DriveFile.update(doc._id, { - $set: { - contentType: doc.metadata.type - }, - $unset: { - 'metadata.type': '' - } - }) - return result.ok === 1 -} - -async function main() { - const query = { - 'metadata.type': { - $exists: true - } - } - - const count = await DriveFile.count(query); - - const dop = Number.parseInt(process.argv[2]) || 5 - const idop = ((count - (count % dop)) / dop) + 1 - - return zip( - 1, - async (time) => { - console.log(`${time} / ${idop}`) - const doc = await db.get('drive_files').find(query, { - limit: dop, skip: time * dop - }) - return Promise.all(doc.map(migrate)) - }, - idop - ).then(a => { - const rv = [] - a.forEach(e => rv.push(...e)) - return rv - }) -} - -main().then(console.dir).catch(console.error) diff --git a/tools/migration/like-to-reactions.js b/tools/migration/like-to-reactions.js deleted file mode 100644 index 962a0f00ef..0000000000 --- a/tools/migration/like-to-reactions.js +++ /dev/null @@ -1,22 +0,0 @@ -db.users.update({}, { - $unset: { - likes_count: 1, - liked_count: 1 - } -}, false, true) - -db.likes.renameCollection('post_reactions') - -db.post_reactions.update({}, { - $set: { - reaction: 'like' - } -}, false, true) - -db.posts.update({}, { - $rename: { - likes_count: 'reaction_counts.like' - } -}, false, true); - -db.notifications.remove({}) diff --git a/tools/migration/node.1509958623.use-gridfs.js b/tools/migration/node.1509958623.use-gridfs.js new file mode 100644 index 0000000000..a9d2b12e95 --- /dev/null +++ b/tools/migration/node.1509958623.use-gridfs.js @@ -0,0 +1,71 @@ +// for Node.js interpret + +const { default: db } = require('../../built/db/mongodb') +const { default: DriveFile, getGridFSBucket } = require('../../built/api/models/drive-file') +const { Duplex } = require('stream') +const { default: zip } = require('@prezzemolo/zip') + +const writeToGridFS = (bucket, buffer, ...rest) => new Promise((resolve, reject) => { + const writeStream = bucket.openUploadStreamWithId(...rest) + + const dataStream = new Duplex() + dataStream.push(buffer) + dataStream.push(null) + + writeStream.once('finish', resolve) + writeStream.on('error', reject) + + dataStream.pipe(writeStream) +}) + +const migrateToGridFS = async (doc) => { + const id = doc._id + const buffer = doc.data ? doc.data.buffer : Buffer.from([0x00]) // アップロードのバグなのか知らないけどなぜか data が存在しない drive_file ドキュメントがまれにあることがわかったので + const created_at = doc.created_at + const name = doc.name + const type = doc.type + + delete doc._id + delete doc.created_at + delete doc.datasize + delete doc.hash + delete doc.data + delete doc.name + delete doc.type + + const bucket = await getGridFSBucket() + const added = await writeToGridFS(bucket, buffer, id, name, { contentType: type, metadata: doc }) + + const result = await DriveFile.update(id, { + $set: { + uploadDate: created_at + } + }) + + return added && result.ok === 1 +} + +async function main() { + const count = await db.get('drive_files').count({}); + + console.log(`there are ${count} files.`) + + const dop = Number.parseInt(process.argv[2]) || 5 + const idop = ((count - (count % dop)) / dop) + 1 + + return zip( + 1, + async (time) => { + console.log(`${time} / ${idop}`) + const doc = await db.get('drive_files').find({}, { limit: dop, skip: time * dop }) + return Promise.all(doc.map(migrateToGridFS)) + }, + idop + ).then(a => { + const rv = [] + a.forEach(e => rv.push(...e)) + return rv + }) +} + +main().then(console.dir).catch(console.error) diff --git a/tools/migration/node.1510016282.change-gridfs-metadata-name-to-filename.js b/tools/migration/node.1510016282.change-gridfs-metadata-name-to-filename.js new file mode 100644 index 0000000000..9128d852cf --- /dev/null +++ b/tools/migration/node.1510016282.change-gridfs-metadata-name-to-filename.js @@ -0,0 +1,50 @@ +// for Node.js interpret +/** + * change usage of GridFS filename + * see commit fb422b4d603c53a70712caba55b35a48a8c2e619 + */ + +const { default: DriveFile } = require('../../built/api/models/drive-file') + +async function applyNewChange (doc) { + const result = await DriveFile.update(doc._id, { + $set: { + filename: doc.metadata.name + }, + $unset: { + 'metadata.name': '' + } + }) + return result.ok === 1 +} + +async function main () { + const query = { + 'metadata.name': { + $exists: true + } + } + + const count = await DriveFile.count(query) + + const dop = Number.parseInt(process.argv[2]) || 5 + const idop = ((count - (count % dop)) / dop) + 1 + + return zip( + 1, + async (time) => { + console.log(`${time} / ${idop}`) + const doc = await db.get('drive_files').find(query, { + limit: dop, skip: time * dop + }) + return Promise.all(doc.map(applyNewChange)) + }, + idop + ).then(a => { + const rv = [] + a.forEach(e => rv.push(...e)) + return rv + }) +} + +main().then(console.dir).catch(console.error) diff --git a/tools/migration/node.1510056272.issue_882.js b/tools/migration/node.1510056272.issue_882.js new file mode 100644 index 0000000000..aa11413255 --- /dev/null +++ b/tools/migration/node.1510056272.issue_882.js @@ -0,0 +1,47 @@ +// for Node.js interpret + +const { default: DriveFile } = require('../../built/api/models/drive-file') +const { default: zip } = require('@prezzemolo/zip') + +const migrate = async (doc) => { + const result = await DriveFile.update(doc._id, { + $set: { + contentType: doc.metadata.type + }, + $unset: { + 'metadata.type': '' + } + }) + return result.ok === 1 +} + +async function main() { + const query = { + 'metadata.type': { + $exists: true + } + } + + const count = await DriveFile.count(query); + + const dop = Number.parseInt(process.argv[2]) || 5 + const idop = ((count - (count % dop)) / dop) + 1 + + return zip( + 1, + async (time) => { + console.log(`${time} / ${idop}`) + const doc = await db.get('drive_files').find(query, { + limit: dop, skip: time * dop + }) + return Promise.all(doc.map(migrate)) + }, + idop + ).then(a => { + const rv = [] + a.forEach(e => rv.push(...e)) + return rv + }) +} + +main().then(console.dir).catch(console.error) diff --git a/tools/migration/reply_to-to-reply.js b/tools/migration/reply_to-to-reply.js deleted file mode 100644 index ceb272ebc9..0000000000 --- a/tools/migration/reply_to-to-reply.js +++ /dev/null @@ -1,5 +0,0 @@ -db.posts.update({}, { - $rename: { - reply_to_id: 'reply_id' - } -}, false, true); diff --git a/tools/migration/shell.1487734995.user-profile.js b/tools/migration/shell.1487734995.user-profile.js new file mode 100644 index 0000000000..e6666319e1 --- /dev/null +++ b/tools/migration/shell.1487734995.user-profile.js @@ -0,0 +1,18 @@ +db.users.find({}).forEach(function(user) { + print(user._id); + db.users.update({ _id: user._id }, { + $rename: { + bio: 'description' + }, + $unset: { + location: '', + birthday: '' + }, + $set: { + profile: { + location: user.location || null, + birthday: user.birthday || null + } + } + }, false, false); +}); diff --git a/tools/migration/shell.1489951459.like-to-reactions.js b/tools/migration/shell.1489951459.like-to-reactions.js new file mode 100644 index 0000000000..962a0f00ef --- /dev/null +++ b/tools/migration/shell.1489951459.like-to-reactions.js @@ -0,0 +1,22 @@ +db.users.update({}, { + $unset: { + likes_count: 1, + liked_count: 1 + } +}, false, true) + +db.likes.renameCollection('post_reactions') + +db.post_reactions.update({}, { + $set: { + reaction: 'like' + } +}, false, true) + +db.posts.update({}, { + $rename: { + likes_count: 'reaction_counts.like' + } +}, false, true); + +db.notifications.remove({}) diff --git a/tools/migration/shell.1509507382.reply_to-to-reply.js b/tools/migration/shell.1509507382.reply_to-to-reply.js new file mode 100644 index 0000000000..ceb272ebc9 --- /dev/null +++ b/tools/migration/shell.1509507382.reply_to-to-reply.js @@ -0,0 +1,5 @@ +db.posts.update({}, { + $rename: { + reply_to_id: 'reply_id' + } +}, false, true); diff --git a/tools/migration/use-gridfs.js b/tools/migration/use-gridfs.js deleted file mode 100644 index a9d2b12e95..0000000000 --- a/tools/migration/use-gridfs.js +++ /dev/null @@ -1,71 +0,0 @@ -// for Node.js interpret - -const { default: db } = require('../../built/db/mongodb') -const { default: DriveFile, getGridFSBucket } = require('../../built/api/models/drive-file') -const { Duplex } = require('stream') -const { default: zip } = require('@prezzemolo/zip') - -const writeToGridFS = (bucket, buffer, ...rest) => new Promise((resolve, reject) => { - const writeStream = bucket.openUploadStreamWithId(...rest) - - const dataStream = new Duplex() - dataStream.push(buffer) - dataStream.push(null) - - writeStream.once('finish', resolve) - writeStream.on('error', reject) - - dataStream.pipe(writeStream) -}) - -const migrateToGridFS = async (doc) => { - const id = doc._id - const buffer = doc.data ? doc.data.buffer : Buffer.from([0x00]) // アップロードのバグなのか知らないけどなぜか data が存在しない drive_file ドキュメントがまれにあることがわかったので - const created_at = doc.created_at - const name = doc.name - const type = doc.type - - delete doc._id - delete doc.created_at - delete doc.datasize - delete doc.hash - delete doc.data - delete doc.name - delete doc.type - - const bucket = await getGridFSBucket() - const added = await writeToGridFS(bucket, buffer, id, name, { contentType: type, metadata: doc }) - - const result = await DriveFile.update(id, { - $set: { - uploadDate: created_at - } - }) - - return added && result.ok === 1 -} - -async function main() { - const count = await db.get('drive_files').count({}); - - console.log(`there are ${count} files.`) - - const dop = Number.parseInt(process.argv[2]) || 5 - const idop = ((count - (count % dop)) / dop) + 1 - - return zip( - 1, - async (time) => { - console.log(`${time} / ${idop}`) - const doc = await db.get('drive_files').find({}, { limit: dop, skip: time * dop }) - return Promise.all(doc.map(migrateToGridFS)) - }, - idop - ).then(a => { - const rv = [] - a.forEach(e => rv.push(...e)) - return rv - }) -} - -main().then(console.dir).catch(console.error) diff --git a/tools/migration/user-profile.js b/tools/migration/user-profile.js deleted file mode 100644 index e6666319e1..0000000000 --- a/tools/migration/user-profile.js +++ /dev/null @@ -1,18 +0,0 @@ -db.users.find({}).forEach(function(user) { - print(user._id); - db.users.update({ _id: user._id }, { - $rename: { - bio: 'description' - }, - $unset: { - location: '', - birthday: '' - }, - $set: { - profile: { - location: user.location || null, - birthday: user.birthday || null - } - } - }, false, false); -}); -- cgit v1.2.3-freya