From 64aedcaa6b3e9170e77c428ee306830464b42bcf Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 03:46:30 +0900 Subject: add-file-to-drive - Promise百烈拳とメモリ削減 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/endpoints/drive/files/upload_from_url.ts | 74 +++++++++++++++++++----- 1 file changed, 61 insertions(+), 13 deletions(-) (limited to 'src/api/endpoints') diff --git a/src/api/endpoints/drive/files/upload_from_url.ts b/src/api/endpoints/drive/files/upload_from_url.ts index 46cfffb69c..9c759994e0 100644 --- a/src/api/endpoints/drive/files/upload_from_url.ts +++ b/src/api/endpoints/drive/files/upload_from_url.ts @@ -2,11 +2,17 @@ * Module dependencies */ import * as URL from 'url'; -const download = require('download'); import $ from 'cafy'; import { validateFileName } from '../../../models/drive-file'; import serialize from '../../../serializers/drive-file'; import create from '../../../common/add-file-to-drive'; +import * as debug from 'debug'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +import * as request from 'request'; +import * as crypto from 'crypto'; + +const log = debug('misskey:endpoint:upload_from_url') /** * Create a file from a URL @@ -15,7 +21,7 @@ import create from '../../../common/add-file-to-drive'; * @param {any} user * @return {Promise} */ -module.exports = (params, user) => new Promise(async (res, rej) => { +module.exports = (params, user) => new Promise((res, rej) => { // Get 'url' parameter // TODO: Validate this url const [url, urlErr] = $(params.url).string().$; @@ -30,15 +36,57 @@ module.exports = (params, user) => new Promise(async (res, rej) => { const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$; if (folderIdErr) return rej('invalid folder_id param'); - // Download file - const data = await download(url); - - // Create file - const driveFile = await create(user, data, name, null, folderId); - - // Serialize - const fileObj = await serialize(driveFile); - - // Response - res(fileObj); + // Create temp file + new Promise((res, rej) => { + tmp.file((e, path) => { + if (e) return rej(e) + res(path) + }) + }) + // Download file + .then((path: string) => new Promise((res, rej) => { + const writable = fs.createWriteStream(path) + request(url) + .on('error', rej) + .on('end', () => { + writable.close() + res(path) + }) + .pipe(writable) + .on('error', rej) + })) + // Calculate hash & content-type + .then((path: string) => new Promise((res, rej) => { + const readable = fs.createReadStream(path) + const hash = crypto.createHash('md5') + readable + .on('error', rej) + .on('end', () => { + hash.end() + res([path, hash.digest('hex')]) + }) + .pipe(hash) + .on('error', rej) + })) + // Create file + .then((rv: string[]) => new Promise((res, rej) => { + const [path, hash] = rv + create(user, { + stream: fs.createReadStream(path), + name, + hash + }, null, folderId) + .then(driveFile => { + res(driveFile) + // crean-up + fs.unlink(path, (e) => { + if (e) log(e.stack) + }) + }) + .catch(rej) + })) + // Serialize + .then(serialize) + .then(res) + .catch(rej) }); -- cgit v1.2.3-freya From e56f716a89227c76dfc05e48b3ca438f766f85b4 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 03:47:42 +0900 Subject: format --- src/api/common/add-file-to-drive.ts | 92 ++++++++++++------------ src/api/endpoints/drive/files/upload_from_url.ts | 38 +++++----- 2 files changed, 65 insertions(+), 65 deletions(-) (limited to 'src/api/endpoints') diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index a7c7cb4644..c6a4c4791d 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -18,10 +18,10 @@ const log = debug('misskey:register-drive-file'); const tmpFile = (): Promise => new Promise((resolve, reject) => { tmp.file((e, path) => { - if (e) return reject(e) - resolve(path) - }) -}) + if (e) return reject(e); + resolve(path); + }); +}); const addToGridFS = (name: string, readable: stream.Readable, type: string, metadata: any): Promise => getGridFSBucket() @@ -30,7 +30,7 @@ const addToGridFS = (name: string, readable: stream.Readable, type: string, meta writeStream.once('finish', (doc) => { resolve(doc); }); writeStream.on('error', reject); readable.pipe(writeStream); - })) + })); /** * Add file to drive @@ -56,76 +56,76 @@ export default ( // Get file path new Promise((res: (v: string) => void, rej) => { if (typeof file === 'string') { - res(file) - return + res(file); + return; } if (file instanceof Buffer) { tmpFile() .then(path => { fs.writeFile(path, file, (err) => { - if (err) rej(err) - res(path) - }) + if (err) rej(err); + res(path); + }); }) - .catch(rej) - return + .catch(rej); + return; } if (typeof file === 'object' && typeof file.read === 'function') { tmpFile() .then(path => { - const readable: stream.Readable = file - const writable = fs.createWriteStream(path) + const readable: stream.Readable = file; + const writable = fs.createWriteStream(path); readable .on('error', rej) .on('end', () => { - res(path) + res(path); }) .pipe(writable) - .on('error', rej) + .on('error', rej); }) - .catch(rej) + .catch(rej); } - rej(new Error('un-compatible file.')) + rej(new Error('un-compatible file.')); }) // Calculate hash, get content type and get file size .then(path => Promise.all([ path, // hash ((): Promise => new Promise((res, rej) => { - const readable = fs.createReadStream(path) - const hash = crypto.createHash('md5') + const readable = fs.createReadStream(path); + const hash = crypto.createHash('md5'); readable .on('error', rej) .on('end', () => { - res(hash.digest('hex')) + res(hash.digest('hex')); }) .pipe(hash) - .on('error', rej) + .on('error', rej); }))(), // mime ((): Promise<[string, string | null]> => new Promise((res, rej) => { - const readable = fs.createReadStream(path) + const readable = fs.createReadStream(path); readable .on('error', rej) .once('data', (buffer: Buffer) => { - readable.destroy() - const type = fileType(buffer) + readable.destroy(); + const type = fileType(buffer); if (!type) { - return res(['application/octet-stream', null]) + return res(['application/octet-stream', null]); } - return res([type.mime, type.ext]) - }) + return res([type.mime, type.ext]); + }); }))(), // size ((): Promise => new Promise((res, rej) => { fs.stat(path, (err, stats) => { - if (err) return rej(err) - res(stats.size) - }) + if (err) return rej(err); + res(stats.size); + }); }))() ])) .then(async ([path, hash, [mime, ext], size]) => { - log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`) + log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`); // detect name const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled'); @@ -149,7 +149,7 @@ export default ( // properties (async () => { if (!/^image\/.*$/.test(mime)) { - return null + return null; } // If the file is an image, calculate width and height to save in property const g = gm(data, name); @@ -159,21 +159,21 @@ export default ( height: size.height }; log('image width and height is calculated'); - return properties + return properties; })(), // folder (async () => { if (!folderId) { - return null + return null; } const driveFolder = await DriveFolder.findOne({ _id: folderId, user_id: user._id - }) + }); if (!driveFolder) { - throw 'folder-not-found' + throw 'folder-not-found'; } - return driveFolder + return driveFolder; })(), // usage checker (async () => { @@ -195,9 +195,9 @@ export default ( ]) .then((aggregates: any[]) => { if (aggregates.length > 0) { - return aggregates[0].usage + return aggregates[0].usage; } - return 0 + return 0; }); log(`drive usage is ${usage}`); @@ -207,21 +207,21 @@ export default ( throw 'no-free-space'; } })() - ]) + ]); - const readable = fs.createReadStream(path) + const readable = fs.createReadStream(path); return addToGridFS(name, readable, mime, { user_id: user._id, folder_id: folder !== null ? folder._id : null, comment: comment, properties: properties - }) + }); }) .then(file => { log(`drive file has been created ${file._id}`); - resolve(file) - return serialize(file) + resolve(file); + return serialize(file); }) .then(serializedFile => { // Publish drive_file_created event @@ -241,5 +241,5 @@ export default ( }); } }) - .catch(reject) + .catch(reject); }); diff --git a/src/api/endpoints/drive/files/upload_from_url.ts b/src/api/endpoints/drive/files/upload_from_url.ts index 9c759994e0..60332b4afe 100644 --- a/src/api/endpoints/drive/files/upload_from_url.ts +++ b/src/api/endpoints/drive/files/upload_from_url.ts @@ -12,7 +12,7 @@ import * as fs from 'fs'; import * as request from 'request'; import * as crypto from 'crypto'; -const log = debug('misskey:endpoint:upload_from_url') +const log = debug('misskey:endpoint:upload_from_url'); /** * Create a file from a URL @@ -39,54 +39,54 @@ module.exports = (params, user) => new Promise((res, rej) => { // Create temp file new Promise((res, rej) => { tmp.file((e, path) => { - if (e) return rej(e) - res(path) - }) + if (e) return rej(e); + res(path); + }); }) // Download file .then((path: string) => new Promise((res, rej) => { - const writable = fs.createWriteStream(path) + const writable = fs.createWriteStream(path); request(url) .on('error', rej) .on('end', () => { - writable.close() - res(path) + writable.close(); + res(path); }) .pipe(writable) - .on('error', rej) + .on('error', rej); })) // Calculate hash & content-type .then((path: string) => new Promise((res, rej) => { - const readable = fs.createReadStream(path) - const hash = crypto.createHash('md5') + const readable = fs.createReadStream(path); + const hash = crypto.createHash('md5'); readable .on('error', rej) .on('end', () => { - hash.end() - res([path, hash.digest('hex')]) + hash.end(); + res([path, hash.digest('hex')]); }) .pipe(hash) - .on('error', rej) + .on('error', rej); })) // Create file .then((rv: string[]) => new Promise((res, rej) => { - const [path, hash] = rv + const [path, hash] = rv; create(user, { stream: fs.createReadStream(path), name, hash }, null, folderId) .then(driveFile => { - res(driveFile) + res(driveFile); // crean-up fs.unlink(path, (e) => { - if (e) log(e.stack) - }) + if (e) log(e.stack); + }); }) - .catch(rej) + .catch(rej); })) // Serialize .then(serialize) .then(res) - .catch(rej) + .catch(rej); }); -- cgit v1.2.3-freya From 47f98fbab76e8680f5a7c99037b3b237c7256ca2 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 04:28:51 +0900 Subject: バグ修正 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/common/add-file-to-drive.ts | 35 +++++----- src/api/endpoints/drive/files/upload_from_url.ts | 81 ++++++++---------------- 2 files changed, 46 insertions(+), 70 deletions(-) (limited to 'src/api/endpoints') diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index 5f7d255e4b..1c8965e31d 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -214,7 +214,7 @@ export default ( const readable = fs.createReadStream(path); - return addToGridFS(name, readable, mime, { + return addToGridFS(detectedName, readable, mime, { user_id: user._id, folder_id: folder !== null ? folder._id : null, comment: comment, @@ -224,25 +224,26 @@ export default ( .then(file => { log(`drive file has been created ${file._id}`); resolve(file); - return serialize(file); - }) - .then(serializedFile => { - // Publish drive_file_created event - event(user._id, 'drive_file_created', fileObj); - // Register to search database - if (config.elasticsearch.enable) { - const es = require('../../db/elasticsearch'); - es.index({ - index: 'misskey', - type: 'drive_file', - id: file._id.toString(), - body: { - name: file.name, - user_id: user._id.toString() + serialize(file) + .then(serializedFile => { + // Publish drive_file_created event + event(user._id, 'drive_file_created', serializedFile); + + // Register to search database + if (config.elasticsearch.enable) { + const es = require('../../db/elasticsearch'); + es.index({ + index: 'misskey', + type: 'drive_file', + id: file._id.toString(), + body: { + name: file.name, + user_id: user._id.toString() + } + }); } }); - } }) .catch(reject); }); diff --git a/src/api/endpoints/drive/files/upload_from_url.ts b/src/api/endpoints/drive/files/upload_from_url.ts index 60332b4afe..519e0bdf65 100644 --- a/src/api/endpoints/drive/files/upload_from_url.ts +++ b/src/api/endpoints/drive/files/upload_from_url.ts @@ -10,7 +10,6 @@ import * as debug from 'debug'; import * as tmp from 'tmp'; import * as fs from 'fs'; import * as request from 'request'; -import * as crypto from 'crypto'; const log = debug('misskey:endpoint:upload_from_url'); @@ -21,11 +20,11 @@ const log = debug('misskey:endpoint:upload_from_url'); * @param {any} user * @return {Promise} */ -module.exports = (params, user) => new Promise((res, rej) => { +module.exports = async (params, user): Promise => { // Get 'url' parameter // TODO: Validate this url const [url, urlErr] = $(params.url).string().$; - if (urlErr) return rej('invalid url param'); + if (urlErr) throw 'invalid url param'; let name = URL.parse(url).pathname.split('/').pop(); if (!validateFileName(name)) { @@ -34,59 +33,35 @@ module.exports = (params, user) => new Promise((res, rej) => { // Get 'folder_id' parameter const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$; - if (folderIdErr) return rej('invalid folder_id param'); + if (folderIdErr) throw 'invalid folder_id param'; // Create temp file - new Promise((res, rej) => { + const path = await new Promise((res: (string) => void, rej) => { tmp.file((e, path) => { if (e) return rej(e); res(path); }); - }) - // Download file - .then((path: string) => new Promise((res, rej) => { - const writable = fs.createWriteStream(path); - request(url) - .on('error', rej) - .on('end', () => { - writable.close(); - res(path); - }) - .pipe(writable) - .on('error', rej); - })) - // Calculate hash & content-type - .then((path: string) => new Promise((res, rej) => { - const readable = fs.createReadStream(path); - const hash = crypto.createHash('md5'); - readable - .on('error', rej) - .on('end', () => { - hash.end(); - res([path, hash.digest('hex')]); - }) - .pipe(hash) - .on('error', rej); - })) - // Create file - .then((rv: string[]) => new Promise((res, rej) => { - const [path, hash] = rv; - create(user, { - stream: fs.createReadStream(path), - name, - hash - }, null, folderId) - .then(driveFile => { - res(driveFile); - // crean-up - fs.unlink(path, (e) => { - if (e) log(e.stack); - }); - }) - .catch(rej); - })) - // Serialize - .then(serialize) - .then(res) - .catch(rej); -}); + }); + + // write content at URL to temp file + await new Promise((res, rej) => { + const writable = fs.createWriteStream(path); + request(url) + .on('error', rej) + .on('end', () => { + writable.close(); + res(path); + }) + .pipe(writable) + .on('error', rej); + }); + + const driveFile = await create(user, path, name, null, folderId); + + // clean-up + fs.unlink(path, (e) => { + if (e) log(e.stack); + }); + + return serialize(driveFile); +}; -- cgit v1.2.3-freya From 342345cc2fb95e86d65095e0c9996441950ad628 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 04:35:25 +0900 Subject: create - バッファを使用しないように MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/endpoints/drive/files/create.ts | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) (limited to 'src/api/endpoints') diff --git a/src/api/endpoints/drive/files/create.ts b/src/api/endpoints/drive/files/create.ts index 7967c31187..7546eca309 100644 --- a/src/api/endpoints/drive/files/create.ts +++ b/src/api/endpoints/drive/files/create.ts @@ -1,7 +1,6 @@ /** * Module dependencies */ -import * as fs from 'fs'; import $ from 'cafy'; import { validateFileName } from '../../../models/drive-file'; import serialize from '../../../serializers/drive-file'; @@ -15,15 +14,11 @@ import create from '../../../common/add-file-to-drive'; * @param {any} user * @return {Promise} */ -module.exports = (file, params, user) => new Promise(async (res, rej) => { +module.exports = async (file, params, user): Promise => { if (file == null) { - return rej('file is required'); + throw 'file is required'; } - // TODO: 非同期にしたい。Promise対応してないんだろうか... - const buffer = fs.readFileSync(file.path); - fs.unlink(file.path, (err) => { if (err) console.log(err); }); - // Get 'name' parameter let name = file.originalname; if (name !== undefined && name !== null) { @@ -33,7 +28,7 @@ module.exports = (file, params, user) => new Promise(async (res, rej) => { } else if (name === 'blob') { name = null; } else if (!validateFileName(name)) { - return rej('invalid name'); + throw 'invalid name'; } } else { name = null; @@ -41,14 +36,11 @@ module.exports = (file, params, user) => new Promise(async (res, rej) => { // Get 'folder_id' parameter const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$; - if (folderIdErr) return rej('invalid folder_id param'); + if (folderIdErr) throw 'invalid folder_id param'; // Create file - const driveFile = await create(user, buffer, name, null, folderId); + const driveFile = await create(user, file.path, name, null, folderId); // Serialize - const fileObj = await serialize(driveFile); - - // Response - res(fileObj); -}); + return serialize(driveFile); +}; -- cgit v1.2.3-freya