add-file-to-drive - Promise百烈拳とメモリ削減

2 files changed, 258 insertions, 142 deletions

diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts
index a96906d291..a7c7cb4644 100644
--- a/src/api/common/add-file-to-drive.ts
+++ b/src/api/common/add-file-to-drive.ts
@@ -9,28 +9,34 @@ import DriveFolder from '../models/drive-folder';
 import serialize from '../serializers/drive-file';
 import event from '../event';
 import config from '../../conf';
-import { Duplex } from 'stream';
+import { Buffer } from 'buffer';
+import * as fs from 'fs';
+import * as tmp from 'tmp';
+import * as stream from 'stream';
 
 const log = debug('misskey:register-drive-file');
 
-const addToGridFS = (name, binary, type, metadata): Promise<any> => new Promise(async (resolve, reject) => {
-	const dataStream = new Duplex();
-	dataStream.push(binary);
-	dataStream.push(null);
+const tmpFile = (): Promise<string> => new Promise((resolve, reject) => {
+	tmp.file((e, path) => {
+		if (e) return reject(e)
+		resolve(path)
+	})
+})
 
-	const bucket = await getGridFSBucket();
-	const writeStream = bucket.openUploadStream(name, { contentType: type, metadata });
-	writeStream.once('finish', (doc) => { resolve(doc); });
-	writeStream.on('error', reject);
-	dataStream.pipe(writeStream);
-});
+const addToGridFS = (name: string, readable: stream.Readable, type: string, metadata: any): Promise<any> =>
+	getGridFSBucket()
+		.then(bucket => new Promise((resolve, reject) => {
+			const writeStream = bucket.openUploadStream(name, { contentType: type, metadata });
+			writeStream.once('finish', (doc) => { resolve(doc); });
+			writeStream.on('error', reject);
+			readable.pipe(writeStream);
+		}))
 
 /**
  * Add file to drive
  *
  * @param user User who wish to add file
- * @param fileName File name
- * @param data Contents
+ * @param file File path, binary, or readableStream
  * @param comment Comment
  * @param type File type
  * @param folderId Folder ID
@@ -39,139 +45,201 @@ const addToGridFS = (name, binary, type, metadata): Promise<any> => new Promise(
  */
 export default (
 	user: any,
-	data: Buffer,
+	file: string | Buffer | stream.Readable,
 	name: string = null,
 	comment: string = null,
 	folderId: mongodb.ObjectID = null,
 	force: boolean = false
-) => new Promise<any>(async (resolve, reject) => {
+) => new Promise<any>((resolve, reject) => {
 	log(`registering ${name} (user: ${user.username})`);
 
-	// File size
-	const size = data.byteLength;
-
-	log(`size is ${size}`);
-
-	// File type
-	let mime = 'application/octet-stream';
-	const type = fileType(data);
-	if (type !== null) {
-		mime = type.mime;
-
-		if (name === null) {
-			name = `untitled.${type.ext}`;
+	// Get file path
+	new Promise((res: (v: string) => void, rej) => {
+		if (typeof file === 'string') {
+			res(file)
+			return
 		}
-	} else {
-		if (name === null) {
-			name = 'untitled';
-		}
-	}
-
-	log(`type is ${mime}`);
-
-	// Generate hash
-	const hash = crypto
-		.createHash('md5')
-		.update(data)
-		.digest('hex') as string;
-
-	log(`hash is ${hash}`);
-
-	if (!force) {
-		// Check if there is a file with the same hash
-		const much = await DriveFile.findOne({
-			md5: hash,
-			'metadata.user_id': user._id
-		});
-
-		if (much !== null) {
-			log('file with same hash is found');
-			return resolve(much);
-		} else {
-			log('file with same hash is not found');
+		if (file instanceof Buffer) {
+			tmpFile()
+				.then(path => {
+					fs.writeFile(path, file, (err) => {
+						if (err) rej(err)
+						res(path)
+					})
+				})
+				.catch(rej)
+			return
 		}
-	}
-
-	// Calculate drive usage
-	const usage = ((await DriveFile
-		.aggregate([
-			{ $match: { 'metadata.user_id': user._id } },
-			{ $project: {
-				length: true
-			}},
-			{ $group: {
-				_id: null,
-				usage: { $sum: '$length' }
-			}}
-		]))[0] || {
-			usage: 0
-		}).usage;
-
-	log(`drive usage is ${usage}`);
-
-	// If usage limit exceeded
-	if (usage + size > user.drive_capacity) {
-		return reject('no-free-space');
-	}
-
-	// If the folder is specified
-	let folder: any = null;
-	if (folderId !== null) {
-		folder = await DriveFolder
-			.findOne({
-				_id: folderId,
-				user_id: user._id
-			});
-
-		if (folder === null) {
-			return reject('folder-not-found');
+		if (typeof file === 'object' && typeof file.read === 'function') {
+			tmpFile()
+				.then(path => {
+					const readable: stream.Readable = file
+					const writable = fs.createWriteStream(path)
+					readable
+						.on('error', rej)
+						.on('end', () => {
+							res(path)
+						})
+						.pipe(writable)
+						.on('error', rej)
+				})
+				.catch(rej)
 		}
-	}
+		rej(new Error('un-compatible file.'))
+	})
+		// Calculate hash, get content type and get file size
+		.then(path => Promise.all([
+			path,
+			// hash
+			((): Promise<string> => new Promise((res, rej) => {
+				const readable = fs.createReadStream(path)
+				const hash = crypto.createHash('md5')
+				readable
+					.on('error', rej)
+					.on('end', () => {
+						res(hash.digest('hex'))
+					})
+					.pipe(hash)
+					.on('error', rej)
+			}))(),
+			// mime
+			((): Promise<[string, string | null]> => new Promise((res, rej) => {
+				const readable = fs.createReadStream(path)
+				readable
+					.on('error', rej)
+					.once('data', (buffer: Buffer) => {
+						readable.destroy()
+						const type = fileType(buffer)
+						if (!type) {
+							return res(['application/octet-stream', null])
+						}
+						return res([type.mime, type.ext])
+					})
+			}))(),
+			// size
+			((): Promise<number> => new Promise((res, rej) => {
+				fs.stat(path, (err, stats) => {
+					if (err) return rej(err)
+					res(stats.size)
+				})
+			}))()
+		]))
+		.then(async ([path, hash, [mime, ext], size]) => {
+			log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`)
 
-	let properties: any = null;
+			// detect name
+			const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled');
 
-	// If the file is an image
-	if (/^image\/.*$/.test(mime)) {
-		// Calculate width and height to save in property
-		const g = gm(data, name);
-		const size = await prominence(g).size();
-		properties = {
-			width: size.width,
-			height: size.height
-		};
+			if (!force) {
+				// Check if there is a file with the same hash
+				const much = await DriveFile.findOne({
+					md5: hash,
+					'metadata.user_id': user._id
+				});
 
-		log('image width and height is calculated');
-	}
+				if (much !== null) {
+					log('file with same hash is found');
+					return resolve(much);
+				} else {
+					log('file with same hash is not found');
+				}
+			}
 
-	// Create DriveFile document
-	const file = await addToGridFS(name, data, mime, {
-		user_id: user._id,
-		folder_id: folder !== null ? folder._id : null,
-		comment: comment,
-		properties: properties
-	});
+			const [properties, folder] = await Promise.all([
+				// properties
+				(async () => {
+					if (!/^image\/.*$/.test(mime)) {
+						return null
+					}
+					// If the file is an image, calculate width and height to save in property
+					const g = gm(data, name);
+					const size = await prominence(g).size();
+					const properties = {
+						width: size.width,
+						height: size.height
+					};
+					log('image width and height is calculated');
+					return properties
+				})(),
+				// folder
+				(async () => {
+					if (!folderId) {
+						return null
+					}
+					const driveFolder = await DriveFolder.findOne({
+						_id: folderId,
+						user_id: user._id
+					})
+					if (!driveFolder) {
+						throw 'folder-not-found'
+					}
+					return driveFolder
+				})(),
+				// usage checker
+				(async () => {
+					// Calculate drive usage
+					const usage = await DriveFile
+						.aggregate([
+							{ $match: { 'metadata.user_id': user._id } },
+							{
+								$project: {
+									length: true
+								}
+							},
+							{
+								$group: {
+									_id: null,
+									usage: { $sum: '$length' }
+								}
+							}
+						])
+						.then((aggregates: any[]) => {
+							if (aggregates.length > 0) {
+								return aggregates[0].usage
+							}
+							return 0
+						});
 
-	log(`drive file has been created ${file._id}`);
+					log(`drive usage is ${usage}`);
 
-	resolve(file);
+					// If usage limit exceeded
+					if (usage + size > user.drive_capacity) {
+						throw 'no-free-space';
+					}
+				})()
+			])
 
-	// Serialize
-	const fileObj = await serialize(file);
+			const readable = fs.createReadStream(path)
 
-	// Publish drive_file_created event
-	event(user._id, 'drive_file_created', fileObj);
+			return addToGridFS(name, readable, mime, {
+				user_id: user._id,
+				folder_id: folder !== null ? folder._id : null,
+				comment: comment,
+				properties: properties
+			})
+		})
+		.then(file => {
+			log(`drive file has been created ${file._id}`);
+			resolve(file)
+			return serialize(file)
+		})
+		.then(serializedFile => {
+			// Publish drive_file_created event
+			event(user._id, 'drive_file_created', fileObj);
 
-	// Register to search database
-	if (config.elasticsearch.enable) {
-		const es = require('../../db/elasticsearch');
-		es.index({
-			index: 'misskey',
-			type: 'drive_file',
-			id: file._id.toString(),
-			body: {
-				name: file.name,
-				user_id: user._id.toString()
+			// Register to search database
+			if (config.elasticsearch.enable) {
+				const es = require('../../db/elasticsearch');
+				es.index({
+					index: 'misskey',
+					type: 'drive_file',
+					id: file._id.toString(),
+					body: {
+						name: file.name,
+						user_id: user._id.toString()
+					}
+				});
 			}
-		});
-	}
+		})
+		.catch(reject)
 });
diff --git a/src/api/endpoints/drive/files/upload_from_url.ts b/src/api/endpoints/drive/files/upload_from_url.ts
index 46cfffb69c..9c759994e0 100644
--- a/src/api/endpoints/drive/files/upload_from_url.ts
+++ b/src/api/endpoints/drive/files/upload_from_url.ts
@@ -2,11 +2,17 @@
  * Module dependencies
  */
 import * as URL from 'url';
-const download = require('download');
 import $ from 'cafy';
 import { validateFileName } from '../../../models/drive-file';
 import serialize from '../../../serializers/drive-file';
 import create from '../../../common/add-file-to-drive';
+import * as debug from 'debug';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
+import * as request from 'request';
+import * as crypto from 'crypto';
+
+const log = debug('misskey:endpoint:upload_from_url')
 
 /**
  * Create a file from a URL
@@ -15,7 +21,7 @@ import create from '../../../common/add-file-to-drive';
  * @param {any} user
  * @return {Promise<any>}
  */
-module.exports = (params, user) => new Promise(async (res, rej) => {
+module.exports = (params, user) => new Promise((res, rej) => {
 	// Get 'url' parameter
 	// TODO: Validate this url
 	const [url, urlErr] = $(params.url).string().$;
@@ -30,15 +36,57 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
 	const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
 	if (folderIdErr) return rej('invalid folder_id param');
 
-	// Download file
-	const data = await download(url);
-
-	// Create file
-	const driveFile = await create(user, data, name, null, folderId);
-
-	// Serialize
-	const fileObj = await serialize(driveFile);
-
-	// Response
-	res(fileObj);
+	// Create temp file
+	new Promise((res, rej) => {
+		tmp.file((e, path) => {
+			if (e) return rej(e)
+			res(path)
+		})
+	})
+		// Download file
+		.then((path: string) => new Promise((res, rej) => {
+			const writable = fs.createWriteStream(path)
+			request(url)
+				.on('error', rej)
+				.on('end', () => {
+					writable.close()
+					res(path)
+				})
+				.pipe(writable)
+				.on('error', rej)
+		}))
+		// Calculate hash & content-type
+		.then((path: string) => new Promise((res, rej) => {
+			const readable = fs.createReadStream(path)
+			const hash = crypto.createHash('md5')
+			readable
+				.on('error', rej)
+				.on('end', () => {
+					hash.end()
+					res([path, hash.digest('hex')])
+				})
+				.pipe(hash)
+				.on('error', rej)
+		}))
+		// Create file
+		.then((rv: string[]) => new Promise((res, rej) => {
+			const [path, hash] = rv
+			create(user, {
+				stream: fs.createReadStream(path),
+				name,
+				hash
+			}, null, folderId)
+				.then(driveFile => {
+					res(driveFile)
+					// crean-up
+					fs.unlink(path, (e) => {
+						if (e) log(e.stack)
+					})
+				})
+				.catch(rej)
+		}))
+		// Serialize
+		.then(serialize)
+		.then(res)
+		.catch(rej)
 });