From 68ce6d574882c1badbb4a3d2772451534014dd01 Mon Sep 17 00:00:00 2001
From: Akihiko Odaki <nekomanma@pixiv.co.jp>
Date: Tue, 27 Mar 2018 16:51:12 +0900
Subject: Implement remote account resolution

---
 src/api/bot/core.ts                              |  14 +-
 src/api/bot/interfaces/line.ts                   |  12 +-
 src/api/common/add-file-to-drive.ts              | 306 ----------------------
 src/api/common/drive/add-file.ts                 | 307 +++++++++++++++++++++++
 src/api/common/drive/upload_from_url.ts          |  46 ++++
 src/api/common/get-host-lower.ts                 |   5 +
 src/api/common/text/elements/mention.ts          |   7 +-
 src/api/endpoints/drive/files/create.ts          |   2 +-
 src/api/endpoints/drive/files/upload_from_url.ts |  46 +---
 src/api/endpoints/posts/create.ts                |  12 +-
 src/api/endpoints/username/available.ts          |   1 +
 src/api/endpoints/users/posts.ts                 |  13 +-
 src/api/endpoints/users/recommendation.ts        |  12 +-
 src/api/endpoints/users/show.ts                  | 189 +++++++++++++-
 src/api/limitter.ts                              |   5 +-
 src/api/models/user.ts                           | 104 ++++----
 src/api/private/signin.ts                        |  13 +-
 src/api/private/signup.ts                        |   3 +-
 src/api/service/twitter.ts                       |   3 +
 src/api/streaming.ts                             |   1 +
 20 files changed, 657 insertions(+), 444 deletions(-)
 delete mode 100644 src/api/common/add-file-to-drive.ts
 create mode 100644 src/api/common/drive/add-file.ts
 create mode 100644 src/api/common/drive/upload_from_url.ts
 create mode 100644 src/api/common/get-host-lower.ts

(limited to 'src/api')

diff --git a/src/api/bot/core.ts b/src/api/bot/core.ts
index ad29f1003e..77a68aaee6 100644
--- a/src/api/bot/core.ts
+++ b/src/api/bot/core.ts
@@ -1,10 +1,11 @@
 import * as EventEmitter from 'events';
 import * as bcrypt from 'bcryptjs';
 
-import User, { IUser, init as initUser } from '../models/user';
+import User, { ILocalAccount, IUser, init as initUser } from '../models/user';
 
 import getPostSummary from '../../common/get-post-summary';
-import getUserSummary from '../../common/get-user-summary';
+import getUserSummary from '../../common/user/get-summary';
+import parseAcct from '../../common/user/parse-acct';
 import getNotificationSummary from '../../common/get-notification-summary';
 
 const hmm = [
@@ -163,9 +164,7 @@ export default class BotCore extends EventEmitter {
 
 	public async showUserCommand(q: string): Promise<string> {
 		try {
-			const user = await require('../endpoints/users/show')({
-				username: q.substr(1)
-			}, this.user);
+			const user = await require('../endpoints/users/show')(parseAcct(q.substr(1)), this.user);
 
 			const text = getUserSummary(user);
 
@@ -209,7 +208,8 @@ class SigninContext extends Context {
 		if (this.temporaryUser == null) {
 			// Fetch user
 			const user: IUser = await User.findOne({
-				username_lower: query.toLowerCase()
+				username_lower: query.toLowerCase(),
+				host: null
 			}, {
 				fields: {
 					data: false
@@ -225,7 +225,7 @@ class SigninContext extends Context {
 			}
 		} else {
 			// Compare password
-			const same = await bcrypt.compare(query, this.temporaryUser.account.password);
+			const same = await bcrypt.compare(query, (this.temporaryUser.account as ILocalAccount).password);
 
 			if (same) {
 				this.bot.signin(this.temporaryUser);
diff --git a/src/api/bot/interfaces/line.ts b/src/api/bot/interfaces/line.ts
index 6b2ebdec88..8036b2fde4 100644
--- a/src/api/bot/interfaces/line.ts
+++ b/src/api/bot/interfaces/line.ts
@@ -7,6 +7,8 @@ import config from '../../../conf';
 import BotCore from '../core';
 import _redis from '../../../db/redis';
 import prominence = require('prominence');
+import getAcct from '../../../common/user/get-acct';
+import parseAcct from '../../../common/user/parse-acct';
 import getPostSummary from '../../../common/get-post-summary';
 
 const redis = prominence(_redis);
@@ -98,10 +100,9 @@ class LineBot extends BotCore {
 	}
 
 	public async showUserCommand(q: string) {
-		const user = await require('../../endpoints/users/show')({
-			username: q.substr(1)
-		}, this.user);
+		const user = await require('../../endpoints/users/show')(parseAcct(q.substr(1)), this.user);
 
+		const acct = getAcct(user);
 		const actions = [];
 
 		actions.push({
@@ -121,7 +122,7 @@ class LineBot extends BotCore {
 		actions.push({
 			type: 'uri',
 			label: 'Webで見る',
-			uri: `${config.url}/@${user.username}`
+			uri: `${config.url}/@${acct}`
 		});
 
 		this.reply([{
@@ -130,7 +131,7 @@ class LineBot extends BotCore {
 			template: {
 				type: 'buttons',
 				thumbnailImageUrl: `${user.avatar_url}?thumbnail&size=1024`,
-				title: `${user.name} (@${user.username})`,
+				title: `${user.name} (@${acct})`,
 				text: user.description || '(no description)',
 				actions: actions
 			}
@@ -171,6 +172,7 @@ module.exports = async (app: express.Application) => {
 
 		if (session == null) {
 			const user = await User.findOne({
+				host: null,
 				'account.line': {
 					user_id: sourceId
 				}
diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts
deleted file mode 100644
index 1ee455c092..0000000000
--- a/src/api/common/add-file-to-drive.ts
+++ /dev/null
@@ -1,306 +0,0 @@
-import { Buffer } from 'buffer';
-import * as fs from 'fs';
-import * as tmp from 'tmp';
-import * as stream from 'stream';
-
-import * as mongodb from 'mongodb';
-import * as crypto from 'crypto';
-import * as _gm from 'gm';
-import * as debug from 'debug';
-import fileType = require('file-type');
-import prominence = require('prominence');
-
-import DriveFile, { getGridFSBucket } from '../models/drive-file';
-import DriveFolder from '../models/drive-folder';
-import { pack } from '../models/drive-file';
-import event, { publishDriveStream } from '../event';
-import config from '../../conf';
-
-const gm = _gm.subClass({
-	imageMagick: true
-});
-
-const log = debug('misskey:register-drive-file');
-
-const tmpFile = (): Promise<string> => new Promise((resolve, reject) => {
-	tmp.file((e, path) => {
-		if (e) return reject(e);
-		resolve(path);
-	});
-});
-
-const addToGridFS = (name: string, readable: stream.Readable, type: string, metadata: any): Promise<any> =>
-	getGridFSBucket()
-		.then(bucket => new Promise((resolve, reject) => {
-			const writeStream = bucket.openUploadStream(name, { contentType: type, metadata });
-			writeStream.once('finish', (doc) => { resolve(doc); });
-			writeStream.on('error', reject);
-			readable.pipe(writeStream);
-		}));
-
-const addFile = async (
-	user: any,
-	path: string,
-	name: string = null,
-	comment: string = null,
-	folderId: mongodb.ObjectID = null,
-	force: boolean = false
-) => {
-	log(`registering ${name} (user: ${user.username}, path: ${path})`);
-
-	// Calculate hash, get content type and get file size
-	const [hash, [mime, ext], size] = await Promise.all([
-		// hash
-		((): Promise<string> => new Promise((res, rej) => {
-			const readable = fs.createReadStream(path);
-			const hash = crypto.createHash('md5');
-			const chunks = [];
-			readable
-				.on('error', rej)
-				.pipe(hash)
-				.on('error', rej)
-				.on('data', (chunk) => chunks.push(chunk))
-				.on('end', () => {
-					const buffer = Buffer.concat(chunks);
-					res(buffer.toString('hex'));
-				});
-		}))(),
-		// mime
-		((): Promise<[string, string | null]> => new Promise((res, rej) => {
-			const readable = fs.createReadStream(path);
-			readable
-				.on('error', rej)
-				.once('data', (buffer: Buffer) => {
-					readable.destroy();
-					const type = fileType(buffer);
-					if (type) {
-						return res([type.mime, type.ext]);
-					} else {
-						// 種類が同定できなかったら application/octet-stream にする
-						return res(['application/octet-stream', null]);
-					}
-				});
-		}))(),
-		// size
-		((): Promise<number> => new Promise((res, rej) => {
-			fs.stat(path, (err, stats) => {
-				if (err) return rej(err);
-				res(stats.size);
-			});
-		}))()
-	]);
-
-	log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`);
-
-	// detect name
-	const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled');
-
-	if (!force) {
-		// Check if there is a file with the same hash
-		const much = await DriveFile.findOne({
-			md5: hash,
-			'metadata.user_id': user._id
-		});
-
-		if (much !== null) {
-			log('file with same hash is found');
-			return much;
-		} else {
-			log('file with same hash is not found');
-		}
-	}
-
-	const [wh, averageColor, folder] = await Promise.all([
-		// Width and height (when image)
-		(async () => {
-			// 画像かどうか
-			if (!/^image\/.*$/.test(mime)) {
-				return null;
-			}
-
-			const imageType = mime.split('/')[1];
-
-			// 画像でもPNGかJPEGかGIFでないならスキップ
-			if (imageType != 'png' && imageType != 'jpeg' && imageType != 'gif') {
-				return null;
-			}
-
-			log('calculate image width and height...');
-
-			// Calculate width and height
-			const g = gm(fs.createReadStream(path), name);
-			const size = await prominence(g).size();
-
-			log(`image width and height is calculated: ${size.width}, ${size.height}`);
-
-			return [size.width, size.height];
-		})(),
-		// average color (when image)
-		(async () => {
-			// 画像かどうか
-			if (!/^image\/.*$/.test(mime)) {
-				return null;
-			}
-
-			const imageType = mime.split('/')[1];
-
-			// 画像でもPNGかJPEGでないならスキップ
-			if (imageType != 'png' && imageType != 'jpeg') {
-				return null;
-			}
-
-			log('calculate average color...');
-
-			const buffer = await prominence(gm(fs.createReadStream(path), name)
-				.setFormat('ppm')
-				.resize(1, 1)) // 1pxのサイズに縮小して平均色を取得するというハック
-				.toBuffer();
-
-			const r = buffer.readUInt8(buffer.length - 3);
-			const g = buffer.readUInt8(buffer.length - 2);
-			const b = buffer.readUInt8(buffer.length - 1);
-
-			log(`average color is calculated: ${r}, ${g}, ${b}`);
-
-			return [r, g, b];
-		})(),
-		// folder
-		(async () => {
-			if (!folderId) {
-				return null;
-			}
-			const driveFolder = await DriveFolder.findOne({
-				_id: folderId,
-				user_id: user._id
-			});
-			if (!driveFolder) {
-				throw 'folder-not-found';
-			}
-			return driveFolder;
-		})(),
-		// usage checker
-		(async () => {
-			// Calculate drive usage
-			const usage = await DriveFile
-				.aggregate([{
-					$match: { 'metadata.user_id': user._id }
-				}, {
-					$project: {
-						length: true
-					}
-				}, {
-					$group: {
-						_id: null,
-						usage: { $sum: '$length' }
-					}
-				}])
-				.then((aggregates: any[]) => {
-					if (aggregates.length > 0) {
-						return aggregates[0].usage;
-					}
-					return 0;
-				});
-
-			log(`drive usage is ${usage}`);
-
-			// If usage limit exceeded
-			if (usage + size > user.drive_capacity) {
-				throw 'no-free-space';
-			}
-		})()
-	]);
-
-	const readable = fs.createReadStream(path);
-
-	const properties = {};
-
-	if (wh) {
-		properties['width'] = wh[0];
-		properties['height'] = wh[1];
-	}
-
-	if (averageColor) {
-		properties['average_color'] = averageColor;
-	}
-
-	return addToGridFS(detectedName, readable, mime, {
-		user_id: user._id,
-		folder_id: folder !== null ? folder._id : null,
-		comment: comment,
-		properties: properties
-	});
-};
-
-/**
- * Add file to drive
- *
- * @param user User who wish to add file
- * @param file File path or readableStream
- * @param comment Comment
- * @param type File type
- * @param folderId Folder ID
- * @param force If set to true, forcibly upload the file even if there is a file with the same hash.
- * @return Object that represents added file
- */
-export default (user: any, file: string | stream.Readable, ...args) => new Promise<any>((resolve, reject) => {
-	// Get file path
-	new Promise((res: (v: [string, boolean]) => void, rej) => {
-		if (typeof file === 'string') {
-			res([file, false]);
-			return;
-		}
-		if (typeof file === 'object' && typeof file.read === 'function') {
-			tmpFile()
-				.then(path => {
-					const readable: stream.Readable = file;
-					const writable = fs.createWriteStream(path);
-					readable
-						.on('error', rej)
-						.on('end', () => {
-							res([path, true]);
-						})
-						.pipe(writable)
-						.on('error', rej);
-				})
-				.catch(rej);
-		}
-		rej(new Error('un-compatible file.'));
-	})
-	.then(([path, shouldCleanup]): Promise<any> => new Promise((res, rej) => {
-		addFile(user, path, ...args)
-			.then(file => {
-				res(file);
-				if (shouldCleanup) {
-					fs.unlink(path, (e) => {
-						if (e) log(e.stack);
-					});
-				}
-			})
-			.catch(rej);
-	}))
-	.then(file => {
-		log(`drive file has been created ${file._id}`);
-		resolve(file);
-
-		pack(file).then(serializedFile => {
-			// Publish drive_file_created event
-			event(user._id, 'drive_file_created', serializedFile);
-			publishDriveStream(user._id, 'file_created', serializedFile);
-
-			// Register to search database
-			if (config.elasticsearch.enable) {
-				const es = require('../../db/elasticsearch');
-				es.index({
-					index: 'misskey',
-					type: 'drive_file',
-					id: file._id.toString(),
-					body: {
-						name: file.name,
-						user_id: user._id.toString()
-					}
-				});
-			}
-		});
-	})
-	.catch(reject);
-});
diff --git a/src/api/common/drive/add-file.ts b/src/api/common/drive/add-file.ts
new file mode 100644
index 0000000000..c4f2f212ac
--- /dev/null
+++ b/src/api/common/drive/add-file.ts
@@ -0,0 +1,307 @@
+import { Buffer } from 'buffer';
+import * as fs from 'fs';
+import * as tmp from 'tmp';
+import * as stream from 'stream';
+
+import * as mongodb from 'mongodb';
+import * as crypto from 'crypto';
+import * as _gm from 'gm';
+import * as debug from 'debug';
+import fileType = require('file-type');
+import prominence = require('prominence');
+
+import DriveFile, { getGridFSBucket } from '../../models/drive-file';
+import DriveFolder from '../../models/drive-folder';
+import { pack } from '../../models/drive-file';
+import event, { publishDriveStream } from '../../event';
+import getAcct from '../../../common/user/get-acct';
+import config from '../../../conf';
+
+const gm = _gm.subClass({
+	imageMagick: true
+});
+
+const log = debug('misskey:drive:add-file');
+
+const tmpFile = (): Promise<string> => new Promise((resolve, reject) => {
+	tmp.file((e, path) => {
+		if (e) return reject(e);
+		resolve(path);
+	});
+});
+
+const addToGridFS = (name: string, readable: stream.Readable, type: string, metadata: any): Promise<any> =>
+	getGridFSBucket()
+		.then(bucket => new Promise((resolve, reject) => {
+			const writeStream = bucket.openUploadStream(name, { contentType: type, metadata });
+			writeStream.once('finish', (doc) => { resolve(doc); });
+			writeStream.on('error', reject);
+			readable.pipe(writeStream);
+		}));
+
+const addFile = async (
+	user: any,
+	path: string,
+	name: string = null,
+	comment: string = null,
+	folderId: mongodb.ObjectID = null,
+	force: boolean = false
+) => {
+	log(`registering ${name} (user: ${getAcct(user)}, path: ${path})`);
+
+	// Calculate hash, get content type and get file size
+	const [hash, [mime, ext], size] = await Promise.all([
+		// hash
+		((): Promise<string> => new Promise((res, rej) => {
+			const readable = fs.createReadStream(path);
+			const hash = crypto.createHash('md5');
+			const chunks = [];
+			readable
+				.on('error', rej)
+				.pipe(hash)
+				.on('error', rej)
+				.on('data', (chunk) => chunks.push(chunk))
+				.on('end', () => {
+					const buffer = Buffer.concat(chunks);
+					res(buffer.toString('hex'));
+				});
+		}))(),
+		// mime
+		((): Promise<[string, string | null]> => new Promise((res, rej) => {
+			const readable = fs.createReadStream(path);
+			readable
+				.on('error', rej)
+				.once('data', (buffer: Buffer) => {
+					readable.destroy();
+					const type = fileType(buffer);
+					if (type) {
+						return res([type.mime, type.ext]);
+					} else {
+						// 種類が同定できなかったら application/octet-stream にする
+						return res(['application/octet-stream', null]);
+					}
+				});
+		}))(),
+		// size
+		((): Promise<number> => new Promise((res, rej) => {
+			fs.stat(path, (err, stats) => {
+				if (err) return rej(err);
+				res(stats.size);
+			});
+		}))()
+	]);
+
+	log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`);
+
+	// detect name
+	const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled');
+
+	if (!force) {
+		// Check if there is a file with the same hash
+		const much = await DriveFile.findOne({
+			md5: hash,
+			'metadata.user_id': user._id
+		});
+
+		if (much !== null) {
+			log('file with same hash is found');
+			return much;
+		} else {
+			log('file with same hash is not found');
+		}
+	}
+
+	const [wh, averageColor, folder] = await Promise.all([
+		// Width and height (when image)
+		(async () => {
+			// 画像かどうか
+			if (!/^image\/.*$/.test(mime)) {
+				return null;
+			}
+
+			const imageType = mime.split('/')[1];
+
+			// 画像でもPNGかJPEGかGIFでないならスキップ
+			if (imageType != 'png' && imageType != 'jpeg' && imageType != 'gif') {
+				return null;
+			}
+
+			log('calculate image width and height...');
+
+			// Calculate width and height
+			const g = gm(fs.createReadStream(path), name);
+			const size = await prominence(g).size();
+
+			log(`image width and height is calculated: ${size.width}, ${size.height}`);
+
+			return [size.width, size.height];
+		})(),
+		// average color (when image)
+		(async () => {
+			// 画像かどうか
+			if (!/^image\/.*$/.test(mime)) {
+				return null;
+			}
+
+			const imageType = mime.split('/')[1];
+
+			// 画像でもPNGかJPEGでないならスキップ
+			if (imageType != 'png' && imageType != 'jpeg') {
+				return null;
+			}
+
+			log('calculate average color...');
+
+			const buffer = await prominence(gm(fs.createReadStream(path), name)
+				.setFormat('ppm')
+				.resize(1, 1)) // 1pxのサイズに縮小して平均色を取得するというハック
+				.toBuffer();
+
+			const r = buffer.readUInt8(buffer.length - 3);
+			const g = buffer.readUInt8(buffer.length - 2);
+			const b = buffer.readUInt8(buffer.length - 1);
+
+			log(`average color is calculated: ${r}, ${g}, ${b}`);
+
+			return [r, g, b];
+		})(),
+		// folder
+		(async () => {
+			if (!folderId) {
+				return null;
+			}
+			const driveFolder = await DriveFolder.findOne({
+				_id: folderId,
+				user_id: user._id
+			});
+			if (!driveFolder) {
+				throw 'folder-not-found';
+			}
+			return driveFolder;
+		})(),
+		// usage checker
+		(async () => {
+			// Calculate drive usage
+			const usage = await DriveFile
+				.aggregate([{
+					$match: { 'metadata.user_id': user._id }
+				}, {
+					$project: {
+						length: true
+					}
+				}, {
+					$group: {
+						_id: null,
+						usage: { $sum: '$length' }
+					}
+				}])
+				.then((aggregates: any[]) => {
+					if (aggregates.length > 0) {
+						return aggregates[0].usage;
+					}
+					return 0;
+				});
+
+			log(`drive usage is ${usage}`);
+
+			// If usage limit exceeded
+			if (usage + size > user.drive_capacity) {
+				throw 'no-free-space';
+			}
+		})()
+	]);
+
+	const readable = fs.createReadStream(path);
+
+	const properties = {};
+
+	if (wh) {
+		properties['width'] = wh[0];
+		properties['height'] = wh[1];
+	}
+
+	if (averageColor) {
+		properties['average_color'] = averageColor;
+	}
+
+	return addToGridFS(detectedName, readable, mime, {
+		user_id: user._id,
+		folder_id: folder !== null ? folder._id : null,
+		comment: comment,
+		properties: properties
+	});
+};
+
+/**
+ * Add file to drive
+ *
+ * @param user User who wish to add file
+ * @param file File path or readableStream
+ * @param comment Comment
+ * @param type File type
+ * @param folderId Folder ID
+ * @param force If set to true, forcibly upload the file even if there is a file with the same hash.
+ * @return Object that represents added file
+ */
+export default (user: any, file: string | stream.Readable, ...args) => new Promise<any>((resolve, reject) => {
+	// Get file path
+	new Promise((res: (v: [string, boolean]) => void, rej) => {
+		if (typeof file === 'string') {
+			res([file, false]);
+			return;
+		}
+		if (typeof file === 'object' && typeof file.read === 'function') {
+			tmpFile()
+				.then(path => {
+					const readable: stream.Readable = file;
+					const writable = fs.createWriteStream(path);
+					readable
+						.on('error', rej)
+						.on('end', () => {
+							res([path, true]);
+						})
+						.pipe(writable)
+						.on('error', rej);
+				})
+				.catch(rej);
+		}
+		rej(new Error('un-compatible file.'));
+	})
+	.then(([path, shouldCleanup]): Promise<any> => new Promise((res, rej) => {
+		addFile(user, path, ...args)
+			.then(file => {
+				res(file);
+				if (shouldCleanup) {
+					fs.unlink(path, (e) => {
+						if (e) log(e.stack);
+					});
+				}
+			})
+			.catch(rej);
+	}))
+	.then(file => {
+		log(`drive file has been created ${file._id}`);
+		resolve(file);
+
+		pack(file).then(serializedFile => {
+			// Publish drive_file_created event
+			event(user._id, 'drive_file_created', serializedFile);
+			publishDriveStream(user._id, 'file_created', serializedFile);
+
+			// Register to search database
+			if (config.elasticsearch.enable) {
+				const es = require('../../db/elasticsearch');
+				es.index({
+					index: 'misskey',
+					type: 'drive_file',
+					id: file._id.toString(),
+					body: {
+						name: file.name,
+						user_id: user._id.toString()
+					}
+				});
+			}
+		});
+	})
+	.catch(reject);
+});
diff --git a/src/api/common/drive/upload_from_url.ts b/src/api/common/drive/upload_from_url.ts
new file mode 100644
index 0000000000..5dd9695936
--- /dev/null
+++ b/src/api/common/drive/upload_from_url.ts
@@ -0,0 +1,46 @@
+import * as URL from 'url';
+import { IDriveFile, validateFileName } from '../../models/drive-file';
+import create from './add-file';
+import * as debug from 'debug';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
+import * as request from 'request';
+
+const log = debug('misskey:common:drive:upload_from_url');
+
+export default async (url, user, folderId = null): Promise<IDriveFile> => {
+	let name = URL.parse(url).pathname.split('/').pop();
+	if (!validateFileName(name)) {
+		name = null;
+	}
+
+	// Create temp file
+	const path = await new Promise((res: (string) => void, rej) => {
+		tmp.file((e, path) => {
+			if (e) return rej(e);
+			res(path);
+		});
+	});
+
+	// write content at URL to temp file
+	await new Promise((res, rej) => {
+		const writable = fs.createWriteStream(path);
+		request(url)
+			.on('error', rej)
+			.on('end', () => {
+				writable.close();
+				res(path);
+			})
+			.pipe(writable)
+			.on('error', rej);
+	});
+
+	const driveFile = await create(user, path, name, null, folderId);
+
+	// clean-up
+	fs.unlink(path, (e) => {
+		if (e) log(e.stack);
+	});
+
+	return driveFile;
+};
diff --git a/src/api/common/get-host-lower.ts b/src/api/common/get-host-lower.ts
new file mode 100644
index 0000000000..fc4b30439e
--- /dev/null
+++ b/src/api/common/get-host-lower.ts
@@ -0,0 +1,5 @@
+import { toUnicode } from 'punycode';
+
+export default host => {
+	return toUnicode(host).replace(/[A-Z]+/, match => match.toLowerCase());
+};
diff --git a/src/api/common/text/elements/mention.ts b/src/api/common/text/elements/mention.ts
index e0fac4dd76..2025dfdaad 100644
--- a/src/api/common/text/elements/mention.ts
+++ b/src/api/common/text/elements/mention.ts
@@ -1,14 +1,17 @@
 /**
  * Mention
  */
+import parseAcct from '../../../../common/user/parse-acct';
 
 module.exports = text => {
-	const match = text.match(/^@[a-zA-Z0-9\-]+/);
+	const match = text.match(/^(?:@[a-zA-Z0-9\-]+){1,2}/);
 	if (!match) return null;
 	const mention = match[0];
+	const { username, host } = parseAcct(mention.substr(1));
 	return {
 		type: 'mention',
 		content: mention,
-		username: mention.substr(1)
+		username,
+		host
 	};
 };
diff --git a/src/api/endpoints/drive/files/create.ts b/src/api/endpoints/drive/files/create.ts
index 96bcace886..db801b61fe 100644
--- a/src/api/endpoints/drive/files/create.ts
+++ b/src/api/endpoints/drive/files/create.ts
@@ -3,7 +3,7 @@
  */
 import $ from 'cafy';
 import { validateFileName, pack } from '../../../models/drive-file';
-import create from '../../../common/add-file-to-drive';
+import create from '../../../common/drive/add-file';
 
 /**
  * Create a file
diff --git a/src/api/endpoints/drive/files/upload_from_url.ts b/src/api/endpoints/drive/files/upload_from_url.ts
index 68428747ef..346633c616 100644
--- a/src/api/endpoints/drive/files/upload_from_url.ts
+++ b/src/api/endpoints/drive/files/upload_from_url.ts
@@ -1,16 +1,9 @@
 /**
  * Module dependencies
  */
-import * as URL from 'url';
 import $ from 'cafy';
-import { validateFileName, pack } from '../../../models/drive-file';
-import create from '../../../common/add-file-to-drive';
-import * as debug from 'debug';
-import * as tmp from 'tmp';
-import * as fs from 'fs';
-import * as request from 'request';
-
-const log = debug('misskey:endpoint:upload_from_url');
+import { pack } from '../../../models/drive-file';
+import uploadFromUrl from '../../../common/drive/upload_from_url';
 
 /**
  * Create a file from a URL
@@ -25,42 +18,9 @@ module.exports = async (params, user): Promise<any> => {
 	const [url, urlErr] = $(params.url).string().$;
 	if (urlErr) throw 'invalid url param';
 
-	let name = URL.parse(url).pathname.split('/').pop();
-	if (!validateFileName(name)) {
-		name = null;
-	}
-
 	// Get 'folder_id' parameter
 	const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
 	if (folderIdErr) throw 'invalid folder_id param';
 
-	// Create temp file
-	const path = await new Promise((res: (string) => void, rej) => {
-		tmp.file((e, path) => {
-			if (e) return rej(e);
-			res(path);
-		});
-	});
-
-	// write content at URL to temp file
-	await new Promise((res, rej) => {
-		const writable = fs.createWriteStream(path);
-		request(url)
-			.on('error', rej)
-			.on('end', () => {
-				writable.close();
-				res(path);
-			})
-			.pipe(writable)
-			.on('error', rej);
-	});
-
-	const driveFile = await create(user, path, name, null, folderId);
-
-	// clean-up
-	fs.unlink(path, (e) => {
-		if (e) log(e.stack);
-	});
-
-	return pack(driveFile);
+	return pack(await uploadFromUrl(url, user, folderId));
 };
diff --git a/src/api/endpoints/posts/create.ts b/src/api/endpoints/posts/create.ts
index f46a84e1f1..286e18bb76 100644
--- a/src/api/endpoints/posts/create.ts
+++ b/src/api/endpoints/posts/create.ts
@@ -5,7 +5,7 @@ import $ from 'cafy';
 import deepEqual = require('deep-equal');
 import parse from '../../common/text';
 import { default as Post, IPost, isValidText } from '../../models/post';
-import { default as User, IUser } from '../../models/user';
+import { default as User, ILocalAccount, IUser } from '../../models/user';
 import { default as Channel, IChannel } from '../../models/channel';
 import Following from '../../models/following';
 import Mute from '../../models/mute';
@@ -16,6 +16,8 @@ import { pack } from '../../models/post';
 import notify from '../../common/notify';
 import watch from '../../common/watch-post';
 import event, { pushSw, publishChannelStream } from '../../event';
+import getAcct from '../../../common/user/get-acct';
+import parseAcct from '../../../common/user/parse-acct';
 import config from '../../../conf';
 
 /**
@@ -390,7 +392,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
 			});
 
 		// この投稿をWatchする
-		if (user.account.settings.auto_watch !== false) {
+		if ((user.account as ILocalAccount).settings.auto_watch !== false) {
 			watch(user._id, reply);
 		}
 
@@ -477,7 +479,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
 		// Extract an '@' mentions
 		const atMentions = tokens
 			.filter(t => t.type == 'mention')
-			.map(m => m.username)
+			.map(getAcct)
 			// Drop dupulicates
 			.filter((v, i, s) => s.indexOf(v) == i);
 
@@ -486,9 +488,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
 			// Fetch mentioned user
 			// SELECT _id
 			const mentionee = await User
-				.findOne({
-					username_lower: mention.toLowerCase()
-				}, { _id: true });
+				.findOne(parseAcct(mention), { _id: true });
 
 			// When mentioned user not found
 			if (mentionee == null) return;
diff --git a/src/api/endpoints/username/available.ts b/src/api/endpoints/username/available.ts
index 3be7bcba32..aac7fadf5a 100644
--- a/src/api/endpoints/username/available.ts
+++ b/src/api/endpoints/username/available.ts
@@ -19,6 +19,7 @@ module.exports = async (params) => new Promise(async (res, rej) => {
 	// Get exist
 	const exist = await User
 		.count({
+			host: null,
 			username_lower: username.toLowerCase()
 		}, {
 			limit: 1
diff --git a/src/api/endpoints/users/posts.ts b/src/api/endpoints/users/posts.ts
index 0c8bceee3d..3c84bf0d80 100644
--- a/src/api/endpoints/users/posts.ts
+++ b/src/api/endpoints/users/posts.ts
@@ -2,6 +2,7 @@
  * Module dependencies
  */
 import $ from 'cafy';
+import getHostLower from '../../common/get-host-lower';
 import Post, { pack } from '../../models/post';
 import User from '../../models/user';
 
@@ -22,7 +23,15 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
 	if (usernameErr) return rej('invalid username param');
 
 	if (userId === undefined && username === undefined) {
-		return rej('user_id or username is required');
+		return rej('user_id or pair of username and host is required');
+	}
+
+	// Get 'host' parameter
+	const [host, hostErr] = $(params.host).optional.string().$;
+	if (hostErr) return rej('invalid host param');
+
+	if (userId === undefined && host === undefined) {
+		return rej('user_id or pair of username and host is required');
 	}
 
 	// Get 'include_replies' parameter
@@ -60,7 +69,7 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
 
 	const q = userId !== undefined
 		? { _id: userId }
-		: { username_lower: username.toLowerCase() } ;
+		: { username_lower: username.toLowerCase(), host_lower: getHostLower(host) } ;
 
 	// Lookup user
 	const user = await User.findOne(q, {
diff --git a/src/api/endpoints/users/recommendation.ts b/src/api/endpoints/users/recommendation.ts
index f1f5bcd0ac..45d90f422b 100644
--- a/src/api/endpoints/users/recommendation.ts
+++ b/src/api/endpoints/users/recommendation.ts
@@ -30,9 +30,15 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
 			_id: {
 				$nin: followingIds
 			},
-			'account.last_used_at': {
-				$gte: new Date(Date.now() - ms('7days'))
-			}
+			$or: [
+				{
+					'account.last_used_at': {
+						$gte: new Date(Date.now() - ms('7days'))
+					}
+				}, {
+					host: { $not: null }
+				}
+			]
 		}, {
 			limit: limit,
 			skip: offset,
diff --git a/src/api/endpoints/users/show.ts b/src/api/endpoints/users/show.ts
index 7aea59296a..78df23f339 100644
--- a/src/api/endpoints/users/show.ts
+++ b/src/api/endpoints/users/show.ts
@@ -2,7 +2,49 @@
  * Module dependencies
  */
 import $ from 'cafy';
-import User, { pack } from '../../models/user';
+import { JSDOM } from 'jsdom';
+import { toUnicode, toASCII } from 'punycode';
+import uploadFromUrl from '../../common/drive/upload_from_url';
+import User, { pack, validateUsername, isValidName, isValidDescription } from '../../models/user';
+const request = require('request-promise-native');
+const WebFinger = require('webfinger.js');
+
+const webFinger = new WebFinger({});
+
+async function getCollectionCount(url) {
+	if (!url) {
+		return null;
+	}
+
+	try {
+		const collection = await request({ url, json: true });
+		return collection ? collection.totalItems : null;
+	} catch (exception) {
+		return null;
+	}
+}
+
+function findUser(q) {
+	return User.findOne(q, {
+		fields: {
+			data: false
+		}
+	});
+}
+
+function webFingerAndVerify(query, verifier) {
+	return new Promise((res, rej) => webFinger.lookup(query, (error, result) => {
+		if (error) {
+			return rej(error);
+		}
+
+		if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) {
+			return rej('WebFinger verfification failed');
+		}
+
+		res(result.object);
+	}));
+}
 
 /**
  * Show a user
@@ -12,6 +54,8 @@ import User, { pack } from '../../models/user';
  * @return {Promise<any>}
  */
 module.exports = (params, me) => new Promise(async (res, rej) => {
+	let user;
+
 	// Get 'user_id' parameter
 	const [userId, userIdErr] = $(params.user_id).optional.id().$;
 	if (userIdErr) return rej('invalid user_id param');
@@ -20,23 +64,142 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
 	const [username, usernameErr] = $(params.username).optional.string().$;
 	if (usernameErr) return rej('invalid username param');
 
-	if (userId === undefined && username === undefined) {
-		return rej('user_id or username is required');
-	}
+	// Get 'host' parameter
+	const [host, hostErr] = $(params.host).optional.string().$;
+	if (hostErr) return rej('invalid username param');
 
-	const q = userId !== undefined
-		? { _id: userId }
-		: { username_lower: username.toLowerCase() };
+	if (userId === undefined && typeof username !== 'string') {
+		return rej('user_id or pair of username and host is required');
+	}
 
 	// Lookup user
-	const user = await User.findOne(q, {
-		fields: {
-			data: false
+	if (typeof host === 'string') {
+		const username_lower = username.toLowerCase();
+		const host_lower_ascii = toASCII(host).toLowerCase();
+		const host_lower = toUnicode(host_lower_ascii);
+
+		user = await findUser({ username_lower, host_lower });
+
+		if (user === null) {
+			const acct_lower = `${username_lower}@${host_lower_ascii}`;
+			let activityStreams;
+			let finger;
+			let followers_count;
+			let following_count;
+			let likes_count;
+			let posts_count;
+
+			if (!validateUsername(username)) {
+				return rej('username validation failed');
+			}
+
+			try {
+				finger = await webFingerAndVerify(acct_lower, acct_lower);
+			} catch (exception) {
+				return rej('WebFinger lookup failed');
+			}
+
+			const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self');
+			if (!self) {
+				return rej('WebFinger has no reference to self representation');
+			}
+
+			try {
+				activityStreams = await request({
+					url: self.href,
+					headers: {
+						Accept: 'application/activity+json, application/ld+json'
+					},
+					json: true
+				});
+			} catch (exception) {
+				return rej('failed to retrieve ActivityStreams representation');
+			}
+
+			if (!(activityStreams &&
+				(Array.isArray(activityStreams['@context']) ?
+					activityStreams['@context'].includes('https://www.w3.org/ns/activitystreams') :
+					activityStreams['@context'] === 'https://www.w3.org/ns/activitystreams') &&
+				activityStreams.type === 'Person' &&
+				typeof activityStreams.preferredUsername === 'string' &&
+				activityStreams.preferredUsername.toLowerCase() === username_lower &&
+				isValidName(activityStreams.name) &&
+				isValidDescription(activityStreams.summary)
+			)) {
+				return rej('failed ActivityStreams validation');
+			}
+
+			try {
+				[followers_count, following_count, likes_count, posts_count] = await Promise.all([
+					getCollectionCount(activityStreams.followers),
+					getCollectionCount(activityStreams.following),
+					getCollectionCount(activityStreams.liked),
+					getCollectionCount(activityStreams.outbox),
+					webFingerAndVerify(activityStreams.id, acct_lower),
+				]);
+			} catch (exception) {
+				return rej('failed to fetch assets');
+			}
+
+			const summaryDOM = JSDOM.fragment(activityStreams.summary);
+
+			// Create user
+			user = await User.insert({
+				avatar_id: null,
+				banner_id: null,
+				created_at: new Date(),
+				description: summaryDOM.textContent,
+				followers_count,
+				following_count,
+				name: activityStreams.name,
+				posts_count,
+				likes_count,
+				liked_count: 0,
+				drive_capacity: 1073741824, // 1GB
+				username: username,
+				username_lower,
+				host: toUnicode(finger.subject.replace(/^.*?@/, '')),
+				host_lower,
+				account: {
+					uri: activityStreams.id,
+				},
+			});
+
+			const [icon, image] = await Promise.all([
+				activityStreams.icon,
+				activityStreams.image,
+			].map(async image => {
+				if (!image || image.type !== 'Image') {
+					return { _id: null };
+				}
+
+				try {
+					return await uploadFromUrl(image.url, user);
+				} catch (exception) {
+					return { _id: null };
+				}
+			}));
+
+			User.update({ _id: user._id }, {
+				$set: {
+					avatar_id: icon._id,
+					banner_id: image._id,
+				},
+			});
+
+			user.avatar_id = icon._id;
+			user.banner_id = icon._id;
 		}
-	});
+	} else {
+		const q = userId !== undefined
+			? { _id: userId }
+			: { username_lower: username.toLowerCase(), host: null };
 
-	if (user === null) {
-		return rej('user not found');
+		user = await findUser(q);
+
+		if (user === null) {
+			return rej('user not found');
+		}
 	}
 
 	// Send response
diff --git a/src/api/limitter.ts b/src/api/limitter.ts
index 10c50c3403..9d2c42d335 100644
--- a/src/api/limitter.ts
+++ b/src/api/limitter.ts
@@ -3,6 +3,7 @@ import * as debug from 'debug';
 import limiterDB from '../db/redis';
 import { Endpoint } from './endpoints';
 import { IAuthContext } from './authenticate';
+import getAcct from '../common/user/get-acct';
 
 const log = debug('misskey:limitter');
 
@@ -42,7 +43,7 @@ export default (endpoint: Endpoint, ctx: IAuthContext) => new Promise((ok, rejec
 				return reject('ERR');
 			}
 
-			log(`@${ctx.user.username} ${endpoint.name} min remaining: ${info.remaining}`);
+			log(`@${getAcct(ctx.user)} ${endpoint.name} min remaining: ${info.remaining}`);
 
 			if (info.remaining === 0) {
 				reject('BRIEF_REQUEST_INTERVAL');
@@ -70,7 +71,7 @@ export default (endpoint: Endpoint, ctx: IAuthContext) => new Promise((ok, rejec
 				return reject('ERR');
 			}
 
-			log(`@${ctx.user.username} ${endpoint.name} max remaining: ${info.remaining}`);
+			log(`@${getAcct(ctx.user)} ${endpoint.name} max remaining: ${info.remaining}`);
 
 			if (info.remaining === 0) {
 				reject('RATE_LIMIT_EXCEEDED');
diff --git a/src/api/models/user.ts b/src/api/models/user.ts
index 46d32963bc..e73c95faf2 100644
--- a/src/api/models/user.ts
+++ b/src/api/models/user.ts
@@ -39,6 +39,39 @@ export function isValidBirthday(birthday: string): boolean {
 	return typeof birthday == 'string' && /^([0-9]{4})\-([0-9]{2})-([0-9]{2})$/.test(birthday);
 }
 
+export type ILocalAccount = {
+	keypair: string;
+	email: string;
+	links: string[];
+	password: string;
+	token: string;
+	twitter: {
+		access_token: string;
+		access_token_secret: string;
+		user_id: string;
+		screen_name: string;
+	};
+	line: {
+		user_id: string;
+	};
+	profile: {
+		location: string;
+		birthday: string; // 'YYYY-MM-DD'
+		tags: string[];
+	};
+	last_used_at: Date;
+	is_bot: boolean;
+	is_pro: boolean;
+	two_factor_secret: string;
+	two_factor_enabled: boolean;
+	client_settings: any;
+	settings: any;
+};
+
+export type IRemoteAccount = {
+	uri: string;
+};
+
 export type IUser = {
 	_id: mongo.ObjectID;
 	created_at: Date;
@@ -60,34 +93,7 @@ export type IUser = {
 	keywords: string[];
 	host: string;
 	host_lower: string;
-	account: {
-		keypair: string;
-		email: string;
-		links: string[];
-		password: string;
-		token: string;
-		twitter: {
-			access_token: string;
-			access_token_secret: string;
-			user_id: string;
-			screen_name: string;
-		};
-		line: {
-			user_id: string;
-		};
-		profile: {
-			location: string;
-			birthday: string; // 'YYYY-MM-DD'
-			tags: string[];
-		};
-		last_used_at: Date;
-		is_bot: boolean;
-		is_pro: boolean;
-		two_factor_secret: string;
-		two_factor_enabled: boolean;
-		client_settings: any;
-		settings: any;
-	};
+	account: ILocalAccount | IRemoteAccount;
 };
 
 export function init(user): IUser {
@@ -162,28 +168,30 @@ export const pack = (
 	// Remove needless properties
 	delete _user.latest_post;
 
-	// Remove private properties
-	delete _user.account.keypair;
-	delete _user.account.password;
-	delete _user.account.token;
-	delete _user.account.two_factor_temp_secret;
-	delete _user.account.two_factor_secret;
-	delete _user.username_lower;
-	if (_user.account.twitter) {
-		delete _user.account.twitter.access_token;
-		delete _user.account.twitter.access_token_secret;
-	}
-	delete _user.account.line;
+	if (!_user.host) {
+		// Remove private properties
+		delete _user.account.keypair;
+		delete _user.account.password;
+		delete _user.account.token;
+		delete _user.account.two_factor_temp_secret;
+		delete _user.account.two_factor_secret;
+		delete _user.username_lower;
+		if (_user.account.twitter) {
+			delete _user.account.twitter.access_token;
+			delete _user.account.twitter.access_token_secret;
+		}
+		delete _user.account.line;
 
-	// Visible via only the official client
-	if (!opts.includeSecrets) {
-		delete _user.account.email;
-		delete _user.account.settings;
-		delete _user.account.client_settings;
-	}
+		// Visible via only the official client
+		if (!opts.includeSecrets) {
+			delete _user.account.email;
+			delete _user.account.settings;
+			delete _user.account.client_settings;
+		}
 
-	if (!opts.detail) {
-		delete _user.account.two_factor_enabled;
+		if (!opts.detail) {
+			delete _user.account.two_factor_enabled;
+		}
 	}
 
 	_user.avatar_url = _user.avatar_id != null
diff --git a/src/api/private/signin.ts b/src/api/private/signin.ts
index ae0be03c73..00dcb8afc8 100644
--- a/src/api/private/signin.ts
+++ b/src/api/private/signin.ts
@@ -1,7 +1,7 @@
 import * as express from 'express';
 import * as bcrypt from 'bcryptjs';
 import * as speakeasy from 'speakeasy';
-import { default as User, IUser } from '../models/user';
+import { default as User, ILocalAccount, IUser } from '../models/user';
 import Signin, { pack } from '../models/signin';
 import event from '../event';
 import signin from '../common/signin';
@@ -32,7 +32,8 @@ export default async (req: express.Request, res: express.Response) => {
 
 	// Fetch user
 	const user: IUser = await User.findOne({
-		username_lower: username.toLowerCase()
+		username_lower: username.toLowerCase(),
+		host: null
 	}, {
 		fields: {
 			data: false,
@@ -47,13 +48,15 @@ export default async (req: express.Request, res: express.Response) => {
 		return;
 	}
 
+	const account = user.account as ILocalAccount;
+
 	// Compare password
-	const same = await bcrypt.compare(password, user.account.password);
+	const same = await bcrypt.compare(password, account.password);
 
 	if (same) {
-		if (user.account.two_factor_enabled) {
+		if (account.two_factor_enabled) {
 			const verified = (speakeasy as any).totp.verify({
-				secret: user.account.two_factor_secret,
+				secret: account.two_factor_secret,
 				encoding: 'base32',
 				token: token
 			});
diff --git a/src/api/private/signup.ts b/src/api/private/signup.ts
index 280153d4f5..96e0495709 100644
--- a/src/api/private/signup.ts
+++ b/src/api/private/signup.ts
@@ -64,7 +64,8 @@ export default async (req: express.Request, res: express.Response) => {
 	// Fetch exist user that same username
 	const usernameExist = await User
 		.count({
-			username_lower: username.toLowerCase()
+			username_lower: username.toLowerCase(),
+			host: null
 		}, {
 			limit: 1
 		});
diff --git a/src/api/service/twitter.ts b/src/api/service/twitter.ts
index 02b613454c..c1f2e48a63 100644
--- a/src/api/service/twitter.ts
+++ b/src/api/service/twitter.ts
@@ -39,6 +39,7 @@ module.exports = (app: express.Application) => {
 		if (userToken == null) return res.send('plz signin');
 
 		const user = await User.findOneAndUpdate({
+			host: null,
 			'account.token': userToken
 		}, {
 			$set: {
@@ -126,6 +127,7 @@ module.exports = (app: express.Application) => {
 				const result = await twAuth.done(JSON.parse(ctx), req.query.oauth_verifier);
 
 				const user = await User.findOne({
+					host: null,
 					'account.twitter.user_id': result.userId
 				});
 
@@ -148,6 +150,7 @@ module.exports = (app: express.Application) => {
 				const result = await twAuth.done(JSON.parse(ctx), verifier);
 
 				const user = await User.findOneAndUpdate({
+					host: null,
 					'account.token': userToken
 				}, {
 					$set: {
diff --git a/src/api/streaming.ts b/src/api/streaming.ts
index 427e01afdd..a6759e414c 100644
--- a/src/api/streaming.ts
+++ b/src/api/streaming.ts
@@ -94,6 +94,7 @@ function authenticate(token: string): Promise<IUser> {
 			// Fetch user
 			const user: IUser = await User
 				.findOne({
+					host: null,
 					'account.token': token
 				});
 
-- 
cgit v1.2.3-freya