Introduce processor

8 files changed, 818 insertions, 0 deletions

diff --git a/src/server/api/endpoints/users/followers.ts b/src/server/api/endpoints/users/followers.ts
new file mode 100644
index 0000000000..b0fb83c683
--- /dev/null
+++ b/src/server/api/endpoints/users/followers.ts
@@ -0,0 +1,92 @@
+/**
+ * Module dependencies
+ */
+import $ from 'cafy';
+import User from '../../models/user';
+import Following from '../../models/following';
+import { pack } from '../../models/user';
+import getFriends from '../../common/get-friends';
+
+/**
+ * Get followers of a user
+ *
+ * @param {any} params
+ * @param {any} me
+ * @return {Promise<any>}
+ */
+module.exports = (params, me) => new Promise(async (res, rej) => {
+	// Get 'user_id' parameter
+	const [userId, userIdErr] = $(params.user_id).id().$;
+	if (userIdErr) return rej('invalid user_id param');
+
+	// Get 'iknow' parameter
+	const [iknow = false, iknowErr] = $(params.iknow).optional.boolean().$;
+	if (iknowErr) return rej('invalid iknow param');
+
+	// Get 'limit' parameter
+	const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
+	if (limitErr) return rej('invalid limit param');
+
+	// Get 'cursor' parameter
+	const [cursor = null, cursorErr] = $(params.cursor).optional.id().$;
+	if (cursorErr) return rej('invalid cursor param');
+
+	// Lookup user
+	const user = await User.findOne({
+		_id: userId
+	}, {
+		fields: {
+			_id: true
+		}
+	});
+
+	if (user === null) {
+		return rej('user not found');
+	}
+
+	// Construct query
+	const query = {
+		followee_id: user._id,
+		deleted_at: { $exists: false }
+	} as any;
+
+	// ログインしていてかつ iknow フラグがあるとき
+	if (me && iknow) {
+		// Get my friends
+		const myFriends = await getFriends(me._id);
+
+		query.follower_id = {
+			$in: myFriends
+		};
+	}
+
+	// カーソルが指定されている場合
+	if (cursor) {
+		query._id = {
+			$lt: cursor
+		};
+	}
+
+	// Get followers
+	const following = await Following
+		.find(query, {
+			limit: limit + 1,
+			sort: { _id: -1 }
+		});
+
+	// 「次のページ」があるかどうか
+	const inStock = following.length === limit + 1;
+	if (inStock) {
+		following.pop();
+	}
+
+	// Serialize
+	const users = await Promise.all(following.map(async f =>
+		await pack(f.follower_id, me, { detail: true })));
+
+	// Response
+	res({
+		users: users,
+		next: inStock ? following[following.length - 1]._id : null,
+	});
+});
diff --git a/src/server/api/endpoints/users/following.ts b/src/server/api/endpoints/users/following.ts
new file mode 100644
index 0000000000..8e88431e92
--- /dev/null
+++ b/src/server/api/endpoints/users/following.ts
@@ -0,0 +1,92 @@
+/**
+ * Module dependencies
+ */
+import $ from 'cafy';
+import User from '../../models/user';
+import Following from '../../models/following';
+import { pack } from '../../models/user';
+import getFriends from '../../common/get-friends';
+
+/**
+ * Get following users of a user
+ *
+ * @param {any} params
+ * @param {any} me
+ * @return {Promise<any>}
+ */
+module.exports = (params, me) => new Promise(async (res, rej) => {
+	// Get 'user_id' parameter
+	const [userId, userIdErr] = $(params.user_id).id().$;
+	if (userIdErr) return rej('invalid user_id param');
+
+	// Get 'iknow' parameter
+	const [iknow = false, iknowErr] = $(params.iknow).optional.boolean().$;
+	if (iknowErr) return rej('invalid iknow param');
+
+	// Get 'limit' parameter
+	const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
+	if (limitErr) return rej('invalid limit param');
+
+	// Get 'cursor' parameter
+	const [cursor = null, cursorErr] = $(params.cursor).optional.id().$;
+	if (cursorErr) return rej('invalid cursor param');
+
+	// Lookup user
+	const user = await User.findOne({
+		_id: userId
+	}, {
+		fields: {
+			_id: true
+		}
+	});
+
+	if (user === null) {
+		return rej('user not found');
+	}
+
+	// Construct query
+	const query = {
+		follower_id: user._id,
+		deleted_at: { $exists: false }
+	} as any;
+
+	// ログインしていてかつ iknow フラグがあるとき
+	if (me && iknow) {
+		// Get my friends
+		const myFriends = await getFriends(me._id);
+
+		query.followee_id = {
+			$in: myFriends
+		};
+	}
+
+	// カーソルが指定されている場合
+	if (cursor) {
+		query._id = {
+			$lt: cursor
+		};
+	}
+
+	// Get followers
+	const following = await Following
+		.find(query, {
+			limit: limit + 1,
+			sort: { _id: -1 }
+		});
+
+	// 「次のページ」があるかどうか
+	const inStock = following.length === limit + 1;
+	if (inStock) {
+		following.pop();
+	}
+
+	// Serialize
+	const users = await Promise.all(following.map(async f =>
+		await pack(f.followee_id, me, { detail: true })));
+
+	// Response
+	res({
+		users: users,
+		next: inStock ? following[following.length - 1]._id : null,
+	});
+});
diff --git a/src/server/api/endpoints/users/get_frequently_replied_users.ts b/src/server/api/endpoints/users/get_frequently_replied_users.ts
new file mode 100644
index 0000000000..87f4f77a5b
--- /dev/null
+++ b/src/server/api/endpoints/users/get_frequently_replied_users.ts
@@ -0,0 +1,99 @@
+/**
+ * Module dependencies
+ */
+import $ from 'cafy';
+import Post from '../../models/post';
+import User, { pack } from '../../models/user';
+
+module.exports = (params, me) => new Promise(async (res, rej) => {
+	// Get 'user_id' parameter
+	const [userId, userIdErr] = $(params.user_id).id().$;
+	if (userIdErr) return rej('invalid user_id param');
+
+	// Get 'limit' parameter
+	const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
+	if (limitErr) return rej('invalid limit param');
+
+	// Lookup user
+	const user = await User.findOne({
+		_id: userId
+	}, {
+		fields: {
+			_id: true
+		}
+	});
+
+	if (user === null) {
+		return rej('user not found');
+	}
+
+	// Fetch recent posts
+	const recentPosts = await Post.find({
+		user_id: user._id,
+		reply_id: {
+			$exists: true,
+			$ne: null
+		}
+	}, {
+		sort: {
+			_id: -1
+		},
+		limit: 1000,
+		fields: {
+			_id: false,
+			reply_id: true
+		}
+	});
+
+	// 投稿が少なかったら中断
+	if (recentPosts.length === 0) {
+		return res([]);
+	}
+
+	const replyTargetPosts = await Post.find({
+		_id: {
+			$in: recentPosts.map(p => p.reply_id)
+		},
+		user_id: {
+			$ne: user._id
+		}
+	}, {
+		fields: {
+			_id: false,
+			user_id: true
+		}
+	});
+
+	const repliedUsers = {};
+
+	// Extract replies from recent posts
+	replyTargetPosts.forEach(post => {
+		const userId = post.user_id.toString();
+		if (repliedUsers[userId]) {
+			repliedUsers[userId]++;
+		} else {
+			repliedUsers[userId] = 1;
+		}
+	});
+
+	// Calc peak
+	let peak = 0;
+	Object.keys(repliedUsers).forEach(user => {
+		if (repliedUsers[user] > peak) peak = repliedUsers[user];
+	});
+
+	// Sort replies by frequency
+	const repliedUsersSorted = Object.keys(repliedUsers).sort((a, b) => repliedUsers[b] - repliedUsers[a]);
+
+	// Extract top replied users
+	const topRepliedUsers = repliedUsersSorted.slice(0, limit);
+
+	// Make replies object (includes weights)
+	const repliesObj = await Promise.all(topRepliedUsers.map(async (user) => ({
+		user: await pack(user, me, { detail: true }),
+		weight: repliedUsers[user] / peak
+	})));
+
+	// Response
+	res(repliesObj);
+});
diff --git a/src/server/api/endpoints/users/posts.ts b/src/server/api/endpoints/users/posts.ts
new file mode 100644
index 0000000000..3c84bf0d80
--- /dev/null
+++ b/src/server/api/endpoints/users/posts.ts
@@ -0,0 +1,137 @@
+/**
+ * Module dependencies
+ */
+import $ from 'cafy';
+import getHostLower from '../../common/get-host-lower';
+import Post, { pack } from '../../models/post';
+import User from '../../models/user';
+
+/**
+ * Get posts of a user
+ *
+ * @param {any} params
+ * @param {any} me
+ * @return {Promise<any>}
+ */
+module.exports = (params, me) => new Promise(async (res, rej) => {
+	// Get 'user_id' parameter
+	const [userId, userIdErr] = $(params.user_id).optional.id().$;
+	if (userIdErr) return rej('invalid user_id param');
+
+	// Get 'username' parameter
+	const [username, usernameErr] = $(params.username).optional.string().$;
+	if (usernameErr) return rej('invalid username param');
+
+	if (userId === undefined && username === undefined) {
+		return rej('user_id or pair of username and host is required');
+	}
+
+	// Get 'host' parameter
+	const [host, hostErr] = $(params.host).optional.string().$;
+	if (hostErr) return rej('invalid host param');
+
+	if (userId === undefined && host === undefined) {
+		return rej('user_id or pair of username and host is required');
+	}
+
+	// Get 'include_replies' parameter
+	const [includeReplies = true, includeRepliesErr] = $(params.include_replies).optional.boolean().$;
+	if (includeRepliesErr) return rej('invalid include_replies param');
+
+	// Get 'with_media' parameter
+	const [withMedia = false, withMediaErr] = $(params.with_media).optional.boolean().$;
+	if (withMediaErr) return rej('invalid with_media param');
+
+	// Get 'limit' parameter
+	const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
+	if (limitErr) return rej('invalid limit param');
+
+	// Get 'since_id' parameter
+	const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$;
+	if (sinceIdErr) return rej('invalid since_id param');
+
+	// Get 'until_id' parameter
+	const [untilId, untilIdErr] = $(params.until_id).optional.id().$;
+	if (untilIdErr) return rej('invalid until_id param');
+
+	// Get 'since_date' parameter
+	const [sinceDate, sinceDateErr] = $(params.since_date).optional.number().$;
+	if (sinceDateErr) throw 'invalid since_date param';
+
+	// Get 'until_date' parameter
+	const [untilDate, untilDateErr] = $(params.until_date).optional.number().$;
+	if (untilDateErr) throw 'invalid until_date param';
+
+	// Check if only one of since_id, until_id, since_date, until_date specified
+	if ([sinceId, untilId, sinceDate, untilDate].filter(x => x != null).length > 1) {
+		throw 'only one of since_id, until_id, since_date, until_date can be specified';
+	}
+
+	const q = userId !== undefined
+		? { _id: userId }
+		: { username_lower: username.toLowerCase(), host_lower: getHostLower(host) } ;
+
+	// Lookup user
+	const user = await User.findOne(q, {
+		fields: {
+			_id: true
+		}
+	});
+
+	if (user === null) {
+		return rej('user not found');
+	}
+
+	//#region Construct query
+	const sort = {
+		_id: -1
+	};
+
+	const query = {
+		user_id: user._id
+	} as any;
+
+	if (sinceId) {
+		sort._id = 1;
+		query._id = {
+			$gt: sinceId
+		};
+	} else if (untilId) {
+		query._id = {
+			$lt: untilId
+		};
+	} else if (sinceDate) {
+		sort._id = 1;
+		query.created_at = {
+			$gt: new Date(sinceDate)
+		};
+	} else if (untilDate) {
+		query.created_at = {
+			$lt: new Date(untilDate)
+		};
+	}
+
+	if (!includeReplies) {
+		query.reply_id = null;
+	}
+
+	if (withMedia) {
+		query.media_ids = {
+			$exists: true,
+			$ne: null
+		};
+	}
+	//#endregion
+
+	// Issue query
+	const posts = await Post
+		.find(query, {
+			limit: limit,
+			sort: sort
+		});
+
+	// Serialize
+	res(await Promise.all(posts.map(async (post) =>
+		await pack(post, me)
+	)));
+});
diff --git a/src/server/api/endpoints/users/recommendation.ts b/src/server/api/endpoints/users/recommendation.ts
new file mode 100644
index 0000000000..45d90f422b
--- /dev/null
+++ b/src/server/api/endpoints/users/recommendation.ts
@@ -0,0 +1,53 @@
+/**
+ * Module dependencies
+ */
+const ms = require('ms');
+import $ from 'cafy';
+import User, { pack } from '../../models/user';
+import getFriends from '../../common/get-friends';
+
+/**
+ * Get recommended users
+ *
+ * @param {any} params
+ * @param {any} me
+ * @return {Promise<any>}
+ */
+module.exports = (params, me) => new Promise(async (res, rej) => {
+	// Get 'limit' parameter
+	const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
+	if (limitErr) return rej('invalid limit param');
+
+	// Get 'offset' parameter
+	const [offset = 0, offsetErr] = $(params.offset).optional.number().min(0).$;
+	if (offsetErr) return rej('invalid offset param');
+
+	// ID list of the user itself and other users who the user follows
+	const followingIds = await getFriends(me._id);
+
+	const users = await User
+		.find({
+			_id: {
+				$nin: followingIds
+			},
+			$or: [
+				{
+					'account.last_used_at': {
+						$gte: new Date(Date.now() - ms('7days'))
+					}
+				}, {
+					host: { $not: null }
+				}
+			]
+		}, {
+			limit: limit,
+			skip: offset,
+			sort: {
+				followers_count: -1
+			}
+		});
+
+	// Serialize
+	res(await Promise.all(users.map(async user =>
+		await pack(user, me, { detail: true }))));
+});
diff --git a/src/server/api/endpoints/users/search.ts b/src/server/api/endpoints/users/search.ts
new file mode 100644
index 0000000000..3c81576440
--- /dev/null
+++ b/src/server/api/endpoints/users/search.ts
@@ -0,0 +1,98 @@
+/**
+ * Module dependencies
+ */
+import * as mongo from 'mongodb';
+import $ from 'cafy';
+import User, { pack } from '../../models/user';
+import config from '../../../../conf';
+const escapeRegexp = require('escape-regexp');
+
+/**
+ * Search a user
+ *
+ * @param {any} params
+ * @param {any} me
+ * @return {Promise<any>}
+ */
+module.exports = (params, me) => new Promise(async (res, rej) => {
+	// Get 'query' parameter
+	const [query, queryError] = $(params.query).string().pipe(x => x != '').$;
+	if (queryError) return rej('invalid query param');
+
+	// Get 'offset' parameter
+	const [offset = 0, offsetErr] = $(params.offset).optional.number().min(0).$;
+	if (offsetErr) return rej('invalid offset param');
+
+	// Get 'max' parameter
+	const [max = 10, maxErr] = $(params.max).optional.number().range(1, 30).$;
+	if (maxErr) return rej('invalid max param');
+
+	// If Elasticsearch is available, search by $
+	// If not, search by MongoDB
+	(config.elasticsearch.enable ? byElasticsearch : byNative)
+		(res, rej, me, query, offset, max);
+});
+
+// Search by MongoDB
+async function byNative(res, rej, me, query, offset, max) {
+	const escapedQuery = escapeRegexp(query);
+
+	// Search users
+	const users = await User
+		.find({
+			$or: [{
+				username_lower: new RegExp(escapedQuery.replace('@', '').toLowerCase())
+			}, {
+				name: new RegExp(escapedQuery)
+			}]
+		}, {
+			limit: max
+		});
+
+	// Serialize
+	res(await Promise.all(users.map(async user =>
+		await pack(user, me, { detail: true }))));
+}
+
+// Search by Elasticsearch
+async function byElasticsearch(res, rej, me, query, offset, max) {
+	const es = require('../../db/elasticsearch');
+
+	es.search({
+		index: 'misskey',
+		type: 'user',
+		body: {
+			size: max,
+			from: offset,
+			query: {
+				simple_query_string: {
+					fields: ['username', 'name', 'bio'],
+					query: query,
+					default_operator: 'and'
+				}
+			}
+		}
+	}, async (error, response) => {
+		if (error) {
+			console.error(error);
+			return res(500);
+		}
+
+		if (response.hits.total === 0) {
+			return res([]);
+		}
+
+		const hits = response.hits.hits.map(hit => new mongo.ObjectID(hit._id));
+
+		const users = await User
+			.find({
+				_id: {
+					$in: hits
+				}
+			});
+
+		// Serialize
+		res(await Promise.all(users.map(async user =>
+			await pack(user, me, { detail: true }))));
+	});
+}
diff --git a/src/server/api/endpoints/users/search_by_username.ts b/src/server/api/endpoints/users/search_by_username.ts
new file mode 100644
index 0000000000..9c5e1905aa
--- /dev/null
+++ b/src/server/api/endpoints/users/search_by_username.ts
@@ -0,0 +1,38 @@
+/**
+ * Module dependencies
+ */
+import $ from 'cafy';
+import User, { pack } from '../../models/user';
+
+/**
+ * Search a user by username
+ *
+ * @param {any} params
+ * @param {any} me
+ * @return {Promise<any>}
+ */
+module.exports = (params, me) => new Promise(async (res, rej) => {
+	// Get 'query' parameter
+	const [query, queryError] = $(params.query).string().$;
+	if (queryError) return rej('invalid query param');
+
+	// Get 'offset' parameter
+	const [offset = 0, offsetErr] = $(params.offset).optional.number().min(0).$;
+	if (offsetErr) return rej('invalid offset param');
+
+	// Get 'limit' parameter
+	const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
+	if (limitErr) return rej('invalid limit param');
+
+	const users = await User
+		.find({
+			username_lower: new RegExp(query.toLowerCase())
+		}, {
+			limit: limit,
+			skip: offset
+		});
+
+	// Serialize
+	res(await Promise.all(users.map(async user =>
+		await pack(user, me, { detail: true }))));
+});
diff --git a/src/server/api/endpoints/users/show.ts b/src/server/api/endpoints/users/show.ts
new file mode 100644
index 0000000000..78df23f339
--- /dev/null
+++ b/src/server/api/endpoints/users/show.ts
@@ -0,0 +1,209 @@
+/**
+ * Module dependencies
+ */
+import $ from 'cafy';
+import { JSDOM } from 'jsdom';
+import { toUnicode, toASCII } from 'punycode';
+import uploadFromUrl from '../../common/drive/upload_from_url';
+import User, { pack, validateUsername, isValidName, isValidDescription } from '../../models/user';
+const request = require('request-promise-native');
+const WebFinger = require('webfinger.js');
+
+const webFinger = new WebFinger({});
+
+async function getCollectionCount(url) {
+	if (!url) {
+		return null;
+	}
+
+	try {
+		const collection = await request({ url, json: true });
+		return collection ? collection.totalItems : null;
+	} catch (exception) {
+		return null;
+	}
+}
+
+function findUser(q) {
+	return User.findOne(q, {
+		fields: {
+			data: false
+		}
+	});
+}
+
+function webFingerAndVerify(query, verifier) {
+	return new Promise((res, rej) => webFinger.lookup(query, (error, result) => {
+		if (error) {
+			return rej(error);
+		}
+
+		if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) {
+			return rej('WebFinger verfification failed');
+		}
+
+		res(result.object);
+	}));
+}
+
+/**
+ * Show a user
+ *
+ * @param {any} params
+ * @param {any} me
+ * @return {Promise<any>}
+ */
+module.exports = (params, me) => new Promise(async (res, rej) => {
+	let user;
+
+	// Get 'user_id' parameter
+	const [userId, userIdErr] = $(params.user_id).optional.id().$;
+	if (userIdErr) return rej('invalid user_id param');
+
+	// Get 'username' parameter
+	const [username, usernameErr] = $(params.username).optional.string().$;
+	if (usernameErr) return rej('invalid username param');
+
+	// Get 'host' parameter
+	const [host, hostErr] = $(params.host).optional.string().$;
+	if (hostErr) return rej('invalid username param');
+
+	if (userId === undefined && typeof username !== 'string') {
+		return rej('user_id or pair of username and host is required');
+	}
+
+	// Lookup user
+	if (typeof host === 'string') {
+		const username_lower = username.toLowerCase();
+		const host_lower_ascii = toASCII(host).toLowerCase();
+		const host_lower = toUnicode(host_lower_ascii);
+
+		user = await findUser({ username_lower, host_lower });
+
+		if (user === null) {
+			const acct_lower = `${username_lower}@${host_lower_ascii}`;
+			let activityStreams;
+			let finger;
+			let followers_count;
+			let following_count;
+			let likes_count;
+			let posts_count;
+
+			if (!validateUsername(username)) {
+				return rej('username validation failed');
+			}
+
+			try {
+				finger = await webFingerAndVerify(acct_lower, acct_lower);
+			} catch (exception) {
+				return rej('WebFinger lookup failed');
+			}
+
+			const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self');
+			if (!self) {
+				return rej('WebFinger has no reference to self representation');
+			}
+
+			try {
+				activityStreams = await request({
+					url: self.href,
+					headers: {
+						Accept: 'application/activity+json, application/ld+json'
+					},
+					json: true
+				});
+			} catch (exception) {
+				return rej('failed to retrieve ActivityStreams representation');
+			}
+
+			if (!(activityStreams &&
+				(Array.isArray(activityStreams['@context']) ?
+					activityStreams['@context'].includes('https://www.w3.org/ns/activitystreams') :
+					activityStreams['@context'] === 'https://www.w3.org/ns/activitystreams') &&
+				activityStreams.type === 'Person' &&
+				typeof activityStreams.preferredUsername === 'string' &&
+				activityStreams.preferredUsername.toLowerCase() === username_lower &&
+				isValidName(activityStreams.name) &&
+				isValidDescription(activityStreams.summary)
+			)) {
+				return rej('failed ActivityStreams validation');
+			}
+
+			try {
+				[followers_count, following_count, likes_count, posts_count] = await Promise.all([
+					getCollectionCount(activityStreams.followers),
+					getCollectionCount(activityStreams.following),
+					getCollectionCount(activityStreams.liked),
+					getCollectionCount(activityStreams.outbox),
+					webFingerAndVerify(activityStreams.id, acct_lower),
+				]);
+			} catch (exception) {
+				return rej('failed to fetch assets');
+			}
+
+			const summaryDOM = JSDOM.fragment(activityStreams.summary);
+
+			// Create user
+			user = await User.insert({
+				avatar_id: null,
+				banner_id: null,
+				created_at: new Date(),
+				description: summaryDOM.textContent,
+				followers_count,
+				following_count,
+				name: activityStreams.name,
+				posts_count,
+				likes_count,
+				liked_count: 0,
+				drive_capacity: 1073741824, // 1GB
+				username: username,
+				username_lower,
+				host: toUnicode(finger.subject.replace(/^.*?@/, '')),
+				host_lower,
+				account: {
+					uri: activityStreams.id,
+				},
+			});
+
+			const [icon, image] = await Promise.all([
+				activityStreams.icon,
+				activityStreams.image,
+			].map(async image => {
+				if (!image || image.type !== 'Image') {
+					return { _id: null };
+				}
+
+				try {
+					return await uploadFromUrl(image.url, user);
+				} catch (exception) {
+					return { _id: null };
+				}
+			}));
+
+			User.update({ _id: user._id }, {
+				$set: {
+					avatar_id: icon._id,
+					banner_id: image._id,
+				},
+			});
+
+			user.avatar_id = icon._id;
+			user.banner_id = icon._id;
+		}
+	} else {
+		const q = userId !== undefined
+			? { _id: userId }
+			: { username_lower: username.toLowerCase(), host: null };
+
+		user = await findUser(q);
+
+		if (user === null) {
+			return rej('user not found');
+		}
+	}
+
+	// Send response
+	res(await pack(user, me, {
+		detail: true
+	}));
+});