From e891b34d6160ff3e3357e75cbe065812be636982 Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Thu, 7 Sep 2017 13:19:28 +0900
Subject: Rename

---
 src/tools/analysis/core.ts                      |  51 ++++
 src/tools/analysis/extract-user-keywords.ts     |  94 ++++++++
 src/tools/analysis/naive-bayes.js               | 302 ++++++++++++++++++++++++
 src/tools/analysis/predict-all-post-category.ts |  35 +++
 src/tools/analysis/predict-user-interst.ts      |  45 ++++
 5 files changed, 527 insertions(+)
 create mode 100644 src/tools/analysis/core.ts
 create mode 100644 src/tools/analysis/extract-user-keywords.ts
 create mode 100644 src/tools/analysis/naive-bayes.js
 create mode 100644 src/tools/analysis/predict-all-post-category.ts
 create mode 100644 src/tools/analysis/predict-user-interst.ts

(limited to 'src/tools/analysis')

diff --git a/src/tools/analysis/core.ts b/src/tools/analysis/core.ts
new file mode 100644
index 0000000000..5dcce26264
--- /dev/null
+++ b/src/tools/analysis/core.ts
@@ -0,0 +1,51 @@
+const bayes = require('./naive-bayes.js');
+const MeCab = require('mecab-async');
+
+import Post from '../../api/models/post';
+import config from '../../conf';
+
+/**
+ * 投稿を学習したり与えられた投稿のカテゴリを予測します
+ */
+export default class Categorizer {
+	private classifier: any;
+	private mecab: any;
+
+	constructor() {
+		this.mecab = new MeCab();
+		if (config.categorizer.mecab_command) this.mecab.command = config.categorizer.mecab_command;
+
+		// BIND -----------------------------------
+		this.tokenizer = this.tokenizer.bind(this);
+	}
+
+	private tokenizer(text: string) {
+		const tokens = this.mecab.parseSync(text)
+			// 名詞だけに制限
+			.filter(token => token[1] === '名詞')
+			// 取り出し
+			.map(token => token[0]);
+
+		return tokens;
+	}
+
+	public async init() {
+		this.classifier = bayes({
+			tokenizer: this.tokenizer
+		});
+
+		// 訓練データ取得
+		const verifiedPosts = await Post.find({
+			is_category_verified: true
+		});
+
+		// 学習
+		verifiedPosts.forEach(post => {
+			this.classifier.learn(post.text, post.category);
+		});
+	}
+
+	public async predict(text) {
+		return this.classifier.categorize(text);
+	}
+}
diff --git a/src/tools/analysis/extract-user-keywords.ts b/src/tools/analysis/extract-user-keywords.ts
new file mode 100644
index 0000000000..9f21ae2e17
--- /dev/null
+++ b/src/tools/analysis/extract-user-keywords.ts
@@ -0,0 +1,94 @@
+const MeCab = require('mecab-async');
+
+import Post from '../../api/models/post';
+import User from '../../api/models/user';
+import config from '../../conf';
+
+const mecab = new MeCab();
+if (config.categorizer.mecab_command) mecab.command = config.categorizer.mecab_command;
+
+function tokenize(text: string) {
+	const tokens = this.mecab.parseSync(text)
+		// キーワードのみ
+		.filter(token => token[1] == '名詞' && (token[2] == '固有名詞' || token[2] == '一般'))
+		// 取り出し
+		.map(token => token[0]);
+
+	return tokens;
+}
+
+// Fetch all users
+User.find({}, {
+	fields: {
+		_id: true
+	}
+}).then(users => {
+	let i = -1;
+
+	const x = cb => {
+		if (++i == users.length) return cb();
+		extractKeywordsOne(users[i]._id, () => x(cb));
+	};
+
+	x(() => {
+		console.log('complete');
+	});
+});
+
+async function extractKeywordsOne(id, cb) {
+	console.log(`extract keywords of ${id} ...`);
+
+	// Fetch recent posts
+	const recentPosts = await Post.find({
+		user_id: id,
+		text: {
+			$exists: true
+		}
+	}, {
+		sort: {
+			_id: -1
+		},
+		limit: 1000,
+		fields: {
+			_id: false,
+			text: true
+		}
+	});
+
+	// 投稿が少なかったら中断
+	if (recentPosts.length < 10) {
+		return cb();
+	}
+
+	const keywords = {};
+
+	// Extract keywords from recent posts
+	recentPosts.forEach(post => {
+		const keywordsOfPost = tokenize(post.text);
+
+		keywordsOfPost.forEach(keyword => {
+			if (keywords[keyword]) {
+				keywords[keyword]++;
+			} else {
+				keywords[keyword] = 1;
+			}
+		});
+	});
+
+	// Sort keywords by frequency
+	const keywordsSorted = Object.keys(keywords).sort((a, b) => keywords[b] - keywords[a]);
+
+	// Lookup top 10 keywords
+	const topKeywords = keywordsSorted.slice(0, 10);
+
+	process.stdout.write(' >>> ' + topKeywords.join(' '));
+
+	// Save
+	User.update({ _id: id }, {
+		$set: {
+			keywords: topKeywords
+		}
+	}).then(() => {
+		cb();
+	});
+}
diff --git a/src/tools/analysis/naive-bayes.js b/src/tools/analysis/naive-bayes.js
new file mode 100644
index 0000000000..78f07153cf
--- /dev/null
+++ b/src/tools/analysis/naive-bayes.js
@@ -0,0 +1,302 @@
+// Original source code: https://github.com/ttezel/bayes/blob/master/lib/naive_bayes.js (commit: 2c20d3066e4fc786400aaedcf3e42987e52abe3c)
+// CUSTOMIZED BY SYUILO
+
+/*
+		Expose our naive-bayes generator function
+*/
+module.exports = function (options) {
+	return new Naivebayes(options)
+}
+
+// keys we use to serialize a classifier's state
+var STATE_KEYS = module.exports.STATE_KEYS = [
+	'categories', 'docCount', 'totalDocuments', 'vocabulary', 'vocabularySize',
+	'wordCount', 'wordFrequencyCount', 'options'
+];
+
+/**
+ * Initializes a NaiveBayes instance from a JSON state representation.
+ * Use this with classifier.toJson().
+ *
+ * @param  {String} jsonStr   state representation obtained by classifier.toJson()
+ * @return {NaiveBayes}       Classifier
+ */
+module.exports.fromJson = function (jsonStr) {
+	var parsed;
+	try {
+		parsed = JSON.parse(jsonStr)
+	} catch (e) {
+		throw new Error('Naivebayes.fromJson expects a valid JSON string.')
+	}
+	// init a new classifier
+	var classifier = new Naivebayes(parsed.options)
+
+	// override the classifier's state
+	STATE_KEYS.forEach(function (k) {
+		if (!parsed[k]) {
+			throw new Error('Naivebayes.fromJson: JSON string is missing an expected property: `'+k+'`.')
+		}
+		classifier[k] = parsed[k]
+	})
+
+	return classifier
+}
+
+/**
+ * Given an input string, tokenize it into an array of word tokens.
+ * This is the default tokenization function used if user does not provide one in `options`.
+ *
+ * @param  {String} text
+ * @return {Array}
+ */
+var defaultTokenizer = function (text) {
+	//remove punctuation from text - remove anything that isn't a word char or a space
+	var rgxPunctuation = /[^(a-zA-ZA-Яa-я0-9_)+\s]/g
+
+	var sanitized = text.replace(rgxPunctuation, ' ')
+
+	return sanitized.split(/\s+/)
+}
+
+/**
+ * Naive-Bayes Classifier
+ *
+ * This is a naive-bayes classifier that uses Laplace Smoothing.
+ *
+ * Takes an (optional) options object containing:
+ *   - `tokenizer`  => custom tokenization function
+ *
+ */
+function Naivebayes (options) {
+	// set options object
+	this.options = {}
+	if (typeof options !== 'undefined') {
+		if (!options || typeof options !== 'object' || Array.isArray(options)) {
+			throw TypeError('NaiveBayes got invalid `options`: `' + options + '`. Pass in an object.')
+		}
+		this.options = options
+	}
+
+	this.tokenizer = this.options.tokenizer || defaultTokenizer
+
+	//initialize our vocabulary and its size
+	this.vocabulary = {}
+	this.vocabularySize = 0
+
+	//number of documents we have learned from
+	this.totalDocuments = 0
+
+	//document frequency table for each of our categories
+	//=> for each category, how often were documents mapped to it
+	this.docCount = {}
+
+	//for each category, how many words total were mapped to it
+	this.wordCount = {}
+
+	//word frequency table for each category
+	//=> for each category, how frequent was a given word mapped to it
+	this.wordFrequencyCount = {}
+
+	//hashmap of our category names
+	this.categories = {}
+}
+
+/**
+ * Initialize each of our data structure entries for this new category
+ *
+ * @param  {String} categoryName
+ */
+Naivebayes.prototype.initializeCategory = function (categoryName) {
+	if (!this.categories[categoryName]) {
+		this.docCount[categoryName] = 0
+		this.wordCount[categoryName] = 0
+		this.wordFrequencyCount[categoryName] = {}
+		this.categories[categoryName] = true
+	}
+	return this
+}
+
+/**
+ * train our naive-bayes classifier by telling it what `category`
+ * the `text` corresponds to.
+ *
+ * @param  {String} text
+ * @param  {String} class
+ */
+Naivebayes.prototype.learn = function (text, category) {
+	var self = this
+
+	//initialize category data structures if we've never seen this category
+	self.initializeCategory(category)
+
+	//update our count of how many documents mapped to this category
+	self.docCount[category]++
+
+	//update the total number of documents we have learned from
+	self.totalDocuments++
+
+	//normalize the text into a word array
+	var tokens = self.tokenizer(text)
+
+	//get a frequency count for each token in the text
+	var frequencyTable = self.frequencyTable(tokens)
+
+	/*
+			Update our vocabulary and our word frequency count for this category
+	*/
+
+	Object
+	.keys(frequencyTable)
+	.forEach(function (token) {
+		//add this word to our vocabulary if not already existing
+		if (!self.vocabulary[token]) {
+			self.vocabulary[token] = true
+			self.vocabularySize++
+		}
+
+		var frequencyInText = frequencyTable[token]
+
+		//update the frequency information for this word in this category
+		if (!self.wordFrequencyCount[category][token])
+			self.wordFrequencyCount[category][token] = frequencyInText
+		else
+			self.wordFrequencyCount[category][token] += frequencyInText
+
+		//update the count of all words we have seen mapped to this category
+		self.wordCount[category] += frequencyInText
+	})
+
+	return self
+}
+
+/**
+ * Determine what category `text` belongs to.
+ *
+ * @param  {String} text
+ * @return {String} category
+ */
+Naivebayes.prototype.categorize = function (text) {
+	var self = this
+		, maxProbability = -Infinity
+		, chosenCategory = null
+
+	var tokens = self.tokenizer(text)
+	var frequencyTable = self.frequencyTable(tokens)
+
+	//iterate thru our categories to find the one with max probability for this text
+	Object
+	.keys(self.categories)
+	.forEach(function (category) {
+
+		//start by calculating the overall probability of this category
+		//=>  out of all documents we've ever looked at, how many were
+		//    mapped to this category
+		var categoryProbability = self.docCount[category] / self.totalDocuments
+
+		//take the log to avoid underflow
+		var logProbability = Math.log(categoryProbability)
+
+		//now determine P( w | c ) for each word `w` in the text
+		Object
+		.keys(frequencyTable)
+		.forEach(function (token) {
+			var frequencyInText = frequencyTable[token]
+			var tokenProbability = self.tokenProbability(token, category)
+
+			// console.log('token: %s category: `%s` tokenProbability: %d', token, category, tokenProbability)
+
+			//determine the log of the P( w | c ) for this word
+			logProbability += frequencyInText * Math.log(tokenProbability)
+		})
+
+		if (logProbability > maxProbability) {
+			maxProbability = logProbability
+			chosenCategory = category
+		}
+	})
+
+	return chosenCategory
+}
+
+/**
+ * Calculate probability that a `token` belongs to a `category`
+ *
+ * @param  {String} token
+ * @param  {String} category
+ * @return {Number} probability
+ */
+Naivebayes.prototype.tokenProbability = function (token, category) {
+	//how many times this word has occurred in documents mapped to this category
+	var wordFrequencyCount = this.wordFrequencyCount[category][token] || 0
+
+	//what is the count of all words that have ever been mapped to this category
+	var wordCount = this.wordCount[category]
+
+	//use laplace Add-1 Smoothing equation
+	return ( wordFrequencyCount + 1 ) / ( wordCount + this.vocabularySize )
+}
+
+/**
+ * Build a frequency hashmap where
+ * - the keys are the entries in `tokens`
+ * - the values are the frequency of each entry in `tokens`
+ *
+ * @param  {Array} tokens  Normalized word array
+ * @return {Object}
+ */
+Naivebayes.prototype.frequencyTable = function (tokens) {
+	var frequencyTable = Object.create(null)
+
+	tokens.forEach(function (token) {
+		if (!frequencyTable[token])
+			frequencyTable[token] = 1
+		else
+			frequencyTable[token]++
+	})
+
+	return frequencyTable
+}
+
+/**
+ * Dump the classifier's state as a JSON string.
+ * @return {String} Representation of the classifier.
+ */
+Naivebayes.prototype.toJson = function () {
+	var state = {}
+	var self = this
+	STATE_KEYS.forEach(function (k) {
+		state[k] = self[k]
+	})
+
+	var jsonStr = JSON.stringify(state)
+
+	return jsonStr
+}
+
+// (original method)
+Naivebayes.prototype.export = function () {
+	var state = {}
+	var self = this
+	STATE_KEYS.forEach(function (k) {
+		state[k] = self[k]
+	})
+
+	return state
+}
+
+module.exports.import = function (data) {
+	var parsed = data
+
+	// init a new classifier
+	var classifier = new Naivebayes()
+
+	// override the classifier's state
+	STATE_KEYS.forEach(function (k) {
+		if (!parsed[k]) {
+			throw new Error('Naivebayes.import: data is missing an expected property: `'+k+'`.')
+		}
+		classifier[k] = parsed[k]
+	})
+
+	return classifier
+}
diff --git a/src/tools/analysis/predict-all-post-category.ts b/src/tools/analysis/predict-all-post-category.ts
new file mode 100644
index 0000000000..058c4f99ef
--- /dev/null
+++ b/src/tools/analysis/predict-all-post-category.ts
@@ -0,0 +1,35 @@
+import Post from '../../api/models/post';
+import Core from './core';
+
+const c = new Core();
+
+c.init().then(() => {
+	// 全ての(人間によって証明されていない)投稿を取得
+	Post.find({
+		text: {
+			$exists: true
+		},
+		is_category_verified: {
+			$ne: true
+		}
+	}, {
+		sort: {
+			_id: -1
+		},
+		fields: {
+			_id: true,
+			text: true
+		}
+	}).then(posts => {
+		posts.forEach(post => {
+			console.log(`predicting... ${post._id}`);
+			const category = c.predict(post.text);
+
+			Post.update({ _id: post._id }, {
+				$set: {
+					category: category
+				}
+			});
+		});
+	});
+});
diff --git a/src/tools/analysis/predict-user-interst.ts b/src/tools/analysis/predict-user-interst.ts
new file mode 100644
index 0000000000..99bdfa4206
--- /dev/null
+++ b/src/tools/analysis/predict-user-interst.ts
@@ -0,0 +1,45 @@
+import Post from '../../api/models/post';
+import User from '../../api/models/user';
+
+export async function predictOne(id) {
+	console.log(`predict interest of ${id} ...`);
+
+	// TODO: repostなども含める
+	const recentPosts = await Post.find({
+		user_id: id,
+		category: {
+			$exists: true
+		}
+	}, {
+		sort: {
+			_id: -1
+		},
+		limit: 1000,
+		fields: {
+			_id: false,
+			category: true
+		}
+	});
+
+	const categories = {};
+
+	recentPosts.forEach(post => {
+		if (categories[post.category]) {
+			categories[post.category]++;
+		} else {
+			categories[post.category] = 1;
+		}
+	});
+}
+
+export async function predictAll() {
+	const allUsers = await User.find({}, {
+		fields: {
+			_id: true
+		}
+	});
+
+	allUsers.forEach(user => {
+		predictOne(user._id);
+	});
+}
-- 
cgit v1.2.3-freya


From 1ed468911ed8a268c38362218564050217ede07f Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Thu, 7 Sep 2017 13:31:31 +0900
Subject: Rename

---
 src/config.ts                               | 2 +-
 src/tools/analysis/extract-user-keywords.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'src/tools/analysis')

diff --git a/src/config.ts b/src/config.ts
index f333a1f5a9..f8facdee2e 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -68,7 +68,7 @@ type Source = {
 		hook_secret: string;
 		username: string;
 	};
-	categorizer?: {
+	analysis?: {
 		mecab_command?: string;
 	};
 };
diff --git a/src/tools/analysis/extract-user-keywords.ts b/src/tools/analysis/extract-user-keywords.ts
index 9f21ae2e17..49fc9a5c9b 100644
--- a/src/tools/analysis/extract-user-keywords.ts
+++ b/src/tools/analysis/extract-user-keywords.ts
@@ -5,7 +5,7 @@ import User from '../../api/models/user';
 import config from '../../conf';
 
 const mecab = new MeCab();
-if (config.categorizer.mecab_command) mecab.command = config.categorizer.mecab_command;
+if (config.analysis.mecab_command) mecab.command = config.analysis.mecab_command;
 
 function tokenize(text: string) {
 	const tokens = this.mecab.parseSync(text)
-- 
cgit v1.2.3-freya


From 5b36fe0e3af3eb993cac0b577b2812ec3654313f Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Thu, 7 Sep 2017 13:32:07 +0900
Subject: Fix bug

---
 src/tools/analysis/extract-user-keywords.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/tools/analysis')

diff --git a/src/tools/analysis/extract-user-keywords.ts b/src/tools/analysis/extract-user-keywords.ts
index 49fc9a5c9b..d34bb7cd6f 100644
--- a/src/tools/analysis/extract-user-keywords.ts
+++ b/src/tools/analysis/extract-user-keywords.ts
@@ -8,7 +8,7 @@ const mecab = new MeCab();
 if (config.analysis.mecab_command) mecab.command = config.analysis.mecab_command;
 
 function tokenize(text: string) {
-	const tokens = this.mecab.parseSync(text)
+	const tokens = mecab.parseSync(text)
 		// キーワードのみ
 		.filter(token => token[1] == '名詞' && (token[2] == '固有名詞' || token[2] == '一般'))
 		// 取り出し
-- 
cgit v1.2.3-freya


From af0dd2622412a1e55b68f52628c71b61d5adf1aa Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Thu, 7 Sep 2017 13:33:39 +0900
Subject: :v:

---
 src/tools/analysis/extract-user-keywords.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/tools/analysis')

diff --git a/src/tools/analysis/extract-user-keywords.ts b/src/tools/analysis/extract-user-keywords.ts
index d34bb7cd6f..9ef7784390 100644
--- a/src/tools/analysis/extract-user-keywords.ts
+++ b/src/tools/analysis/extract-user-keywords.ts
@@ -36,7 +36,7 @@ User.find({}, {
 });
 
 async function extractKeywordsOne(id, cb) {
-	console.log(`extract keywords of ${id} ...`);
+	console.log(`extracting keywords of ${id} ...`);
 
 	// Fetch recent posts
 	const recentPosts = await Post.find({
-- 
cgit v1.2.3-freya


From 1e3f93d68ee5db3739084039be57279c14d0a62b Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Thu, 7 Sep 2017 14:52:07 +0900
Subject: :v:

---
 src/tools/analysis/core.ts                  |  4 +-
 src/tools/analysis/extract-user-keywords.ts | 59 ++++++++++++++++----
 src/tools/analysis/mecab.js                 | 85 +++++++++++++++++++++++++++++
 3 files changed, 135 insertions(+), 13 deletions(-)
 create mode 100644 src/tools/analysis/mecab.js

(limited to 'src/tools/analysis')

diff --git a/src/tools/analysis/core.ts b/src/tools/analysis/core.ts
index 5dcce26264..20e5fa6c51 100644
--- a/src/tools/analysis/core.ts
+++ b/src/tools/analysis/core.ts
@@ -1,8 +1,7 @@
 const bayes = require('./naive-bayes.js');
-const MeCab = require('mecab-async');
 
+const MeCab = require('./mecab');
 import Post from '../../api/models/post';
-import config from '../../conf';
 
 /**
  * 投稿を学習したり与えられた投稿のカテゴリを予測します
@@ -13,7 +12,6 @@ export default class Categorizer {
 
 	constructor() {
 		this.mecab = new MeCab();
-		if (config.categorizer.mecab_command) this.mecab.command = config.categorizer.mecab_command;
 
 		// BIND -----------------------------------
 		this.tokenizer = this.tokenizer.bind(this);
diff --git a/src/tools/analysis/extract-user-keywords.ts b/src/tools/analysis/extract-user-keywords.ts
index 9ef7784390..5251a0d1d0 100644
--- a/src/tools/analysis/extract-user-keywords.ts
+++ b/src/tools/analysis/extract-user-keywords.ts
@@ -1,18 +1,56 @@
-const MeCab = require('mecab-async');
-
+const MeCab = require('./mecab');
 import Post from '../../api/models/post';
 import User from '../../api/models/user';
-import config from '../../conf';
+import parse from '../../api/common/text';
+
+const stopwords = [
+	'ー',
+
+	'の', 'に', 'は', 'を', 'た', 'が', 'で', 'て', 'と', 'し', 'れ', 'さ',
+  'ある', 'いる', 'も', 'する', 'から', 'な', 'こと', 'として', 'い', 'や', 'れる',
+  'など', 'なっ', 'ない', 'この', 'ため', 'その', 'あっ', 'よう', 'また', 'もの',
+  'という', 'あり', 'まで', 'られ', 'なる', 'へ', 'か', 'だ', 'これ', 'によって',
+  'により', 'おり', 'より', 'による', 'ず', 'なり', 'られる', 'において', 'ば', 'なかっ',
+  'なく', 'しかし', 'について', 'せ', 'だっ', 'その後', 'できる', 'それ', 'う', 'ので',
+  'なお', 'のみ', 'でき', 'き', 'つ', 'における', 'および', 'いう', 'さらに', 'でも',
+  'ら', 'たり', 'その他', 'に関する', 'たち', 'ます', 'ん', 'なら', 'に対して', '特に',
+  'せる', '及び', 'これら', 'とき', 'では', 'にて', 'ほか', 'ながら', 'うち', 'そして',
+  'とともに', 'ただし', 'かつて', 'それぞれ', 'または', 'お', 'ほど', 'ものの', 'に対する',
+  'ほとんど', 'と共に', 'といった', 'です', 'とも', 'ところ', 'ここ', '感じ', '気持ち',
+
+	'about', 'after', 'all', 'also', 'am', 'an', 'and', 'another', 'any', 'are', 'as', 'at', 'be',
+  'because', 'been', 'before', 'being', 'between', 'both', 'but', 'by', 'came', 'can',
+  'come', 'could', 'did', 'do', 'each', 'for', 'from', 'get', 'got', 'has', 'had',
+  'he', 'have', 'her', 'here', 'him', 'himself', 'his', 'how', 'if', 'in', 'into',
+  'is', 'it', 'like', 'make', 'many', 'me', 'might', 'more', 'most', 'much', 'must',
+  'my', 'never', 'now', 'of', 'on', 'only', 'or', 'other', 'our', 'out', 'over',
+  'said', 'same', 'see', 'should', 'since', 'some', 'still', 'such', 'take', 'than',
+  'that', 'the', 'their', 'them', 'then', 'there', 'these', 'they', 'this', 'those',
+  'through', 'to', 'too', 'under', 'up', 'very', 'was', 'way', 'we', 'well', 'were',
+	'what', 'where', 'which', 'while', 'who', 'with', 'would', 'you', 'your', 'a', 'i'
+];
 
 const mecab = new MeCab();
-if (config.analysis.mecab_command) mecab.command = config.analysis.mecab_command;
 
 function tokenize(text: string) {
-	const tokens = mecab.parseSync(text)
+	if (text == null) return [];
+
+	// パース
+	const ast = parse(text);
+
+	const plain = ast
+		// テキストのみ(URLなどを除外するという意)
+		.filter(t => t.type == 'text' || t.type == 'bold')
+		.map(t => t.content)
+		.join('');
+
+	const tokens = mecab.parseSync(plain)
 		// キーワードのみ
 		.filter(token => token[1] == '名詞' && (token[2] == '固有名詞' || token[2] == '一般'))
 		// 取り出し
-		.map(token => token[0]);
+		.map(token => token[0].toLowerCase())
+		// ストップワード
+		.filter(word => stopwords.indexOf(word) === -1 && word.length > 1);
 
 	return tokens;
 }
@@ -36,7 +74,7 @@ User.find({}, {
 });
 
 async function extractKeywordsOne(id, cb) {
-	console.log(`extracting keywords of ${id} ...`);
+	process.stdout.write(`extracting keywords of ${id} ...`);
 
 	// Fetch recent posts
 	const recentPosts = await Post.find({
@@ -48,7 +86,7 @@ async function extractKeywordsOne(id, cb) {
 		sort: {
 			_id: -1
 		},
-		limit: 1000,
+		limit: 10000,
 		fields: {
 			_id: false,
 			text: true
@@ -56,7 +94,8 @@ async function extractKeywordsOne(id, cb) {
 	});
 
 	// 投稿が少なかったら中断
-	if (recentPosts.length < 10) {
+	if (recentPosts.length < 300) {
+		process.stdout.write(' >>> -\n');
 		return cb();
 	}
 
@@ -81,7 +120,7 @@ async function extractKeywordsOne(id, cb) {
 	// Lookup top 10 keywords
 	const topKeywords = keywordsSorted.slice(0, 10);
 
-	process.stdout.write(' >>> ' + topKeywords.join(' '));
+	process.stdout.write(' >>> ' + topKeywords.join(', ') + '\n');
 
 	// Save
 	User.update({ _id: id }, {
diff --git a/src/tools/analysis/mecab.js b/src/tools/analysis/mecab.js
new file mode 100644
index 0000000000..82f7d6d529
--- /dev/null
+++ b/src/tools/analysis/mecab.js
@@ -0,0 +1,85 @@
+// Original source code: https://github.com/hecomi/node-mecab-async
+// CUSTOMIZED BY SYUILO
+
+var exec     = require('child_process').exec;
+var execSync = require('child_process').execSync;
+var sq       = require('shell-quote');
+
+const config = require('../../conf').default;
+
+// for backward compatibility
+var MeCab = function() {};
+
+MeCab.prototype = {
+    command : config.analysis.mecab_command ? config.analysis.mecab_command : 'mecab',
+    _format: function(arrayResult) {
+        var result = [];
+        if (!arrayResult) { return result; }
+        // Reference: http://mecab.googlecode.com/svn/trunk/mecab/doc/index.html
+        // 表層形\t品詞,品詞細分類1,品詞細分類2,品詞細分類3,活用形,活用型,原形,読み,発音
+        arrayResult.forEach(function(parsed) {
+            if (parsed.length <= 8) { return; }
+            result.push({
+                kanji         : parsed[0],
+                lexical       : parsed[1],
+                compound      : parsed[2],
+                compound2     : parsed[3],
+                compound3     : parsed[4],
+                conjugation   : parsed[5],
+                inflection    : parsed[6],
+                original      : parsed[7],
+                reading       : parsed[8],
+                pronunciation : parsed[9] || ''
+            });
+        });
+        return result;
+    },
+    _shellCommand : function(str) {
+        return sq.quote(['echo', str]) + ' | ' + this.command;
+    },
+    _parseMeCabResult : function(result) {
+        return result.split('\n').map(function(line) {
+            return line.replace('\t', ',').split(',');
+        });
+    },
+    parse : function(str, callback) {
+        process.nextTick(function() { // for bug
+            exec(MeCab._shellCommand(str), function(err, result) {
+                if (err) { return callback(err); }
+                callback(err, MeCab._parseMeCabResult(result).slice(0,-2));
+            });
+        });
+    },
+    parseSync : function(str) {
+        var result = execSync(MeCab._shellCommand(str));
+        return MeCab._parseMeCabResult(String(result)).slice(0, -2);
+    },
+    parseFormat : function(str, callback) {
+        MeCab.parse(str, function(err, result) {
+            if (err) { return callback(err); }
+            callback(err, MeCab._format(result));
+        });
+    },
+    parseSyncFormat : function(str) {
+        return MeCab._format(MeCab.parseSync(str));
+    },
+    _wakatsu : function(arr) {
+        return arr.map(function(data) { return data[0]; });
+    },
+    wakachi : function(str, callback) {
+        MeCab.parse(str, function(err, arr) {
+            if (err) { return callback(err); }
+            callback(null, MeCab._wakatsu(arr));
+        });
+    },
+    wakachiSync : function(str) {
+        var arr = MeCab.parseSync(str);
+        return MeCab._wakatsu(arr);
+    }
+};
+
+for (var x in MeCab.prototype) {
+    MeCab[x] = MeCab.prototype[x];
+}
+
+module.exports = MeCab;
-- 
cgit v1.2.3-freya


From 9a0a2fdd854ba2a0ca0ce29cbdae4a1a29b2f5ba Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Thu, 7 Sep 2017 16:15:40 +0900
Subject: :v:

---
 package.json                                |   1 +
 src/tools/analysis/extract-user-keywords.ts | 125 ++++++++++++++++------------
 2 files changed, 74 insertions(+), 52 deletions(-)

(limited to 'src/tools/analysis')

diff --git a/package.json b/package.json
index 2e37f647ad..726956140d 100644
--- a/package.json
+++ b/package.json
@@ -121,6 +121,7 @@
     "is-url": "1.2.2",
     "js-yaml": "3.9.1",
     "mecab-async": "^0.1.0",
+    "moji": "^0.5.1",
     "mongodb": "2.2.31",
     "monk": "6.0.3",
     "morgan": "1.8.2",
diff --git a/src/tools/analysis/extract-user-keywords.ts b/src/tools/analysis/extract-user-keywords.ts
index 5251a0d1d0..38a9073f7b 100644
--- a/src/tools/analysis/extract-user-keywords.ts
+++ b/src/tools/analysis/extract-user-keywords.ts
@@ -1,8 +1,12 @@
+const moji = require('moji');
+
 const MeCab = require('./mecab');
 import Post from '../../api/models/post';
 import User from '../../api/models/user';
 import parse from '../../api/common/text';
 
+process.on('unhandledRejection', console.dir);
+
 const stopwords = [
 	'ー',
 
@@ -16,7 +20,8 @@ const stopwords = [
   'ら', 'たり', 'その他', 'に関する', 'たち', 'ます', 'ん', 'なら', 'に対して', '特に',
   'せる', '及び', 'これら', 'とき', 'では', 'にて', 'ほか', 'ながら', 'うち', 'そして',
   'とともに', 'ただし', 'かつて', 'それぞれ', 'または', 'お', 'ほど', 'ものの', 'に対する',
-  'ほとんど', 'と共に', 'といった', 'です', 'とも', 'ところ', 'ここ', '感じ', '気持ち',
+	'ほとんど', 'と共に', 'といった', 'です', 'とも', 'ところ', 'ここ', '感じ', '気持ち',
+	'あと', '自分',
 
 	'about', 'after', 'all', 'also', 'am', 'an', 'and', 'another', 'any', 'are', 'as', 'at', 'be',
   'because', 'been', 'before', 'being', 'between', 'both', 'but', 'by', 'came', 'can',
@@ -47,10 +52,16 @@ function tokenize(text: string) {
 	const tokens = mecab.parseSync(plain)
 		// キーワードのみ
 		.filter(token => token[1] == '名詞' && (token[2] == '固有名詞' || token[2] == '一般'))
-		// 取り出し
-		.map(token => token[0].toLowerCase())
-		// ストップワード
-		.filter(word => stopwords.indexOf(word) === -1 && word.length > 1);
+		// 取り出し(&整形(全角を半角にしたり大文字を小文字で統一したり))
+		.map(token => moji(token[0]).convert('ZE', 'HE').convert('HK', 'ZK').toString().toLowerCase())
+		// ストップワードなど
+		.filter(word =>
+			stopwords.indexOf(word) === -1 &&
+			word.length > 1 &&
+			word.indexOf('！') === -1 &&
+			word.indexOf('!') === -1 &&
+			word.indexOf('？') === -1 &&
+			word.indexOf('?') === -1);
 
 	return tokens;
 }
@@ -65,7 +76,13 @@ User.find({}, {
 
 	const x = cb => {
 		if (++i == users.length) return cb();
-		extractKeywordsOne(users[i]._id, () => x(cb));
+		extractKeywordsOne(users[i]._id).then(() => x(cb), err => {
+			console.error(err);
+			setTimeout(() => {
+				i--;
+				x(cb);
+			}, 1000);
+		});
 	};
 
 	x(() => {
@@ -73,61 +90,65 @@ User.find({}, {
 	});
 });
 
-async function extractKeywordsOne(id, cb) {
-	process.stdout.write(`extracting keywords of ${id} ...`);
+function extractKeywordsOne(id) {
+	return new Promise(async (resolve, reject) => {
+		process.stdout.write(`extracting keywords of ${id} ...`);
 
-	// Fetch recent posts
-	const recentPosts = await Post.find({
-		user_id: id,
-		text: {
-			$exists: true
-		}
-	}, {
-		sort: {
-			_id: -1
-		},
-		limit: 10000,
-		fields: {
-			_id: false,
-			text: true
-		}
-	});
+		// Fetch recent posts
+		const recentPosts = await Post.find({
+			user_id: id,
+			text: {
+				$exists: true
+			}
+		}, {
+			sort: {
+				_id: -1
+			},
+			limit: 10000,
+			fields: {
+				_id: false,
+				text: true
+			}
+		});
 
-	// 投稿が少なかったら中断
-	if (recentPosts.length < 300) {
-		process.stdout.write(' >>> -\n');
-		return cb();
-	}
+		// 投稿が少なかったら中断
+		if (recentPosts.length < 300) {
+			process.stdout.write(' >>> -\n');
+			return resolve();
+		}
 
-	const keywords = {};
+		const keywords = {};
 
-	// Extract keywords from recent posts
-	recentPosts.forEach(post => {
-		const keywordsOfPost = tokenize(post.text);
+		// Extract keywords from recent posts
+		recentPosts.forEach(post => {
+			const keywordsOfPost = tokenize(post.text);
 
-		keywordsOfPost.forEach(keyword => {
-			if (keywords[keyword]) {
-				keywords[keyword]++;
-			} else {
-				keywords[keyword] = 1;
-			}
+			keywordsOfPost.forEach(keyword => {
+				if (keywords[keyword]) {
+					keywords[keyword]++;
+				} else {
+					keywords[keyword] = 1;
+				}
+			});
 		});
-	});
 
-	// Sort keywords by frequency
-	const keywordsSorted = Object.keys(keywords).sort((a, b) => keywords[b] - keywords[a]);
+		// Sort keywords by frequency
+		const keywordsSorted = Object.keys(keywords).sort((a, b) => keywords[b] - keywords[a]);
 
-	// Lookup top 10 keywords
-	const topKeywords = keywordsSorted.slice(0, 10);
+		// Lookup top 10 keywords
+		const topKeywords = keywordsSorted.slice(0, 10);
 
-	process.stdout.write(' >>> ' + topKeywords.join(', ') + '\n');
+		process.stdout.write(' >>> ' + topKeywords.join(', ') + '\n');
 
-	// Save
-	User.update({ _id: id }, {
-		$set: {
-			keywords: topKeywords
-		}
-	}).then(() => {
-		cb();
+		// Save
+		User.update({ _id: id }, {
+			$set: {
+				keywords: topKeywords
+			}
+		}).then(() => {
+			resolve();
+		}, err => {
+			reject(err);
+		});
 	});
 }
-- 
cgit v1.2.3-freya


From 0f6b78df5804a41e4616383e7131605355411ca7 Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Thu, 7 Sep 2017 18:15:24 +0900
Subject: :v:

---
 src/tools/analysis/extract-user-keywords.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/tools/analysis')

diff --git a/src/tools/analysis/extract-user-keywords.ts b/src/tools/analysis/extract-user-keywords.ts
index 38a9073f7b..b99ca93211 100644
--- a/src/tools/analysis/extract-user-keywords.ts
+++ b/src/tools/analysis/extract-user-keywords.ts
@@ -21,7 +21,7 @@ const stopwords = [
   'せる', '及び', 'これら', 'とき', 'では', 'にて', 'ほか', 'ながら', 'うち', 'そして',
   'とともに', 'ただし', 'かつて', 'それぞれ', 'または', 'お', 'ほど', 'ものの', 'に対する',
 	'ほとんど', 'と共に', 'といった', 'です', 'とも', 'ところ', 'ここ', '感じ', '気持ち',
-	'あと', '自分',
+	'あと', '自分', 'すき', '()',
 
 	'about', 'after', 'all', 'also', 'am', 'an', 'and', 'another', 'any', 'are', 'as', 'at', 'be',
   'because', 'been', 'before', 'being', 'between', 'both', 'but', 'by', 'came', 'can',
-- 
cgit v1.2.3-freya


From 5c37b9cef307bf795c4bc49dc7ab21de0c8ba97b Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Fri, 8 Sep 2017 22:10:25 +0900
Subject: Implement #771

---
 CHANGELOG.md                               |   4 +
 locales/en.yml                             |   4 +
 locales/ja.yml                             |   4 +
 src/tools/analysis/extract-user-domains.ts | 120 +++++++++++++++++++++++++++++
 src/web/app/mobile/tags/user.tag           |  40 ++++++++++
 5 files changed, 172 insertions(+)
 create mode 100644 src/tools/analysis/extract-user-domains.ts

(limited to 'src/tools/analysis')

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fc7e2c7db8..b071274d36 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@ ChangeLog (Release Notes)
 =========================
 主に notable な changes を書いていきます
 
+unreleased
+----------
+* New: ユーザーページによく使うドメインを表示 (#771)
+
 2566 (2017/09/07)
 -----------------
 * New: 投稿することの多いキーワードをユーザーページに表示する (#768)
diff --git a/locales/en.yml b/locales/en.yml
index 05b4663c16..fd07be2aed 100644
--- a/locales/en.yml
+++ b/locales/en.yml
@@ -498,6 +498,7 @@ mobile:
       images: "Images"
       activity: "Activity"
       keywords: "Keywords"
+      domains: "Domains"
       followers-you-know: "Followers you know"
       last-used-at: "Latest used at"
 
@@ -512,6 +513,9 @@ mobile:
     mk-user-overview-keywords:
       no-keywords: "No keywords"
 
+    mk-user-overview-domains:
+      no-domains: "No domains"
+
     mk-user-overview-followers-you-know:
       loading: "Loading"
       no-users: "No users"
diff --git a/locales/ja.yml b/locales/ja.yml
index 86da082723..832390f5fe 100644
--- a/locales/ja.yml
+++ b/locales/ja.yml
@@ -498,6 +498,7 @@ mobile:
       images: "画像"
       activity: "アクティビティ"
       keywords: "キーワード"
+      domains: "頻出ドメイン"
       followers-you-know: "知り合いのフォロワー"
       last-used-at: "最終ログイン"
 
@@ -512,6 +513,9 @@ mobile:
     mk-user-overview-keywords:
       no-keywords: "キーワードはありません(十分な数の投稿をしていない可能性があります)"
 
+    mk-user-overview-domains:
+      no-domains: "よく表れるドメインは検出されませんでした"
+
     mk-user-overview-followers-you-know:
       loading: "読み込み中"
       no-users: "知り合いのユーザーはいません"
diff --git a/src/tools/analysis/extract-user-domains.ts b/src/tools/analysis/extract-user-domains.ts
new file mode 100644
index 0000000000..bc120f5c17
--- /dev/null
+++ b/src/tools/analysis/extract-user-domains.ts
@@ -0,0 +1,120 @@
+import * as URL from 'url';
+
+import Post from '../../api/models/post';
+import User from '../../api/models/user';
+import parse from '../../api/common/text';
+
+process.on('unhandledRejection', console.dir);
+
+function tokenize(text: string) {
+	if (text == null) return [];
+
+	// パース
+	const ast = parse(text);
+
+	const domains = ast
+		// URLを抽出
+		.filter(t => t.type == 'url' || t.type == 'link')
+		.map(t => URL.parse(t.url).hostname);
+
+	return domains;
+}
+
+// Fetch all users
+User.find({}, {
+	fields: {
+		_id: true
+	}
+}).then(users => {
+	let i = -1;
+
+	const x = cb => {
+		if (++i == users.length) return cb();
+		extractDomainsOne(users[i]._id).then(() => x(cb), err => {
+			console.error(err);
+			setTimeout(() => {
+				i--;
+				x(cb);
+			}, 1000);
+		});
+	};
+
+	x(() => {
+		console.log('complete');
+	});
+});
+
+function extractDomainsOne(id) {
+	return new Promise(async (resolve, reject) => {
+		process.stdout.write(`extracting domains of ${id} ...`);
+
+		// Fetch recent posts
+		const recentPosts = await Post.find({
+			user_id: id,
+			text: {
+				$exists: true
+			}
+		}, {
+			sort: {
+				_id: -1
+			},
+			limit: 10000,
+			fields: {
+				_id: false,
+				text: true
+			}
+		});
+
+		// 投稿が少なかったら中断
+		if (recentPosts.length < 100) {
+			process.stdout.write(' >>> -\n');
+			return resolve();
+		}
+
+		const domains = {};
+
+		// Extract domains from recent posts
+		recentPosts.forEach(post => {
+			const domainsOfPost = tokenize(post.text);
+
+			domainsOfPost.forEach(domain => {
+				if (domains[domain]) {
+					domains[domain]++;
+				} else {
+					domains[domain] = 1;
+				}
+			});
+		});
+
+		// Calc peak
+		let peak = 0;
+		Object.keys(domains).forEach(domain => {
+			if (domains[domain] > peak) peak = domains[domain];
+		});
+
+		// Sort domains by frequency
+		const domainsSorted = Object.keys(domains).sort((a, b) => domains[b] - domains[a]);
+
+		// Lookup top 10 domains
+		const topDomains = domainsSorted.slice(0, 10);
+
+		process.stdout.write(' >>> ' + topDomains.join(', ') + '\n');
+
+		// Make domains object (includes weights)
+		const domainsObj = topDomains.map(domain => ({
+			domain: domain,
+			weight: domains[domain] / peak
+		}));
+
+		// Save
+		User.update({ _id: id }, {
+			$set: {
+				domains: domainsObj
+			}
+		}).then(() => {
+			resolve();
+		}, err => {
+			reject(err);
+		});
+	});
+}
diff --git a/src/web/app/mobile/tags/user.tag b/src/web/app/mobile/tags/user.tag
index ca777b8fd1..a323559218 100644
--- a/src/web/app/mobile/tags/user.tag
+++ b/src/web/app/mobile/tags/user.tag
@@ -240,6 +240,12 @@
 			<mk-user-overview-keywords user={ user }/>
 		</div>
 	</section>
+	<section class="domains">
+		<h2><i class="fa fa-globe"></i>%i18n:mobile.tags.mk-user-overview.domains%</h2>
+		<div>
+			<mk-user-overview-domains user={ user }/>
+		</div>
+	</section>
 	<section class="followers-you-know" if={ SIGNIN && I.id !== user.id }>
 		<h2><i class="fa fa-users"></i>%i18n:mobile.tags.mk-user-overview.followers-you-know%</h2>
 		<div>
@@ -579,6 +585,40 @@
 	</script>
 </mk-user-overview-keywords>
 
+<mk-user-overview-domains>
+	<div if={ user.domains != null && user.domains.length > 1 }>
+		<virtual each={ domain in user.domains }>
+			<a style="opacity: { 0.5 + (domain.weight / 2) }">{ domain.domain }</a>
+		</virtual>
+	</div>
+	<p class="empty" if={ user.domains == null || user.domains.length == 0 }>%i18n:mobile.tags.mk-user-overview-domains.no-domains%</p>
+	<style>
+		:scope
+			display block
+
+			> div
+				padding 4px
+
+				> a
+					display inline-block
+					margin 4px
+					color #555
+
+			> .empty
+				margin 0
+				padding 16px
+				text-align center
+				color #aaa
+
+				> i
+					margin-right 4px
+
+	</style>
+	<script>
+		this.user = this.opts.user;
+	</script>
+</mk-user-overview-domains>
+
 <mk-user-overview-followers-you-know>
 	<p class="initializing" if={ initializing }><i class="fa fa-spinner fa-pulse fa-fw"></i>%i18n:mobile.tags.mk-user-overview-followers-you-know.loading%<mk-ellipsis/></p>
 	<div if={ !initializing && users.length > 0 }>
-- 
cgit v1.2.3-freya