blob: 9f21ae2e17815f903521723f34e2acb2b0c2610f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
const MeCab = require('mecab-async');
import Post from '../../api/models/post';
import User from '../../api/models/user';
import config from '../../conf';
const mecab = new MeCab();
if (config.categorizer.mecab_command) mecab.command = config.categorizer.mecab_command;
function tokenize(text: string) {
const tokens = this.mecab.parseSync(text)
// キーワードのみ
.filter(token => token[1] == '名詞' && (token[2] == '固有名詞' || token[2] == '一般'))
// 取り出し
.map(token => token[0]);
return tokens;
}
// Fetch all users
User.find({}, {
fields: {
_id: true
}
}).then(users => {
let i = -1;
const x = cb => {
if (++i == users.length) return cb();
extractKeywordsOne(users[i]._id, () => x(cb));
};
x(() => {
console.log('complete');
});
});
async function extractKeywordsOne(id, cb) {
console.log(`extract keywords of ${id} ...`);
// Fetch recent posts
const recentPosts = await Post.find({
user_id: id,
text: {
$exists: true
}
}, {
sort: {
_id: -1
},
limit: 1000,
fields: {
_id: false,
text: true
}
});
// 投稿が少なかったら中断
if (recentPosts.length < 10) {
return cb();
}
const keywords = {};
// Extract keywords from recent posts
recentPosts.forEach(post => {
const keywordsOfPost = tokenize(post.text);
keywordsOfPost.forEach(keyword => {
if (keywords[keyword]) {
keywords[keyword]++;
} else {
keywords[keyword] = 1;
}
});
});
// Sort keywords by frequency
const keywordsSorted = Object.keys(keywords).sort((a, b) => keywords[b] - keywords[a]);
// Lookup top 10 keywords
const topKeywords = keywordsSorted.slice(0, 10);
process.stdout.write(' >>> ' + topKeywords.join(' '));
// Save
User.update({ _id: id }, {
$set: {
keywords: topKeywords
}
}).then(() => {
cb();
});
}
|
