UNPKG

@thi.ng/text-analysis

Version:

Text tokenization, transformation & analysis transducers, utilities, stop words, porter stemming, vector encodings, similarities

thi.ng/text-analysis

thi-ng/umbrella

106 lines (105 loc) • 1.23 kB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
import { filter } from "@thi.ng/transducers/filter"; const DEFAULT_STOP_WORDS_EN = /* @__PURE__ */ new Set([ "a", "above", "across", "after", "against", "along", "also", "among", "an", "and", "any", "are", "around", "at", "be", "been", "before", "being", "below", "beneath", "between", "beyond", "both", "but", "by", "could", "despite", "do", "does", "down", "during", "each", "eg", "etc", "even", "every", "few", "for", "from", "he", "i", "ie", "in", "inside", "into", "is", "it", "just", "less", "many", "may", "might", "more", "much", "must", "of", "on", "one", "onto", "or", "out", "outside", "over", "over", "quite", "really", "several", "she", "should", "since", "so", "some", "such", "that", "the", "these", "they", "this", "those", "three", "through", "to", "too", "toward", "two", "under", "until", "up", "very", "was", "we", "were", "with", "within", "without", "would", "you" ]); const removeStopWords = (words = DEFAULT_STOP_WORDS_EN) => filter((x) => !words.has(x)); export { DEFAULT_STOP_WORDS_EN, removeStopWords };