[python] Adding word_token_types

2015-10-25 18:33:09 -04:00
parent 2d4b3a6e2f
commit 4f784060a3
1 changed files with 9 additions and 0 deletions
--- a/python/postal/text/token_types.py
+++ b/python/postal/text/token_types.py
@@ -51,3 +51,12 @@ class token_types(Enum):
    OTHER = EnumValue(200)
    WHITESPACE = EnumValue(300)
    NEWLINE = EnumValue(301)
+
+
+word_token_types = set([
+    token_types.WORD,
+    token_types.ABBREVIATION,
+    token_types.IDEOGRAPHIC_CHAR,
+    token_types.HANGUL_SYLLABLE,
+    token_types.ACRONYM
+])