tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[python] Adding word_token_types

Browse Source
This commit is contained in:
Al
2015-10-25 18:33:09 -04:00
parent 2d4b3a6e2f
commit 4f784060a3
1 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
python/postal/text/token_types.py
View File

@@ -51,3 +51,12 @@ class token_types(Enum):
OTHER = EnumValue(200) OTHER = EnumValue(200)
WHITESPACE = EnumValue(300) WHITESPACE = EnumValue(300)
NEWLINE = EnumValue(301) NEWLINE = EnumValue(301)
word_token_types = set([
token_types.WORD,
token_types.ABBREVIATION,
token_types.IDEOGRAPHIC_CHAR,
token_types.HANGUL_SYLLABLE,
token_types.ACRONYM
])