From 4f784060a3259cc072c585e67d4a23efb20b2303 Mon Sep 17 00:00:00 2001 From: Al Date: Sun, 25 Oct 2015 18:33:09 -0400 Subject: [PATCH] [python] Adding word_token_types --- python/postal/text/token_types.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/postal/text/token_types.py b/python/postal/text/token_types.py index af161e2b..4c8db65a 100644 --- a/python/postal/text/token_types.py +++ b/python/postal/text/token_types.py @@ -51,3 +51,12 @@ class token_types(Enum): OTHER = EnumValue(200) WHITESPACE = EnumValue(300) NEWLINE = EnumValue(301) + + +word_token_types = set([ + token_types.WORD, + token_types.ABBREVIATION, + token_types.IDEOGRAPHIC_CHAR, + token_types.HANGUL_SYLLABLE, + token_types.ACRONYM +])