[gazetteers] moving PHRASE to a token type

2016-04-27 15:11:38 -04:00
parent 04a5a9e611
commit d5dc34ec1d
2 changed files with 12 additions and 9 deletions
--- a/scripts/geodata/text/token_types.py
+++ b/scripts/geodata/text/token_types.py
@@ -52,6 +52,9 @@ class token_types(Enum):
    WHITESPACE = EnumValue(300)
    NEWLINE = EnumValue(301)

+    # Phrase, special application-level type not returned by the tokenizer
+    PHRASE = EnumValue(999)
+
    WORD_TOKEN_TYPES = set([
        WORD,
        ABBREVIATION,