[gazetteers] moving PHRASE to a token type

2016-04-27 15:11:38 -04:00
parent dff4a5e76e
commit 1266f5e9b5
2 changed files with 12 additions and 9 deletions
--- a/scripts/geodata/text/token_types.py
+++ b/scripts/geodata/text/token_types.py
@@ -52,6 +52,9 @@ class token_types(Enum):
    WHITESPACE = EnumValue(300)
    NEWLINE = EnumValue(301)

+    # Phrase, special application-level type not returned by the tokenizer
+    PHRASE = EnumValue(999)
+
    WORD_TOKEN_TYPES = set([
        WORD,
        ABBREVIATION,