tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[normalization] Adding default token options for numbers so we split alpha from numeric tokens and don't normalize digits

Browse Source
This commit is contained in:
Al
2016-04-28 13:03:16 -04:00
parent 974cef7ec4
commit 7f81e0ed5b
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/geodata/text/normalize.py
View File

@@ -38,6 +38,8 @@ DEFAULT_TOKEN_OPTIONS = NORMALIZE_TOKEN_REPLACE_HYPHENS | \
NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE | \ NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE | \
NORMALIZE_TOKEN_REPLACE_DIGITS NORMALIZE_TOKEN_REPLACE_DIGITS
DEFAULT_TOKEN_OPTIONS_NUMERIC = (DEFAULT_TOKEN_OPTIONS ^ NORMALIZE_TOKEN_REPLACE_DIGITS) | NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC
def remove_parens(tokens): def remove_parens(tokens):
new_tokens = [] new_tokens = []