tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[normalization] normalize tokens should not replace digits by default

Browse Source
This commit is contained in:
Al
2016-04-28 18:03:44 -04:00
parent 3a9ac9d96f
commit ee1aa564c4
2 changed files with 5 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scripts/geodata/names/deduping.py
View File

@@ -39,8 +39,7 @@ class NameDeduper(object):
@classmethod
def tokenize(cls, s):
token_options = DEFAULT_TOKEN_OPTIONS ^ NORMALIZE_TOKEN_REPLACE_DIGITS
return normalized_tokens(s, token_options=token_options)
return normalized_tokens(s)
@classmethod
def content_tokens(cls, s):
@@ -49,8 +48,8 @@ class NameDeduper(object):
tokens = remove_parens(tokens)
return [(cls.replacements.get(t, t), c)
for t, c in tokens
if c in cls.content_categories
and t not in cls.stopwords]
if c in cls.content_categories and
t not in cls.stopwords]
@classmethod
def possible_match(cls, tokens1, tokens2):

5
scripts/geodata/text/normalize.py
View File

@@ -35,10 +35,9 @@ DEFAULT_TOKEN_OPTIONS = NORMALIZE_TOKEN_REPLACE_HYPHENS | \
NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | \
NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS | \
NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES | \
NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE | \
NORMALIZE_TOKEN_REPLACE_DIGITS
NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE
DEFAULT_TOKEN_OPTIONS_NUMERIC = (DEFAULT_TOKEN_OPTIONS ^ NORMALIZE_TOKEN_REPLACE_DIGITS) | NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC
DEFAULT_TOKEN_OPTIONS_NUMERIC = (DEFAULT_TOKEN_OPTIONS | NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC)
def remove_parens(tokens):