tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] abbreviation recasing

Browse Source
This commit is contained in:
Al
2016-08-28 12:04:36 -04:00
parent 3da80b0706
commit 3cf3e401db
1 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
scripts/geodata/address_expansions/abbreviations.py
View File

@@ -28,10 +28,9 @@ expansion_token_regex = re.compile('([^ \-\.]+)([\.\- ]+|$)')
def recase_abbreviation(expansion, tokens, space_token=six.u(' ')):
expansion_tokens = expansion_token_regex.findall(expansion)
print expansion, expansion_tokens, tokens
if len(tokens) > len(expansion_tokens) and all((token_capitalization(t) != LOWER for t, c in tokens)):
expansion_tokens = tokenize(expansion)
is_acronym = len(expansion_tokens) == 1 and expansion_tokens[0][1] == token_types.ACRONYM
expansion_tokenized = tokenize(expansion)
is_acronym = len(expansion_tokenized) == 1 and expansion_tokenized[0][1] == token_types.ACRONYM
if len(expansion) <= 3 or is_acronym:
return expansion.upper()
else:
@@ -57,7 +56,15 @@ def recase_abbreviation(expansion, tokens, space_token=six.u(' ')):
strings.append(space_token)
return six.u('').join(strings)
else:
return space_token.join([t.title() for t in expansion_tokens])
strings = []
for e, suf in expansion_tokens:
strings.append(e.title())
if suf == six.u(' '):
strings.append(space_token)
else:
strings.append(suf)
return six.u('').join(strings)
def abbreviate(gazetteer, s, language, abbreviate_prob=0.3, separate_prob=0.2, add_period_hyphen_prob=0.3):