tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] using new pypostal tokenize API

Browse Source
This commit is contained in:
Al
2015-11-22 18:01:07 -05:00
parent 71afcafe11
commit ce065bb9ec
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/geodata/language_id/disambiguation.py
View File

@@ -197,7 +197,7 @@ def disambiguate_language(text, languages):
read_len += script_len
num_defaults = sum((1 for lang, default in valid_languages.iteritems() if default))
tokens = [(c, t.rstrip('.')) for c, t in tokenize(safe_decode(text).replace(u'-', u' ').lower())]
tokens = [(c, t.rstrip('.')) for t, c in tokenize(safe_decode(text).replace(u'-', u' ').lower())]
current_lang = None
possible_lang = None