tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] Accounting for unknown scripts in disambiguation

Browse Source
This commit is contained in:
Al
2015-09-21 18:05:28 -04:00
parent 3ac89d7ed9
commit 747de1944b
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/geodata/language_id/disambiguation.py
View File

@@ -186,7 +186,7 @@ def disambiguate_language(text, languages):
while read_len < len(text): while read_len < len(text):
script, script_len, is_ascii = get_string_script(text[read_len:]) script, script_len, is_ascii = get_string_script(text[read_len:])
if script != LATIN_SCRIPT: if script != LATIN_SCRIPT:
script_langs[script] = set([l for l, d in languages if l in script_languages[script]]) script_langs[script] = set([l for l, d in languages if l in script_languages.get(script, [])])
read_len += script_len read_len += script_len
num_defaults = sum((1 for lang, default in valid_languages.iteritems() if default)) num_defaults = sum((1 for lang, default in valid_languages.iteritems() if default))