tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[languages] encoding in language disambiguation

Browse Source
This commit is contained in:
Al
2016-01-19 03:22:03 -05:00
parent 3262d2ccd3
commit 8b94a018e6
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scripts/geodata/language_id/disambiguation.py
View File

@@ -192,8 +192,8 @@ script_languages = {}
def init_disambiguation(): def init_disambiguation():
global char_scripts, script_languages global char_scripts, script_languages
char_scripts.extend(get_chars_by_script()) char_scripts = get_chars_by_script()
script_languages.update({script: set(langs) for script, langs in get_script_languages().iteritems()}) script_languages = {script: set(langs) for script, langs in get_script_languages().iteritems()}
UNKNOWN_SCRIPT = 'Unknown' UNKNOWN_SCRIPT = 'Unknown'
COMMON_SCRIPT = 'Common' COMMON_SCRIPT = 'Common'
@@ -229,6 +229,7 @@ AMBIGUOUS_LANGUAGE = 'xxx'
def disambiguate_language(text, languages): def disambiguate_language(text, languages):
text = safe_decode(text)
valid_languages = OrderedDict(languages) valid_languages = OrderedDict(languages)
script_langs = {} script_langs = {}
read_len = 0 read_len = 0
@@ -245,7 +246,7 @@ def disambiguate_language(text, languages):
num_defaults = sum((1 for lang, default in valid_languages.iteritems() if default)) num_defaults = sum((1 for lang, default in valid_languages.iteritems() if default))
tokens = normalized_tokens((safe_decode(text))) tokens = normalized_tokens(text)
current_lang = None current_lang = None
possible_lang = None possible_lang = None