tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] language dismabiguation module

Browse Source
This commit is contained in:
Al
2015-08-21 08:03:20 -04:00
parent 4976be64e5
commit baa60aab65
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
scripts/geodata/language_id/disambiguation.py
View File

@@ -1,7 +1,7 @@
import os import os
import sys import sys
from collections import defaultdict from collections import defaultdict, OrderedDict
from marisa_trie import BytesTrie from marisa_trie import BytesTrie
@@ -15,6 +15,7 @@ from geodata.i18n.unicode_paths import DATA_DIR
from address_normalizer.text.normalize import PhraseFilter from address_normalizer.text.normalize import PhraseFilter
from address_normalizer.text.tokenize import * from address_normalizer.text.tokenize import *
WELL_REPRESENTED_LANGUAGES = set(['en', 'fr', 'it', 'de', 'nl', 'es'])
DICTIONARIES_DIR = os.path.join(DATA_DIR, 'dictionaries') DICTIONARIES_DIR = os.path.join(DATA_DIR, 'dictionaries')