From ff3a3c2201baae10af8d44566738de069d842dc3 Mon Sep 17 00:00:00 2001 From: Al Date: Wed, 21 Oct 2015 16:35:55 -0400 Subject: [PATCH] [fix] disambiguation tokenizer to pypostal --- scripts/geodata/language_id/disambiguation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/geodata/language_id/disambiguation.py b/scripts/geodata/language_id/disambiguation.py index 7d9e6ef0..e1e7d715 100644 --- a/scripts/geodata/language_id/disambiguation.py +++ b/scripts/geodata/language_id/disambiguation.py @@ -10,13 +10,13 @@ sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir))) sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir, os.pardir, 'python'))) +from address_normalizer.text.normalize import PhraseFilter from geodata.encoding import safe_decode from geodata.string_utils import wide_iter, wide_ord from geodata.i18n.unicode_paths import DATA_DIR from geodata.i18n.normalize import strip_accents from geodata.i18n.unicode_properties import get_chars_by_script, get_script_languages -from address_normalizer.text.normalize import PhraseFilter -from address_normalizer.text.tokenize import * +from postal.text.tokenize import * WELL_REPRESENTED_LANGUAGES = set(['en', 'fr', 'it', 'de', 'nl', 'es', 'pt'])