120 lines
4.6 KiB
Python
120 lines
4.6 KiB
Python
import os
|
|
import re
|
|
import six
|
|
import yaml
|
|
|
|
from geodata.encoding import safe_decode
|
|
|
|
this_dir = os.path.realpath(os.path.dirname(__file__))
|
|
|
|
AFFIX_CONFIG_DIR = os.path.join(this_dir, os.pardir, os.pardir, os.pardir,
|
|
'resources', 'boundaries', 'names', 'languages')
|
|
|
|
|
|
class NameAffixes(object):
|
|
def __init__(self, config_dir=AFFIX_CONFIG_DIR):
|
|
self.config_dir = config_dir
|
|
|
|
self.language_prefixes = {}
|
|
self.language_suffixes = {}
|
|
|
|
self.language_prefix_regexes = {}
|
|
self.language_suffix_regexes = {}
|
|
|
|
self.language_prefix_sim_only_regexes = {}
|
|
self.language_suffix_sim_only_regexes = {}
|
|
|
|
for filename in os.listdir(config_dir):
|
|
if not filename.endswith('.yaml'):
|
|
continue
|
|
lang = filename.rsplit('.yaml')[0]
|
|
|
|
conf = yaml.load(open(os.path.join(config_dir, filename)))
|
|
self.add_affixes(lang, conf)
|
|
|
|
for country, country_conf in six.iteritems(conf.get('countries', {})):
|
|
country_lang = (country, lang)
|
|
self.add_affixes(country_lang, country_conf)
|
|
|
|
def add_affixes(self, lang, *confs):
|
|
prefixes = [safe_decode(phrase).lower() for conf in confs for phrase in conf.get('prefixes', [])]
|
|
prefixes_no_whitespace = [safe_decode(phrase).lower() for conf in confs for phrase in conf.get('prefixes_no_whitespace', [])]
|
|
|
|
self.language_prefixes[lang] = prefixes + prefixes_no_whitespace
|
|
|
|
suffixes = [safe_decode(phrase).lower() for conf in confs for phrase in conf.get('suffixes', [])]
|
|
suffixes_no_whitespace = [safe_decode(phrase).lower() for conf in confs for phrase in conf.get('suffixes_no_whitespace', [])]
|
|
|
|
self.language_suffixes[lang] = suffixes + suffixes_no_whitespace
|
|
|
|
whitespace_phrase = six.u('[ \-]')
|
|
|
|
all_prefixes = [six.u('{}{}').format(s, whitespace_phrase) for s in prefixes] + prefixes_no_whitespace
|
|
all_suffixes = [six.u('{}{}').format(whitespace_phrase, s) for s in suffixes] + suffixes_no_whitespace
|
|
|
|
if all_prefixes:
|
|
prefix_regex = six.u('^(?:{})').format(six.u('|').join(all_prefixes))
|
|
self.language_prefix_regexes[lang] = re.compile(prefix_regex, re.I | re.UNICODE)
|
|
|
|
if all_suffixes:
|
|
suffix_regex = six.u('(?:{})$').format(six.u('|').join(all_suffixes))
|
|
self.language_suffix_regexes[lang] = re.compile(suffix_regex, re.I | re.UNICODE)
|
|
|
|
sim_only_prefixes = [six.u('{}{}').format(safe_decode(phrase.lower()), whitespace_phrase) for conf in confs for phrase in conf.get('prefixes_similarity_only', [])]
|
|
if sim_only_prefixes:
|
|
sim_only_prefix_regex = six.u('^(?:{})').format(six.u('|').join(sim_only_prefixes + all_prefixes))
|
|
self.language_prefix_sim_only_regexes[lang] = re.compile(sim_only_prefix_regex, re.I | re.UNICODE)
|
|
|
|
sim_only_suffixes = [six.u('(?:{})$').format(whitespace_phrase, safe_decode(phrase.lower())) for conf in confs for phrase in conf.get('suffixes_similarity_only', [])]
|
|
if sim_only_suffixes:
|
|
sim_only_suffix_regex = six.u('(?:{})$').format(six.u('|').join(sim_only_suffixes + all_suffixes))
|
|
|
|
self.language_suffix_sim_only_regexes[lang] = re.compile(sim_only_suffix_regex, re.I | re.UNICODE)
|
|
|
|
def replace_prefixes(self, name, lang, country=None, sim_only=False):
|
|
name = safe_decode(name).strip()
|
|
|
|
if not sim_only or lang not in self.language_prefix_sim_only_regexes:
|
|
d = self.language_prefix_regexes
|
|
else:
|
|
d = self.language_prefix_sim_only_regexes
|
|
|
|
re = None
|
|
if country is not None:
|
|
re = d.get((country, lang))
|
|
if re:
|
|
name = re.sub(six.u(''), name)
|
|
|
|
re = d.get(lang)
|
|
|
|
if not re:
|
|
return name
|
|
|
|
return re.sub(six.u(''), name)
|
|
|
|
def replace_suffixes(self, name, lang, country=None, sim_only=False):
|
|
name = safe_decode(name).strip()
|
|
|
|
if not sim_only or lang not in self.language_suffix_sim_only_regexes:
|
|
d = self.language_suffix_regexes
|
|
else:
|
|
d = self.language_suffix_sim_only_regexes
|
|
|
|
re = None
|
|
if country is not None:
|
|
re = d.get((country, lang))
|
|
if re:
|
|
name = re.sub(six.u(''), name)
|
|
|
|
re = d.get(lang)
|
|
|
|
if not re:
|
|
return name
|
|
|
|
return re.sub(six.u(''), name)
|
|
|
|
def replace_affixes(self, name, lang, country=None, sim_only=False):
|
|
return self.replace_prefixes(self.replace_suffixes(name, lang, country=country, sim_only=sim_only), lang, country=country, sim_only=sim_only)
|
|
|
|
name_affixes = NameAffixes()
|