tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[formatting] making regex-based tests during insert_component optional.If exact_order=True, insert the given component directly before/after the reference component, otherwise for components that already exist in the template only need to care about relative position. Adding a method to determine if template language is important for a particular country/language pair.

Browse Source
This commit is contained in:
Al
2016-10-12 14:42:15 -04:00
parent 3db6b7fbf1
commit f8664b0deb
1 changed files with 23 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
scripts/geodata/address_formatting/formatter.py
View File

@@ -394,26 +394,28 @@ class AddressFormatter(object):
last_removed = False last_removed = False
return ''.join(new_components).strip() return ''.join(new_components).strip()
def insert_component(self, template, tag, before=None, after=None, first=False, last=False, separate=True, is_reverse=False): def insert_component(self, template, tag, before=None, after=None, first=False, last=False,
separate=True, is_reverse=False, exact_order=True):
if not before and not after and not first and not last: if not before and not after and not first and not last:
return return
template = template.rstrip() template = template.rstrip()
first_template_regex = re.compile(six.u('{{#first}}.*?{{/first}}'), re.UNICODE) if not exact_order:
sans_firsts = first_template_regex.sub(six.u(''), template) first_template_regex = re.compile(six.u('{{#first}}.*?{{/first}}'), re.UNICODE)
sans_firsts = first_template_regex.sub(six.u(''), template)
tag_match = re.compile(self.tag_token(tag)).search(sans_firsts) tag_match = re.compile(self.tag_token(tag)).search(sans_firsts)
if before: if before:
before_match = re.compile(self.tag_token(after)).search(sans_firsts) before_match = re.compile(self.tag_token(after)).search(sans_firsts)
if before_match and tag_match and before_match.start() > tag_match.start(): if before_match and tag_match and before_match.start() > tag_match.start():
return template return template
if after: if after:
after_match = re.compile(self.tag_token(after)).search(sans_firsts) after_match = re.compile(self.tag_token(after)).search(sans_firsts)
if after_match and tag_match and tag_match.start() > after_match.start(): if after_match and tag_match and tag_match.start() > after_match.start():
return template return template
key_added = False key_added = False
skip_next_non_token = False skip_next_non_token = False
@@ -449,6 +451,9 @@ class AddressFormatter(object):
elif hasattr(el, 'key'): elif hasattr(el, 'key'):
if el.key == tag: if el.key == tag:
if i == num_tokens - 1 and last:
new_components.append('{{{{{{{key}}}}}}}'.format(key=el.key))
skip_next_non_token = True skip_next_non_token = True
continue continue
@@ -501,7 +506,7 @@ class AddressFormatter(object):
if i > 1: if i > 1:
for component in self.BOUNDARY_COMPONENTS_ORDERED[i - 1:0:-1]: for component in self.BOUNDARY_COMPONENTS_ORDERED[i - 1:0:-1]:
kw = {'before': prev} if not is_reverse else {'after': prev} kw = {'before': prev} if not is_reverse else {'after': prev}
template = self.insert_component(template, component, **kw) template = self.insert_component(template, component, exact_order=False, **kw)
prev = component prev = component
@@ -510,7 +515,7 @@ class AddressFormatter(object):
if i < len(self.BOUNDARY_COMPONENTS_ORDERED) - 1: if i < len(self.BOUNDARY_COMPONENTS_ORDERED) - 1:
for component in self.BOUNDARY_COMPONENTS_ORDERED[i + 1:]: for component in self.BOUNDARY_COMPONENTS_ORDERED[i + 1:]:
kw = {'after': prev} if not is_reverse else {'before': prev} kw = {'after': prev} if not is_reverse else {'before': prev}
template = self.insert_component(template, component, **kw) template = self.insert_component(template, component, exact_order=False, **kw)
prev = component prev = component
@@ -590,7 +595,7 @@ class AddressFormatter(object):
if insertions is None and language: if insertions is None and language:
insertions = nested_get(self.language_insertions, (language, component), default=None) insertions = nested_get(self.language_insertions, (language, component), default=None)
scope = language scope = 'lang:{}'.format(language)
if conditionals is None and language: if conditionals is None and language:
conditionals = nested_get(self.language_conditionals, (language, component), default=None) conditionals = nested_get(self.language_conditionals, (language, component), default=None)
@@ -768,6 +773,9 @@ class AddressFormatter(object):
def tagged_tokens(self, name, label): def tagged_tokens(self, name, label):
return six.u(' ').join([six.u('{}/{}').format(t.replace(' ', ''), label if t != ',' else self.separator_tag) for t, c in tokenize(name)]) return six.u(' ').join([six.u('{}/{}').format(t.replace(' ', ''), label if t != ',' else self.separator_tag) for t, c in tokenize(name)])
def template_language_matters(self, country, language):
return '{}_{}'.format(country.upper(), language) in self.country_formats or '{}_{}'.format(country, language) in self.country_formats
def format_category_query(self, category_query, address_components, country, language, tag_components=True): def format_category_query(self, category_query, address_components, country, language, tag_components=True):
if tag_components: if tag_components:
components = {self.CATEGORY: self.tagged_tokens(category_query.category, self.CATEGORY)} components = {self.CATEGORY: self.tagged_tokens(category_query.category, self.CATEGORY)}