tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[transliteration] Adding html escapes as the first step in the Latin-ASCII transformation

Browse Source
This commit is contained in:
Al
2015-05-20 14:44:55 -04:00
parent 1fee0a3e35
commit d65f7747f0
1 changed files with 31 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/geodata/i18n/transliteration_rules.py
View File

@@ -1066,10 +1066,21 @@ STEP_UNICODE_NORMALIZATION = 'STEP_UNICODE_NORMALIZATION'
NEW_STEP = 'NEW_STEP'
EXISTING_STEP = 'EXISTING_STEP'
PREPEND_STEP = 'PREPEND_STEP'
html_escapes = {'&{};'.format(name): escape_string(safe_encode(unichr(value)))
for name, value in htmlentitydefs.name2codepoint.iteritems()
}
# Extra rules defined here
supplemental_transliterations = {
'latin-ascii': (EXISTING_STEP, [
'latin-ascii': [
(PREPEND_STEP, [(quote_string(name), str(len(name) + 2), CONTEXT_TYPE_NONE, '0', 'NULL', '0', CONTEXT_TYPE_NONE, '0', 'NULL', '0', quote_string(value), str(len(value)), '0', 'NULL', '0')
for name, value in html_escapes.iteritems()
]
),
(EXISTING_STEP, [
# German transliterations not handled by standard NFD normalization
# ä => ae
(u'"\\xc3\\xa4"', '2', CONTEXT_TYPE_NONE, '0', 'NULL', '0', CONTEXT_TYPE_NONE, '0', 'NULL', '0', u'"ae"', '2', '0', 'NULL', '0'),
@@ -1081,6 +1092,7 @@ supplemental_transliterations = {
(u'"\\xc3\\x9f"', '2', CONTEXT_TYPE_NONE, '0', 'NULL', '0', CONTEXT_TYPE_NONE, '0', 'NULL', '0', u'"ss"', '2', '0', 'NULL', '0'),
]),
],
}
@@ -1175,11 +1187,13 @@ def get_all_transform_rules():
for name, steps in transforms.iteritems():
if name in supplemental_transliterations:
step_type, rules = supplemental_transliterations[name]
for step_type, rules in supplemental_transliterations[name]:
if step_type == EXISTING_STEP:
steps[-1][1].extend(rules)
elif step_type == PREPEND_STEP:
steps = [(STEP_RULESET, rules)] + steps
else:
steps[-1].append((STEP_RULESET, rules))
steps.append((STEP_RULESET, rules))
# Only care if it's a transform to Latin/ASCII or a dependency
# for a transform to Latin/ASCII
elif name not in retain_transforms: