tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[transliteration] fixing variable assignments, literal wide characters (for narrow Python builds), ignoring rules related to spaced Han

Browse Source
This commit is contained in:
Al
2015-05-13 16:20:52 -04:00
parent cbe83376f2
commit 304dc9525a
1 changed files with 55 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
scripts/geodata/i18n/transliteration_rules.py
View File

@@ -53,6 +53,11 @@ REVISIT_INDICATOR = '|'
WORD_BOUNDARY_VAR_NAME = 'wordBoundary'
WORD_BOUNDARY_VAR = '${}'.format(WORD_BOUNDARY_VAR_NAME)
START_OF_HAN_VAR_NAME = 'startOfHanMarker'
START_OF_HAN_VAR = '${}'.format(START_OF_HAN_VAR_NAME)
start_of_han_regex = re.compile(START_OF_HAN_VAR.replace('$', '\$'))
word_boundary_var_regex = re.compile(WORD_BOUNDARY_VAR.replace('$', '\$'))
WORD_BOUNDARY_CHAR = u'\u0001'
@@ -70,11 +75,12 @@ BEGIN_SET_CHAR = u"\x07"
END_SET_CHAR = u"\x08"
GROUP_INDICATOR_CHAR = u"\x09"
EXCLUDE_TRANSLITERATORS = set([
'hangul-latin',
'interindic-latin',
'jamo-latin',
# Don't care about spaced Han because
# Don't care about spaced Han because our tokenizer does it already
'han-spacedhan',
])
@@ -197,7 +203,8 @@ all_transforms = set()
pre_transform_full_regex = re.compile('::[\s]*(.*)[\s]*', re.UNICODE)
pre_transform_regex = re.compile('[\s]*([^\s\(\)]*)[\s]*(?:\(.*\)[\s]*)?', re.UNICODE)
transform_regex = re.compile(u"(?:[\s]*(?!=[\s])(.*)(?<![\s])[\s]*)((?:<>)|[←<→>↔=])(?:[\s]*(?!=[\s])(.*)(?<![\s])[\s]*)", re.UNICODE)
assignment_regex = re.compile(u"(?:[\s]*(\$[^\s\=]+)[\s]*\=[\s]*(?!=[\s])(.*)(?<![\s])[\s]*)", re.UNICODE)
transform_regex = re.compile(u"(?:[\s]*(?!=[\s])(.*?)(?<![\s])[\s]*)((?:<>)|[←<→>↔])(?:[\s]*(?!=[\s])(.*)(?<![\s])[\s]*)", re.UNICODE)
quoted_string_regex = re.compile(r'\'.*?\'', re.UNICODE)
@@ -208,7 +215,8 @@ END_CHAR = ';'
def unescape_unicode_char(m):
return m.group(0).decode('unicode-escape')
escaped_unicode_regex = re.compile(r'(?:\\u[0-9A-Fa-f]{4}|\\U[0-9A-Fa-f]{8})')
escaped_unicode_regex = re.compile(r'\\u[0-9A-Fa-f]{4}')
escaped_wide_unicode_regex = re.compile(r'\\U[0-9A-Fa-f]{8}')
literal_space_regex = re.compile(r'(?:\\u0020|\\U00000020)')
@@ -259,6 +267,7 @@ CHAR_MULTI_SET = 'CHAR_MULTI_SET'
CHAR_CLASS = 'CHAR_CLASS'
OPTIONAL = 'OPTIONAL'
CHARACTER = 'CHARACTER'
WIDE_CHARACTER = 'WIDE_CHARACTER'
REVISIT = 'REVISIT'
REPEAT = 'REPEAT'
LPAREN = 'LPAREN'
@@ -270,6 +279,7 @@ HTML_ENTITY = 'HTML_ENTITY'
SINGLE_QUOTE = 'SINGLE_QUOTE'
ESCAPED_CHARACTER = 'ESCAPED_CHARACTER'
BEFORE_CONTEXT = '{'
AFTER_CONTEXT = '}'
@@ -297,6 +307,7 @@ transform_scanner = Scanner([
('\)', RPAREN),
('\|', REVISIT),
('[\s]+', WHITESPACE),
(r'[\ud800-\udbff][\udc00-\udfff]', WIDE_CHARACTER),
(r'[\\]?[^\s]', CHARACTER),
], re.UNICODE)
@@ -325,6 +336,7 @@ char_set_scanner = Scanner([
('&', INTERSECTION),
('(?<=[\s])-(?=[\s])', DIFFERENCE),
('\$', WORD_BOUNDARY),
(ur'[\ud800-\udbff][\udc00-\udfff]', WIDE_CHARACTER),
(r'[^\s]', CHARACTER),
])
@@ -332,7 +344,7 @@ NUM_CHARS = 65536
all_chars = set([unichr(i) for i in xrange(NUM_CHARS)])
control_chars = set([c for c in all_chars if unicodedata.category(c) == 'Cc'])
control_chars = set([c for c in all_chars if unicodedata.category(c) in ('Cc', 'Cn', 'Cs')])
def get_transforms():
@@ -465,6 +477,8 @@ def parse_regex_char_set(s, current_filter=all_chars):
is_difference = False
is_word_boundary = False
real_chars = set()
for token, token_class in parse_balanced_sets(s):
if token_class == CHAR_RANGE:
this_char_set = set(parse_regex_char_range(token))
@@ -472,10 +486,15 @@ def parse_regex_char_set(s, current_filter=all_chars):
elif token_class == ESCAPED_CHARACTER:
token = token.strip('\\')
this_group.add(token)
real_chars.add(token)
elif token_class == SINGLE_QUOTE:
this_group.add("'")
t = "'"
this_group.add(t)
real_chars.add(t)
elif token_class == QUOTED_STRING:
this_group.add(token.strip("'"))
t = token.strip("'")
this_group.add(t)
real_chars.add(t)
elif token_class == NEGATION:
is_negation = True
elif token_class in (CHAR_CLASS, CHAR_CLASS_PCRE):
@@ -495,15 +514,18 @@ def parse_regex_char_set(s, current_filter=all_chars):
is_intersection = True
elif token_class == DIFFERENCE:
is_difference = True
elif token_class == CHARACTER:
elif token_class == CHARACTER and token not in control_chars:
this_group.add(token)
real_chars.add(token)
elif token_class == WIDE_CHARACTER:
continue
elif token_class == WORD_BOUNDARY:
is_word_boundary = True
if is_negation:
this_group = current_filter - this_group
return sorted((this_group & current_filter) - control_chars) + ([WORD_BOUNDARY_CHAR] if is_word_boundary else [])
return sorted((this_group & (current_filter | real_chars)) - control_chars) + ([WORD_BOUNDARY_CHAR] if is_word_boundary else [])
for name, regex_range in unicode_property_regexes:
@@ -535,6 +557,7 @@ def get_raw_rules_and_variables(xml):
rule = safe_decode(rule.text.rsplit(COMMENT_CHAR)[0].strip())
rule = literal_space_regex.sub(replace_literal_space, rule)
rule = escaped_wide_unicode_regex.sub('', rule)
rule = escaped_unicode_regex.sub(unescape_unicode_char, rule)
rule = rule.rstrip(END_CHAR).strip()
@@ -548,12 +571,17 @@ def get_raw_rules_and_variables(xml):
in_compound_rule = False
compound_rule = []
assignment = assignment_regex.match(rule)
transform = transform_regex.match(rule)
pre_transform = pre_transform_full_regex.match(rule)
if pre_transform:
rules.append((PRE_TRANSFORM, pre_transform.group(1)))
elif assignment:
lvalue, rvalue = assignment.groups()
var_name = lvalue.strip().lstrip('$')
rvalue = rvalue.strip()
variables[var_name] = rvalue
elif transform:
lvalue, op, rvalue = transform.groups()
lvalue = lvalue.strip()
@@ -565,11 +593,6 @@ def get_raw_rules_and_variables(xml):
rules.append((BIDIRECTIONAL_TRANSFORM, (lvalue, rvalue)))
elif op in BACKWARD_TRANSFORM_OPS:
rules.append((BACKWARD_TRANSFORM, (lvalue, rvalue)))
elif op == ASSIGNMENT_OP:
var_name = lvalue.lstrip('$')
variables[var_name] = rvalue
else:
print 'non-rule', rule, get_source_and_target(xml)
return rules, variables
@@ -626,7 +649,9 @@ def char_permutations(s, current_filter=all_chars):
open_brackets -= 1
current_set.append(token)
if open_brackets == 0:
char_types.append(parse_regex_char_set(u''.join(current_set), current_filter=current_filter))
char_set = parse_regex_char_set(u''.join(current_set), current_filter=current_filter)
if char_set:
char_types.append(char_set)
current_set = []
elif token_type == QUOTED_STRING:
token = token.strip("'")
@@ -648,7 +673,8 @@ def char_permutations(s, current_filter=all_chars):
char_types.append([replace_html_entity(token)])
elif token_type == CHARACTER:
char_types.append([token])
elif token_type == WIDE_CHARACTER:
continue
if in_group and last_token_group_start:
start_group = len(char_types)
last_token_group_start = False
@@ -721,7 +747,6 @@ def format_groups(char_types, groups):
group_regex.append(char_types_string(char_types[last_end + 1:]))
return u''.join(group_regex)
charset_regex = re.compile(r'(?<!\\)\[')
@@ -823,18 +848,23 @@ def parse_transform_rules(xml):
break
variables[WORD_BOUNDARY_VAR_NAME] = WORD_BOUNDARY_VAR
variables[START_OF_HAN_VAR_NAME] = START_OF_HAN_VAR
current_filter = all_chars
for rule_type, rule in rules:
if rule_type in (BIDIRECTIONAL_TRANSFORM, FORWARD_TRANSFORM):
left, right = rule
left = var_regex.sub(get_var, left)
right = var_regex.sub(get_var, right)
left_pre_context, left, left_post_context = context_regex.match(left).groups()
right_pre_context, right, right_post_context = context_regex.match(right).groups()
if start_of_han_regex.search(left) or start_of_han_regex.search(right):
continue
left_pre_context_max_len = 0
left_post_context_max_len = 0
@@ -849,6 +879,9 @@ def parse_transform_rules(xml):
if left_pre_context.strip() == WORD_BOUNDARY_VAR:
left_pre_context = None
left_pre_context_type = CONTEXT_TYPE_WORD_BOUNDARY
elif left_pre_context.strip() == START_OF_HAN_VAR:
left_pre_context = None
left_pre_context_type = CONTEXT_TYPE_NONE
else:
left_pre_context, _, _ = char_permutations(left_pre_context.strip(), current_filter=current_filter)
left_pre_context_max_len = len(left_pre_context or [])
@@ -871,6 +904,9 @@ def parse_transform_rules(xml):
if left_post_context.strip() == WORD_BOUNDARY_VAR:
left_post_context = None
left_post_context_type = CONTEXT_TYPE_WORD_BOUNDARY
elif left_post_context.strip() == START_OF_HAN_VAR:
left_pre_context_type = None
left_pre_context_type = CONTEXT_TYPE_NONE
else:
left_post_context, _, _ = char_permutations(left_post_context.strip(), current_filter=current_filter)
left_post_context_max_len = len(left_post_context or [])
@@ -881,6 +917,8 @@ def parse_transform_rules(xml):
left_post_context_type = CONTEXT_TYPE_STRING
if right:
if start_of_han_regex.search(right):
continue
right, move, right_groups = char_permutations(right.strip(), current_filter=current_filter)
right = char_types_string(right)