tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] do not add a token if prefix/suffix expansions are inseparable and canonical

Browse Source
This commit is contained in:
Al
2015-12-19 01:36:02 -05:00
parent f8da44e8b0
commit 4497c4501e
1 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/libpostal.c
View File

@@ -414,7 +414,7 @@ inline void cat_affix_expansion(char_array *key, char *str, address_expansion_t
} }
} }
void add_affix_expansions(string_tree_t *tree, char *str, char *lang, token_t token, phrase_t prefix, phrase_t suffix, normalize_options_t options) { bool add_affix_expansions(string_tree_t *tree, char *str, char *lang, token_t token, phrase_t prefix, phrase_t suffix, normalize_options_t options) {
cstring_array *strings = tree->strings; cstring_array *strings = tree->strings;
bool have_suffix = suffix.len > 0; bool have_suffix = suffix.len > 0;
@@ -448,6 +448,10 @@ void add_affix_expansions(string_tree_t *tree, char *str, char *lang, token_t to
suffix_expansions = get_affix_expansions(key, str, lang, token, suffix, true, options); suffix_expansions = get_affix_expansions(key, str, lang, token, suffix, true, options);
if (suffix_expansions == NULL) have_suffix = false; if (suffix_expansions == NULL) have_suffix = false;
} }
if (!have_suffix && !have_prefix) {
return false;
}
if (have_prefix && have_suffix) { if (have_prefix && have_suffix) {
for (int i = 0; i < prefix_expansions->n; i++) { for (int i = 0; i < prefix_expansions->n; i++) {
@@ -575,7 +579,7 @@ void add_affix_expansions(string_tree_t *tree, char *str, char *lang, token_t to
char_array_destroy(key); char_array_destroy(key);
cstring_array_destroy(root_strings); cstring_array_destroy(root_strings);
return; return false;
} }
@@ -610,6 +614,8 @@ void add_affix_expansions(string_tree_t *tree, char *str, char *lang, token_t to
cstring_array_destroy(root_strings); cstring_array_destroy(root_strings);
} }
return true;
} }
inline bool expand_affixes(string_tree_t *tree, char *str, char *lang, token_t token, normalize_options_t options) { inline bool expand_affixes(string_tree_t *tree, char *str, char *lang, token_t token, normalize_options_t options) {
@@ -619,9 +625,7 @@ inline bool expand_affixes(string_tree_t *tree, char *str, char *lang, token_t t
if ((suffix.len == 0 && prefix.len == 0)) return false; if ((suffix.len == 0 && prefix.len == 0)) return false;
add_affix_expansions(tree, str, lang, token, prefix, suffix, options); return add_affix_expansions(tree, str, lang, token, prefix, suffix, options);
return true;
} }
inline void add_normalized_strings_tokenized(string_tree_t *tree, char *str, token_array *tokens, normalize_options_t options) { inline void add_normalized_strings_tokenized(string_tree_t *tree, char *str, token_array *tokens, normalize_options_t options) {