tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[expansion] Adding search_address_dictionaries_prefix/suffix for concatenated prefixes/suffixes e.g. in Germanic languages. Adding a flag to the address_expansion struct and trie value to denote separability, adding prefix/suffix keys during dictionary creation

Browse Source
This commit is contained in:
Al
2015-08-10 16:15:01 -04:00
parent 0f77ca1213
commit de5d6945b5
3 changed files with 93 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/address_dictionary_builder.c
View File

@@ -51,6 +51,7 @@ int main(int argc, char **argv) {
uint16_t address_components = 0;
address_expansion_t expansion;
expansion.separable = 0;
strcpy(expansion.language, language);
expansion.num_dictionaries = expansion_rule.num_dictionaries;
@@ -60,6 +61,12 @@ int main(int argc, char **argv) {
expansion.dictionary_ids[d] = dictionary_id;
if (dictionary_id == DICTIONARY_CONCATENATED_PREFIX_SEPARABLE ||
dictionary_id == DICTIONARY_CONCATENATED_SUFFIX_SEPARABLE ||
dictionary_id == DICTIONARY_ELISION) {
expansion.separable = 1;
}
k = kh_get(int_int, dictionary_components, (uint32_t)dictionary_id);
if (k == kh_end(dictionary_components)) {
log_error("Invalid dictionary_type: %d\n", dictionary_id);
@@ -93,20 +100,14 @@ int main(int argc, char **argv) {
// Add the phrase itself to the base namespace for existence checks
if (!address_dictionary_add_expansion(expansion_rule.phrase, expansion)) {
if (!address_dictionary_add_expansion(expansion_rule.phrase, NULL, expansion)) {
log_error("Could not add expansion {%s}\n", expansion_rule.phrase);
exit(EXIT_FAILURE);
}
// Add phrase namespaced by language for language-specific matching
char_array_clear(key);
char_array_cat(key, language);
char_array_cat(key, NAMESPACE_SEPARATOR_CHAR);
char_array_cat(key, expansion_rule.phrase);
char *token = char_array_get_string(key);
if (!address_dictionary_add_expansion(token, expansion)) {
if (!address_dictionary_add_expansion(expansion_rule.phrase, language, expansion)) {
log_error("Could not add language expansion {%s, %s}\n", language, expansion_rule.phrase);
exit(EXIT_FAILURE);
}
@@ -122,4 +123,4 @@ int main(int argc, char **argv) {
kh_destroy(str_int, canonical_indices);
address_dictionary_module_teardown();
}
}