tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[parser] fixing affix-related valgrind errors in address parser features

Browse Source
This commit is contained in:
Al
2017-03-06 20:28:42 -05:00
parent c3581557a1
commit 839a13577d
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/address_parser.c
View File

@@ -1056,6 +1056,8 @@ bool address_parser_features(void *self, void *ctx, tokenized_string_t *tokenize
return false; return false;
} }
char *word_pre_norm = tokenized_string_get_token(tokenized, idx);
size_t word_len = strlen(word); size_t word_len = strlen(word);
log_debug("word=%s\n", word); log_debug("word=%s\n", word);
@@ -1289,7 +1291,7 @@ bool address_parser_features(void *self, void *ctx, tokenized_string_t *tokenize
known_prefix = true; known_prefix = true;
char_array_clear(phrase_tokens); char_array_clear(phrase_tokens);
prefix_len = prefix_phrase.len; prefix_len = prefix_phrase.len;
char_array_add_len(phrase_tokens, word, prefix_len); char_array_add_len(phrase_tokens, word_pre_norm, prefix_len);
prefix = char_array_get_string(phrase_tokens); prefix = char_array_get_string(phrase_tokens);
log_debug("got prefix: %s\n", prefix); log_debug("got prefix: %s\n", prefix);
feature_array_add(features, 2, "prefix", prefix); feature_array_add(features, 2, "prefix", prefix);
@@ -1305,7 +1307,7 @@ bool address_parser_features(void *self, void *ctx, tokenized_string_t *tokenize
known_suffix = true; known_suffix = true;
char_array_clear(context->suffix_phrase); char_array_clear(context->suffix_phrase);
suffix_len = suffix_phrase.len; suffix_len = suffix_phrase.len;
char_array_add_len(context->suffix_phrase, word + (token.len - suffix_phrase.len), suffix_len); char_array_add_len(context->suffix_phrase, word_pre_norm + (token.len - suffix_phrase.len), suffix_len);
suffix = char_array_get_string(context->suffix_phrase); suffix = char_array_get_string(context->suffix_phrase);
log_debug("got suffix: %s\n", suffix); log_debug("got suffix: %s\n", suffix);
feature_array_add(features, 2, "suffix", suffix); feature_array_add(features, 2, "suffix", suffix);
@@ -1402,7 +1404,6 @@ bool address_parser_features(void *self, void *ctx, tokenized_string_t *tokenize
feature_array_add(prev_tag_features, 2, "word", word); feature_array_add(prev_tag_features, 2, "word", word);
feature_array_add(prev_tag_features, 1, "trans"); feature_array_add(prev_tag_features, 1, "trans");
// Previous two tags and current word // Previous two tags and current word
feature_array_add(prev2_tag_features, 2, "word", word); feature_array_add(prev2_tag_features, 2, "word", word);
feature_array_add(prev2_tag_features, 1, "trans"); feature_array_add(prev2_tag_features, 1, "trans");