tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[parser] check for non-null prev/prev2 before creating tag-based features

Browse Source
This commit is contained in:
Al
2017-02-24 02:57:16 -05:00
parent 182d60b623
commit e39d4d2f00
1 changed files with 12 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/averaged_perceptron_tagger.c
View File

@@ -1,7 +1,6 @@
#include "averaged_perceptron_tagger.h" #include "averaged_perceptron_tagger.h"
#include "log/log.h" #include "log/log.h"
bool averaged_perceptron_tagger_predict(averaged_perceptron_t *model, void *tagger, void *context, cstring_array *features, cstring_array *prev_tag_features, cstring_array *prev2_tag_features, cstring_array *labels, ap_tagger_feature_function feature_function, tokenized_string_t *tokenized) { bool averaged_perceptron_tagger_predict(averaged_perceptron_t *model, void *tagger, void *context, cstring_array *features, cstring_array *prev_tag_features, cstring_array *prev2_tag_features, cstring_array *labels, ap_tagger_feature_function feature_function, tokenized_string_t *tokenized) {
// Keep two tags of history in training // Keep two tags of history in training
@@ -15,6 +14,8 @@ bool averaged_perceptron_tagger_predict(averaged_perceptron_t *model, void *tagg
for (uint32_t i = 0; i < num_tokens; i++) { for (uint32_t i = 0; i < num_tokens; i++) {
cstring_array_clear(features); cstring_array_clear(features);
cstring_array_clear(prev_tag_features);
cstring_array_clear(prev2_tag_features);
if (i > 0) { if (i > 0) {
prev = cstring_array_get_string(model->classes, prev_id); prev = cstring_array_get_string(model->classes, prev_id);
@@ -34,13 +35,17 @@ bool averaged_perceptron_tagger_predict(averaged_perceptron_t *model, void *tagg
uint32_t fidx; uint32_t fidx;
const char *feature; const char *feature;
if (prev) {
cstring_array_foreach(prev_tag_features, fidx, feature, { cstring_array_foreach(prev_tag_features, fidx, feature, {
feature_array_add(features, 2, (char *)feature, prev); feature_array_add(features, 2, (char *)feature, prev);
}) })
}
if (prev2) {
cstring_array_foreach(prev2_tag_features, fidx, feature, { cstring_array_foreach(prev2_tag_features, fidx, feature, {
feature_array_add(features, 3, (char *)feature, prev2, prev); feature_array_add(features, 3, (char *)feature, prev2, prev);
}) })
}
uint32_t guess = averaged_perceptron_predict(model, features); uint32_t guess = averaged_perceptron_predict(model, features);
char *predicted = cstring_array_get_string(model->classes, guess); char *predicted = cstring_array_get_string(model->classes, guess);