f9a9dc2224
implementation for the address parser. One of the main issues with the greedy averaged perceptron tagger used currently in libpostal is that it predicts left-to-right and commits to its answers i.e. doesn't revise its previous predictions. The model can use its own previous predictions to classify the current word, but effectively it makes the best local decision it can and never looks back (the YOLO approach to parsing). This can be problematic in a multilingual setting like libpostal, since the order of address components is language/country dependent. It would be preferable to have a model that scores whole _sequences_ instead of individual tagging decisions. That's exactly what a Conditional Random Field (CRF) does. Instead of modeling P(y_i|x_i, y_i-1), we're modeling P(y|x) where y is the whole sequence of labels and x is the whole sequence of features. They achieve state-of-the-art results in many tasks (or are a component in the state-of-the-art model - LSTM-CRFs have been an interesting direction along these lines). The crf_context module is heavily borrowed from the version in CRFSuite (https://github.com/chokkan/crfsuite) though using libpostal's data structures and allowing for "state-transition features." CRFSuite has state features like "word=the", and transition features i.e. "prev tag=house", but no notion of a feature which incorporates both local and transition information e.g. "word=the and prev tag=house". These types of features are useful in our setting where there are many languages and it might not make as much sense to simply have a weight for "house_number => road" because that highly depends on the country. This implementation introduces a T x L^2 matrix for those state-transition scores. For linear-chain CRFs, the Viterbi algorithm is used for computing the most probable sequence. There are versions of Viterbi for computing the N most probable sequences as well, which may come in handy later. This can also compute marginal probabilities of a sequence (though it would need to wait until a gradient-based learning method that produces well-calibrated probabilities is implemented). The cool thing architecturally about crf_context as a separate module is that the weights can be learned through any method we want. As long as the state scores, state-transition scores, and transition scores are populated on the context struct, we have everything we need to run Viterbi inference, etc. without really caring about which training algorithm was used to optimize the weights, what the features are, how they're stored, etc. So far the results have been very encouraging. While it is slower to train a linear-chain CRF, and it will likely add several days to the training process, it's still reasonably fast at runtime and not all that slow at training time. In unscientific tests on a busy MacBook Pro, so far training has been chunking through ~3k addresses / sec, which is only about half the speed of the greedy tagger (haven't benchmarked the runtime difference but anecdotally it's hardly noticeable). Libpostal training runs considerably faster on Linux with gcc, so 3k might be a little low. I'd also guess that re-computing features every iteration means there's a limit on the performance of the greedy tagger. The differences might be more pronounced if features were pre-computed (a possible optimization).
206 lines
6.4 KiB
C
206 lines
6.4 KiB
C
#ifndef CRF_CONTEXT_H
|
|
#define CRF_CONTEXT_H
|
|
|
|
#include <stdlib.h>
|
|
#include <stdint.h>
|
|
#include <stdbool.h>
|
|
|
|
#include "collections.h"
|
|
|
|
#ifdef HAVE_CONFIG_H
|
|
#include <config.h>
|
|
#endif
|
|
|
|
#include <float.h>
|
|
#include <math.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
|
|
#include "collections.h"
|
|
#include "matrix.h"
|
|
|
|
/**
|
|
* Functionality flags for contexts.
|
|
* @see crf_context_new().
|
|
*/
|
|
enum {
|
|
CRF_CONTEXT_BASE = 0x01,
|
|
CRF_CONTEXT_VITERBI = 0x01,
|
|
CRF_CONTEXT_MARGINALS = 0x02,
|
|
CRF_CONTEXT_ALL = 0xFF,
|
|
};
|
|
|
|
/**
|
|
* Reset flags.
|
|
* @see crf_context_reset().
|
|
*/
|
|
|
|
#define CRF_CONTEXT_RESET_STATE (1 << 0)
|
|
#define CRF_CONTEXT_RESET_STATE_TRANS (1 << 1)
|
|
#define CRF_CONTEXT_RESET_ALL ((1 << 16) - 1)
|
|
|
|
|
|
#define CRF_CONTEXT_DEFAULT_NUM_ITEMS 10
|
|
|
|
typedef struct crf_context {
|
|
/**
|
|
* Flag specifying the functionality
|
|
*/
|
|
int flag;
|
|
/**
|
|
* The total number of distinct lables (L)
|
|
*/
|
|
size_t num_labels;
|
|
/**
|
|
* The number of items (T) in the instance
|
|
*/
|
|
size_t num_items;
|
|
/**
|
|
* Logarithm of the normalization factor for the instance.
|
|
* This is equivalent to the total scores of all paths in the lattice.
|
|
*/
|
|
double log_norm;
|
|
|
|
/**
|
|
* State scores.
|
|
* This is a [T][L] matrix whose element [t][l] presents total score
|
|
* of state features associating label l at t
|
|
*/
|
|
double_matrix_t *state;
|
|
|
|
/**
|
|
* State-transition scores.
|
|
* This is a [T][L * L] matrix whose element [t][i * L + j] represents the
|
|
* score of state features associated with label i and j
|
|
*/
|
|
double_matrix_t *state_trans;
|
|
|
|
/**
|
|
* Transition scores.
|
|
* This is a [L][L] matrix whose element [i][j] represents the
|
|
* score of transition features associating labels i and j
|
|
*/
|
|
double_matrix_t *trans;
|
|
|
|
/**
|
|
* Alpha score matrix.
|
|
* This is a [T][L] matrix whose element [t][l] presents the total
|
|
* score of paths starting at BOS and arriving at (t, l).
|
|
*/
|
|
double_matrix_t *alpha_score;
|
|
|
|
/**
|
|
* Beta score matrix.
|
|
* This is a [T][L] matrix whose element [t][l] presents the total
|
|
* score of paths starting at (t, l) and arriving at EOS
|
|
*/
|
|
double_matrix_t *beta_score;
|
|
|
|
/**
|
|
* Scale factor vector.
|
|
* This is a [T] vector whose element [t] presents the scaling
|
|
* coefficient for the alpha score and beta score.
|
|
*/
|
|
double_array *scale_factor;
|
|
|
|
/**
|
|
* Row vector (work space).
|
|
* This is a [T] vector used internally for a work space.
|
|
*/
|
|
double_array *row;
|
|
|
|
/**
|
|
* Row vector for the transitions (work space).
|
|
* This is a [T] vector used internally for a work space.
|
|
*/
|
|
double_array *row_trans;
|
|
|
|
/**
|
|
* This is a [T][L] matrix whose element [t][j] represents the label
|
|
* that yields the maximum score to arrive at (t, j).
|
|
* This member is available only with CRF_CONTEXT_VITERBI flag enabled.
|
|
*/
|
|
uint32_matrix_t *backward_edges;
|
|
|
|
/**
|
|
* Exponents of state scores.
|
|
* This is a [T][L] matrix whose element [t][l] represents the exponent
|
|
* of the total score of state features associating label l at t.
|
|
* This member is available only with CRF_CONTEXT_MARGINALS flag.
|
|
*/
|
|
double_matrix_t *exp_state;
|
|
|
|
/**
|
|
* Exponents of state-transition scores.
|
|
* This is a [T][L * L] matrix whose element [t][i * L + j] represents the
|
|
* exponent of the total score of state features associated with label i and j
|
|
* at t. This member is available only with CRF_CONTEXT_MARGINALS flag.
|
|
*/
|
|
double_matrix_t *exp_state_trans;
|
|
|
|
/**
|
|
* Exponents of transition scores.
|
|
* This is a [L][L] matrix whose element [i][j] represents the exponent
|
|
* of the total score of transition features associating labels i and j.
|
|
* This member is available only with CRF_CONTEXT_MARGINALS flag.
|
|
*/
|
|
double_matrix_t *exp_trans;
|
|
|
|
/**
|
|
* Model expectations of states.
|
|
* This is a [T][L] matrix whose element [t][l] presents the model
|
|
* expectation (marginal probability) of the state (t,l)
|
|
* This member is available only with CRF_CONTEXT_MARGINALS flag.
|
|
*/
|
|
double_matrix_t *mexp_state;
|
|
|
|
/**
|
|
* Model expectations of state transitions.
|
|
* This is a [T][L * L] matrix whose element [t][i * L + j] presents the model
|
|
* expectation (marginal probability) of the state t and transition (i->j)
|
|
* This member is available only with CRF_CONTEXT_MARGINALS flag.
|
|
*/
|
|
double_matrix_t *mexp_state_trans;
|
|
|
|
/**
|
|
* Model expectations of transitions.
|
|
* This is a [L][L] matrix whose element [i][j] presents the model
|
|
* expectation of the transition (i->j).
|
|
* This member is available only with CRF_CONTEXT_MARGINALS flag.
|
|
*/
|
|
double_matrix_t *mexp_trans;
|
|
|
|
} crf_context_t;
|
|
|
|
double *alpha_score(crf_context_t *context, size_t t);
|
|
double *beta_score(crf_context_t *context, size_t t);
|
|
double *state_score(crf_context_t *context, size_t t);
|
|
double *state_trans_score(crf_context_t *context, size_t t, size_t i);
|
|
double *state_trans_score_all(crf_context_t *self, size_t t);
|
|
double *trans_score(crf_context_t *context, size_t i);
|
|
double *exp_state_score(crf_context_t *context, size_t t);
|
|
double *exp_state_trans_score(crf_context_t *context, size_t t, size_t i);
|
|
double *exp_trans_score(crf_context_t *context, size_t i);
|
|
double *state_mexp(crf_context_t *context, size_t t);
|
|
double *state_trans_mexp(crf_context_t *context, size_t t, size_t i);
|
|
double *trans_mexp(crf_context_t *context, size_t i);
|
|
uint32_t *backward_edge_at(crf_context_t *context, size_t t);
|
|
|
|
crf_context_t *crf_context_new(int flag, size_t L, size_t T);
|
|
bool crf_context_set_num_items(crf_context_t *self, size_t T);
|
|
void crf_context_destroy(crf_context_t *self);
|
|
void crf_context_reset(crf_context_t *self, int flag);
|
|
bool crf_context_exp_state(crf_context_t *self);
|
|
bool crf_context_exp_transition(crf_context_t *self);
|
|
|
|
void crf_context_alpha_score(crf_context_t *self);
|
|
void crf_context_beta_score(crf_context_t *self);
|
|
void crf_context_marginals(crf_context_t *self);
|
|
|
|
double crf_context_marginal_point(crf_context_t *self, uint32_t l, uint32_t t);
|
|
double crf_context_marginal_path(crf_context_t *self, const uint32_t *path, size_t begin, size_t end);
|
|
double crf_context_score(crf_context_t *self, const uint32_t *labels);
|
|
double crf_context_lognorm(crf_context_t *self);
|
|
double crf_context_viterbi(crf_context_t *self, uint32_t *labels);
|
|
|
|
#endif |