[merge] merging commit from v1.1

2017-08-14 04:04:58 -06:00
parent bb277fb326
commit 448ca6a61a
10 changed files with 374 additions and 294 deletions
--- a/src/libpostal.c
+++ b/src/libpostal.c
@@ -1137,6 +1137,76 @@ bool libpostal_setup_language_classifier_datadir(char *datadir) {
    return true;
 }

+
+libpostal_token_t *libpostal_tokenize(char *input, bool whitespace, size_t *n) {
+    token_array *tokens = NULL;
+    if (!whitespace) {
+        tokens = tokenize(input);
+    } else {
+        tokens = tokenize_keep_whitespace(input);
+    }
+
+    if (tokens == NULL) {
+        return NULL;
+    }
+
+    libpostal_token_t *a = tokens->a;
+    *n = tokens->n;
+    free(tokens);
+    return a;
+}
+
+char *libpostal_normalize_string(char *str, uint64_t options) {
+    if (options & LIBPOSTAL_NORMALIZE_STRING_LATIN_ASCII) {
+        return normalize_string_latin(str, strlen(str), options);
+    } else {
+        return normalize_string_utf8(str, options);
+    }
+}
+
+libpostal_normalized_token_t *libpostal_normalized_tokens(char *input, uint64_t string_options, uint64_t token_options, bool whitespace, size_t *n) {
+    if (input == NULL) {
+        return NULL;
+    }
+    char *normalized = libpostal_normalize_string(input, string_options);
+    if (normalized == NULL) {
+        return NULL;
+    }
+
+    token_array *tokens = NULL;
+    if (!whitespace) {
+        tokens = tokenize(normalized);
+    } else {
+        tokens = tokenize_keep_whitespace(normalized);
+    }
+
+    if (tokens == NULL || tokens->a == NULL) {
+        free(normalized);
+        return NULL;
+    }
+
+    size_t num_tokens = tokens->n;
+    token_t *token_array = tokens->a;
+    char_array *normalized_token = char_array_new_size(strlen(normalized));
+
+    libpostal_normalized_token_t *result = malloc(sizeof(libpostal_normalized_token_t) * num_tokens);
+
+    for (size_t i = 0; i < num_tokens; i++) {
+        token_t token = token_array[i];
+        char_array_clear(normalized_token);
+        add_normalized_token(normalized_token, normalized, token, token_options);
+        char *token_str = strdup(char_array_get_string(normalized_token));
+        result[i] = (libpostal_normalized_token_t){token_str, token};
+    }
+
+    free(normalized);
+    token_array_destroy(tokens);
+    char_array_destroy(normalized_token);
+
+    *n = num_tokens;
+    return result;
+}
+
 bool libpostal_setup_language_classifier(void) {
    return libpostal_setup_language_classifier_datadir(NULL);
 }
--- a/src/libpostal.h
+++ b/src/libpostal.h
@@ -12,6 +12,67 @@ extern "C" {

 #define LIBPOSTAL_MAX_LANGUAGE_LEN 4

+// Doing these as #defines so we can duplicate the values exactly in Python
+
+
+typedef enum {
+    LIBPOSTAL_TOKEN_TYPE_END = 0,                   // Null byte
+    // Word types
+    LIBPOSTAL_TOKEN_TYPE_WORD = 1,                  // Any letter-only word (includes all unicode letters)
+    LIBPOSTAL_TOKEN_TYPE_ABBREVIATION = 2,          // Loose abbreviations (roughly anything containing a "." as we don't care about sentences in addresses)
+    LIBPOSTAL_TOKEN_TYPE_IDEOGRAPHIC_CHAR = 3,      // For languages that don't separate on whitespace (e.g. Chinese, Japanese, Korean), separate by character
+    LIBPOSTAL_TOKEN_TYPE_HANGUL_SYLLABLE = 4,       // Hangul syllable sequences which contain more than one codepoint
+    LIBPOSTAL_TOKEN_TYPE_ACRONYM = 5,               // Specifically things like U.N. where we may delete internal periods
+
+    LIBPOSTAL_TOKEN_TYPE_PHRASE = 10,               // Not part of the first stage tokenizer, but may be used after phrase parsing
+
+    // Special tokens
+    LIBPOSTAL_TOKEN_TYPE_EMAIL = 20,                // Make sure emails are tokenized altogether
+    LIBPOSTAL_TOKEN_TYPE_URL = 21,                  // Make sure urls are tokenized altogether
+    LIBPOSTAL_TOKEN_TYPE_US_PHONE = 22,             // US phone number (with or without country code)
+    LIBPOSTAL_TOKEN_TYPE_INTL_PHONE = 23,           // A non-US phone number (must have country code)
+
+    // Numbers and numeric types
+    LIBPOSTAL_TOKEN_TYPE_NUMERIC = 50,              // Any sequence containing a digit
+    LIBPOSTAL_TOKEN_TYPE_ORDINAL = 51,              // 1st, 2nd, 1er, 1 etc.
+    LIBPOSTAL_TOKEN_TYPE_ROMAN_NUMERAL = 52,        // II, III, VI, etc.
+    LIBPOSTAL_TOKEN_TYPE_IDEOGRAPHIC_NUMBER = 53,   // All numeric ideographic characters, includes e.g. Han numbers and chars like "²"
+
+    // Punctuation types, may separate a phrase
+    LIBPOSTAL_TOKEN_TYPE_PERIOD = 100,
+    LIBPOSTAL_TOKEN_TYPE_EXCLAMATION = 101,
+    LIBPOSTAL_TOKEN_TYPE_QUESTION_MARK = 102,
+    LIBPOSTAL_TOKEN_TYPE_COMMA = 103,
+    LIBPOSTAL_TOKEN_TYPE_COLON = 104,
+    LIBPOSTAL_TOKEN_TYPE_SEMICOLON = 105,
+    LIBPOSTAL_TOKEN_TYPE_PLUS = 106,
+    LIBPOSTAL_TOKEN_TYPE_AMPERSAND = 107,
+    LIBPOSTAL_TOKEN_TYPE_AT_SIGN = 108,
+    LIBPOSTAL_TOKEN_TYPE_POUND = 109,
+    LIBPOSTAL_TOKEN_TYPE_ELLIPSIS = 110,
+    LIBPOSTAL_TOKEN_TYPE_DASH = 111,
+    LIBPOSTAL_TOKEN_TYPE_BREAKING_DASH = 112,
+    LIBPOSTAL_TOKEN_TYPE_HYPHEN = 113,
+    LIBPOSTAL_TOKEN_TYPE_PUNCT_OPEN = 114,
+    LIBPOSTAL_TOKEN_TYPE_PUNCT_CLOSE = 115,
+    LIBPOSTAL_TOKEN_TYPE_DOUBLE_QUOTE = 119,
+    LIBPOSTAL_TOKEN_TYPE_SINGLE_QUOTE = 120,
+    LIBPOSTAL_TOKEN_TYPE_OPEN_QUOTE = 121,
+    LIBPOSTAL_TOKEN_TYPE_CLOSE_QUOTE = 122,
+    LIBPOSTAL_TOKEN_TYPE_SLASH = 124,
+    LIBPOSTAL_TOKEN_TYPE_BACKSLASH = 125,
+    LIBPOSTAL_TOKEN_TYPE_GREATER_THAN = 126,
+    LIBPOSTAL_TOKEN_TYPE_LESS_THAN = 127,
+
+    // Non-letters and whitespace
+    LIBPOSTAL_TOKEN_TYPE_OTHER = 200,
+    LIBPOSTAL_TOKEN_TYPE_WHITESPACE = 300,
+    LIBPOSTAL_TOKEN_TYPE_NEWLINE = 301,
+
+    LIBPOSTAL_TOKEN_TYPE_INVALID_CHAR = 500
+} libpostal_token_type_t;
+
+
 /* 
 Address dictionaries
 */
@@ -99,6 +160,55 @@ bool libpostal_setup_parser(void);
 bool libpostal_setup_parser_datadir(char *datadir);
 void libpostal_teardown_parser(void);

+typedef struct libpostal_token {
+    size_t offset;
+    size_t len;
+    uint16_t type;
+} libpostal_token_t;
+
+libpostal_token_t *libpostal_tokenize(char *input, bool whitespace, size_t *n);
+
+// Normalize string options
+#define LIBPOSTAL_NORMALIZE_STRING_LATIN_ASCII 1 << 0
+#define LIBPOSTAL_NORMALIZE_STRING_TRANSLITERATE 1 << 1
+#define LIBPOSTAL_NORMALIZE_STRING_STRIP_ACCENTS 1 << 2
+#define LIBPOSTAL_NORMALIZE_STRING_DECOMPOSE 1 << 3
+#define LIBPOSTAL_NORMALIZE_STRING_LOWERCASE 1 << 4
+#define LIBPOSTAL_NORMALIZE_STRING_TRIM 1 << 5
+#define LIBPOSTAL_NORMALIZE_STRING_REPLACE_HYPHENS 1 << 6
+#define LIBPOSTAL_NORMALIZE_STRING_COMPOSE 1 << 7
+#define LIBPOSTAL_NORMALIZE_STRING_SIMPLE_LATIN_ASCII 1 << 8
+#define LIBPOSTAL_NORMALIZE_STRING_REPLACE_NUMEX 1 << 9
+
+// Normalize token options
+#define LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_HYPHENS 1 << 0
+#define LIBPOSTAL_NORMALIZE_TOKEN_DELETE_HYPHENS 1 << 1
+#define LIBPOSTAL_NORMALIZE_TOKEN_DELETE_FINAL_PERIOD 1 << 2
+#define LIBPOSTAL_NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS 1 << 3
+#define LIBPOSTAL_NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES 1 << 4
+#define LIBPOSTAL_NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE 1 << 5
+#define LIBPOSTAL_NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC 1 << 6
+#define LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_DIGITS 1 << 7
+#define LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_NUMERIC_TOKEN_LETTERS 1 << 8
+
+#define LIBPOSTAL_NORMALIZE_DEFAULT_STRING_OPTIONS (LIBPOSTAL_NORMALIZE_STRING_LATIN_ASCII | LIBPOSTAL_NORMALIZE_STRING_COMPOSE | LIBPOSTAL_NORMALIZE_STRING_TRIM | LIBPOSTAL_NORMALIZE_STRING_REPLACE_HYPHENS | LIBPOSTAL_NORMALIZE_STRING_STRIP_ACCENTS | LIBPOSTAL_NORMALIZE_STRING_LOWERCASE)
+
+#define LIBPOSTAL_NORMALIZE_DEFAULT_TOKEN_OPTIONS (LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_HYPHENS | LIBPOSTAL_NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | LIBPOSTAL_NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS | LIBPOSTAL_NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES | LIBPOSTAL_NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE)
+
+#define LIBPOSTAL_NORMALIZE_TOKEN_OPTIONS_DROP_PERIODS (LIBPOSTAL_NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | LIBPOSTAL_NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS)
+
+#define LIBPOSTAL_NORMALIZE_DEFAULT_TOKEN_OPTIONS_NUMERIC (LIBPOSTAL_NORMALIZE_DEFAULT_TOKEN_OPTIONS | LIBPOSTAL_NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC)
+
+char *libpostal_normalize_string(char *input, uint64_t options);
+
+
+typedef struct libpostal_normalized_token {
+    char *str;
+    libpostal_token_t token;
+} libpostal_normalized_token_t;
+
+libpostal_normalized_token_t *libpostal_normalized_tokens(char *input, uint64_t string_options, uint64_t token_options, bool whitespace, size_t *n);
+
 bool libpostal_setup_language_classifier(void);
 bool libpostal_setup_language_classifier_datadir(char *datadir);
 void libpostal_teardown_language_classifier(void);
--- a/src/normalize.h
+++ b/src/normalize.h
@@ -30,6 +30,7 @@ As well as normalizations for individual string tokens:

 #include "constants.h"
 #include "klib/khash.h"
+#include "libpostal.h"
 #include "string_utils.h"
 #include "utf8proc/utf8proc.h"
 #include "unicode_scripts.h"
@@ -39,25 +40,26 @@ As well as normalizations for individual string tokens:
 #include "tokens.h"
 #include "vector.h"

-#define NORMALIZE_STRING_LATIN_ASCII 1 << 0
-#define NORMALIZE_STRING_TRANSLITERATE 1 << 1
-#define NORMALIZE_STRING_STRIP_ACCENTS 1 << 2
-#define NORMALIZE_STRING_DECOMPOSE 1 << 3
-#define NORMALIZE_STRING_LOWERCASE 1 << 4
-#define NORMALIZE_STRING_TRIM 1 << 5
-#define NORMALIZE_STRING_REPLACE_HYPHENS 1 << 6
-#define NORMALIZE_STRING_COMPOSE 1 << 7
-#define NORMALIZE_STRING_SIMPLE_LATIN_ASCII 1 << 8
-#define NORMALIZE_STRING_REPLACE_NUMEX 1 << 9
+#define NORMALIZE_STRING_LATIN_ASCII LIBPOSTAL_NORMALIZE_STRING_LATIN_ASCII
+#define NORMALIZE_STRING_TRANSLITERATE LIBPOSTAL_NORMALIZE_STRING_TRANSLITERATE
+#define NORMALIZE_STRING_STRIP_ACCENTS LIBPOSTAL_NORMALIZE_STRING_STRIP_ACCENTS
+#define NORMALIZE_STRING_DECOMPOSE LIBPOSTAL_NORMALIZE_STRING_DECOMPOSE
+#define NORMALIZE_STRING_LOWERCASE LIBPOSTAL_NORMALIZE_STRING_LOWERCASE
+#define NORMALIZE_STRING_TRIM LIBPOSTAL_NORMALIZE_STRING_TRIM
+#define NORMALIZE_STRING_REPLACE_HYPHENS LIBPOSTAL_NORMALIZE_STRING_REPLACE_HYPHENS
+#define NORMALIZE_STRING_COMPOSE LIBPOSTAL_NORMALIZE_STRING_COMPOSE
+#define NORMALIZE_STRING_SIMPLE_LATIN_ASCII LIBPOSTAL_NORMALIZE_STRING_SIMPLE_LATIN_ASCII
+#define NORMALIZE_STRING_REPLACE_NUMEX LIBPOSTAL_NORMALIZE_STRING_REPLACE_NUMEX

-#define NORMALIZE_TOKEN_REPLACE_HYPHENS 1 << 0
-#define NORMALIZE_TOKEN_DELETE_HYPHENS 1 << 1
-#define NORMALIZE_TOKEN_DELETE_FINAL_PERIOD 1 << 2
-#define NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS 1 << 3
-#define NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES 1 << 4
-#define NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE 1 << 5
-#define NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC 1 << 6
-#define NORMALIZE_TOKEN_REPLACE_DIGITS 1 << 7
+#define NORMALIZE_TOKEN_REPLACE_HYPHENS LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_HYPHENS
+#define NORMALIZE_TOKEN_DELETE_HYPHENS LIBPOSTAL_NORMALIZE_TOKEN_DELETE_HYPHENS
+#define NORMALIZE_TOKEN_DELETE_FINAL_PERIOD LIBPOSTAL_NORMALIZE_TOKEN_DELETE_FINAL_PERIOD
+#define NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS LIBPOSTAL_NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS
+#define NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES LIBPOSTAL_NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES
+#define NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE LIBPOSTAL_NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE
+#define NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC LIBPOSTAL_NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC
+#define NORMALIZE_TOKEN_REPLACE_DIGITS LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_DIGITS
+#define NORMALIZE_TOKEN_REPLACE_NUMERIC_TOKEN_LETTERS LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_NUMERIC_TOKEN_LETTERS

 // Replace digits with capital D e.g. 10013 => DDDDD, intended for use with lowercased strings
 #define DIGIT_CHAR "D"
--- a/src/token_types.h
+++ b/src/token_types.h
@@ -1,64 +1,60 @@
 #ifndef TOKEN_TYPES_H
 #define TOKEN_TYPES_H

+#include "libpostal.h"
+
 // Doing these as #defines so we can duplicate the values exactly in Python

-#define END 0                   // Null byte

-// Word types
-#define WORD 1                  // Any letter-only word (includes all unicode letters)
-#define ABBREVIATION 2          // Loose abbreviations (roughly anything containing a "." as we don't care about sentences in addresses)
-#define IDEOGRAPHIC_CHAR 3      // For languages that don't separate on whitespace (e.g. Chinese, Japanese, Korean), separate by character
-#define HANGUL_SYLLABLE 4       // Hangul syllable sequences which contain more than one codepoint
-#define ACRONYM 5               // Specifically things like U.N. where we may delete internal periods
+#define END LIBPOSTAL_TOKEN_TYPE_END

-#define PHRASE 10               // Not part of the first stage tokenizer, but may be used after phrase parsing
+#define WORD LIBPOSTAL_TOKEN_TYPE_WORD
+#define ABBREVIATION LIBPOSTAL_TOKEN_TYPE_ABBREVIATION
+#define IDEOGRAPHIC_CHAR LIBPOSTAL_TOKEN_TYPE_IDEOGRAPHIC_CHAR
+#define HANGUL_SYLLABLE LIBPOSTAL_TOKEN_TYPE_HANGUL_SYLLABLE
+#define ACRONYM LIBPOSTAL_TOKEN_TYPE_ACRONYM
+#define PHRASE LIBPOSTAL_TOKEN_TYPE_PHRASE

-// Special tokens
-#define EMAIL 20                // Make sure emails are tokenized altogether
-#define URL 21                  // Make sure urls are tokenized altogether
-#define US_PHONE 22             // US phone number (with or without country code)
-#define INTL_PHONE 23           // A non-US phone number (must have country code)
+#define EMAIL LIBPOSTAL_TOKEN_TYPE_EMAIL
+#define URL LIBPOSTAL_TOKEN_TYPE_URL
+#define US_PHONE LIBPOSTAL_TOKEN_TYPE_US_PHONE
+#define INTL_PHONE LIBPOSTAL_TOKEN_TYPE_INTL_PHONE

-// Numbers and numeric types
-#define NUMERIC 50              // Any sequence containing a digit
-#define ORDINAL 51              // 1st, 2nd, 1er, 1 etc.
-#define ROMAN_NUMERAL 52        // II, III, VI, etc.
-#define IDEOGRAPHIC_NUMBER 53   // All numeric ideographic characters, includes e.g. Han numbers and chars like "²"
+#define NUMERIC LIBPOSTAL_TOKEN_TYPE_NUMERIC
+#define ORDINAL LIBPOSTAL_TOKEN_TYPE_ORDINAL
+#define ROMAN_NUMERAL LIBPOSTAL_TOKEN_TYPE_ROMAN_NUMERAL
+#define IDEOGRAPHIC_NUMBER LIBPOSTAL_TOKEN_TYPE_IDEOGRAPHIC_NUMBER

+#define PERIOD LIBPOSTAL_TOKEN_TYPE_PERIOD
+#define EXCLAMATION LIBPOSTAL_TOKEN_TYPE_EXCLAMATION
+#define QUESTION_MARK LIBPOSTAL_TOKEN_TYPE_QUESTION_MARK
+#define COMMA LIBPOSTAL_TOKEN_TYPE_COMMA
+#define COLON LIBPOSTAL_TOKEN_TYPE_COLON
+#define SEMICOLON LIBPOSTAL_TOKEN_TYPE_SEMICOLON
+#define PLUS LIBPOSTAL_TOKEN_TYPE_PLUS
+#define AMPERSAND LIBPOSTAL_TOKEN_TYPE_AMPERSAND
+#define AT_SIGN LIBPOSTAL_TOKEN_TYPE_AT_SIGN
+#define POUND LIBPOSTAL_TOKEN_TYPE_POUND
+#define ELLIPSIS LIBPOSTAL_TOKEN_TYPE_ELLIPSIS
+#define DASH LIBPOSTAL_TOKEN_TYPE_DASH
+#define BREAKING_DASH LIBPOSTAL_TOKEN_TYPE_BREAKING_DASH
+#define HYPHEN LIBPOSTAL_TOKEN_TYPE_HYPHEN
+#define PUNCT_OPEN LIBPOSTAL_TOKEN_TYPE_PUNCT_OPEN
+#define PUNCT_CLOSE LIBPOSTAL_TOKEN_TYPE_PUNCT_CLOSE
+#define DOUBLE_QUOTE LIBPOSTAL_TOKEN_TYPE_DOUBLE_QUOTE
+#define SINGLE_QUOTE LIBPOSTAL_TOKEN_TYPE_SINGLE_QUOTE
+#define OPEN_QUOTE LIBPOSTAL_TOKEN_TYPE_OPEN_QUOTE
+#define CLOSE_QUOTE LIBPOSTAL_TOKEN_TYPE_CLOSE_QUOTE
+#define SLASH LIBPOSTAL_TOKEN_TYPE_SLASH
+#define BACKSLASH LIBPOSTAL_TOKEN_TYPE_BACKSLASH
+#define GREATER_THAN LIBPOSTAL_TOKEN_TYPE_GREATER_THAN
+#define LESS_THAN LIBPOSTAL_TOKEN_TYPE_LESS_THAN

-// Punctuation types, may separate a phrase
-#define PERIOD 100
-#define EXCLAMATION 101
-#define QUESTION_MARK 102
-#define COMMA 103
-#define COLON 104
-#define SEMICOLON 105
-#define PLUS 106
-#define AMPERSAND 107
-#define AT_SIGN 108
-#define POUND 109
-#define ELLIPSIS 110
-#define DASH 111
-#define BREAKING_DASH 112
-#define HYPHEN 113
-#define PUNCT_OPEN 114
-#define PUNCT_CLOSE 115
-#define DOUBLE_QUOTE 119
-#define SINGLE_QUOTE 120
-#define OPEN_QUOTE 121
-#define CLOSE_QUOTE 122
-#define SLASH 124
-#define BACKSLASH 125
-#define GREATER_THAN 126
-#define LESS_THAN 127
+#define OTHER LIBPOSTAL_TOKEN_TYPE_OTHER
+#define WHITESPACE LIBPOSTAL_TOKEN_TYPE_WHITESPACE
+#define NEWLINE LIBPOSTAL_TOKEN_TYPE_NEWLINE

-// Non-letters and whitespace
-#define OTHER 200
-#define WHITESPACE 300
-#define NEWLINE 301
-
-#define INVALID_CHAR 500
+#define INVALID_CHAR LIBPOSTAL_TOKEN_TYPE_INVALID_CHAR


 #define is_word_token(type) ((type) == WORD || (type) == ABBREVIATION || (type) == ACRONYM || (type) == IDEOGRAPHIC_CHAR || (type) == HANGUL_SYLLABLE)
--- a/src/tokens.h
+++ b/src/tokens.h
@@ -12,11 +12,7 @@
 #include "token_types.h"
 #include "vector.h"

-typedef struct token {
-    size_t offset;
-    size_t len;
-    uint16_t type;
-} token_t;
+typedef libpostal_token_t token_t;

 VECTOR_INIT(token_array, token_t)