tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[merge] merging commit from v1.1

Browse Source
This commit is contained in:
Al
2017-08-14 04:04:58 -06:00
parent bb277fb326
commit 448ca6a61a
10 changed files with 374 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
scripts/geodata/text/normalize.py
View File

@@ -2,7 +2,6 @@
import six import six
from geodata.text import _normalize from geodata.text import _normalize
from geodata.text.tokenize import tokenize_raw
from geodata.text.token_types import token_types from geodata.text.token_types import token_types
from geodata.encoding import safe_decode from geodata.encoding import safe_decode
@@ -17,12 +16,7 @@ NORMALIZE_STRING_TRIM = _normalize.NORMALIZE_STRING_TRIM
NORMALIZE_STRING_REPLACE_HYPHENS = _normalize.NORMALIZE_STRING_REPLACE_HYPHENS NORMALIZE_STRING_REPLACE_HYPHENS = _normalize.NORMALIZE_STRING_REPLACE_HYPHENS
NORMALIZE_STRING_SIMPLE_LATIN_ASCII = _normalize.NORMALIZE_STRING_SIMPLE_LATIN_ASCII NORMALIZE_STRING_SIMPLE_LATIN_ASCII = _normalize.NORMALIZE_STRING_SIMPLE_LATIN_ASCII
DEFAULT_STRING_OPTIONS = NORMALIZE_STRING_LATIN_ASCII | \ DEFAULT_STRING_OPTIONS = _normalize.NORMALIZE_DEFAULT_STRING_OPTIONS
NORMALIZE_STRING_DECOMPOSE | \
NORMALIZE_STRING_TRIM | \
NORMALIZE_STRING_REPLACE_HYPHENS | \
NORMALIZE_STRING_STRIP_ACCENTS | \
NORMALIZE_STRING_LOWERCASE
# Token options # Token options
NORMALIZE_TOKEN_REPLACE_HYPHENS = _normalize.NORMALIZE_TOKEN_REPLACE_HYPHENS NORMALIZE_TOKEN_REPLACE_HYPHENS = _normalize.NORMALIZE_TOKEN_REPLACE_HYPHENS
@@ -34,16 +28,10 @@ NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE = _normalize.NORMALIZE_TOKEN_DELETE_OTHE
NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC = _normalize.NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC = _normalize.NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC
NORMALIZE_TOKEN_REPLACE_DIGITS = _normalize.NORMALIZE_TOKEN_REPLACE_DIGITS NORMALIZE_TOKEN_REPLACE_DIGITS = _normalize.NORMALIZE_TOKEN_REPLACE_DIGITS
DEFAULT_TOKEN_OPTIONS = NORMALIZE_TOKEN_REPLACE_HYPHENS | \ DEFAULT_TOKEN_OPTIONS = _normalize.NORMALIZE_DEFAULT_TOKEN_OPTIONS
NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | \
NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS | \
NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES | \
NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE
TOKEN_OPTIONS_DROP_PERIODS = NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | \ TOKEN_OPTIONS_DROP_PERIODS = _normalize.NORMALIZE_TOKEN_OPTIONS_DROP_PERIODS
NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS DEFAULT_TOKEN_OPTIONS_NUMERIC = _normalize.NORMALIZE_DEFAULT_TOKEN_OPTIONS_NUMERIC
DEFAULT_TOKEN_OPTIONS_NUMERIC = (DEFAULT_TOKEN_OPTIONS | NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC)
def remove_parens(tokens): def remove_parens(tokens):
@@ -62,33 +50,7 @@ def remove_parens(tokens):
def normalize_string(s, string_options=DEFAULT_STRING_OPTIONS): def normalize_string(s, string_options=DEFAULT_STRING_OPTIONS):
s = safe_decode(s) s = safe_decode(s)
if string_options & _normalize.NORMALIZE_STRING_LATIN_ASCII: return _normalize.normalize_string(s, string_options)
normalized = _normalize.normalize_string_latin(s, string_options)
else:
normalized = _normalize.normalize_string_utf8(s, string_options)
return normalized
def normalize_token(s, t, token_options=DEFAULT_TOKEN_OPTIONS):
return _normalize.normalize_token(s, t, token_options)
def normalize_tokens_whitespace(s, raw_tokens, token_options=DEFAULT_TOKEN_OPTIONS):
last_end = 0
tokens = []
for t in raw_tokens:
t_norm = _normalize.normalize_token(s, t, token_options)
t_class = token_types.from_id(t[-1])
if last_end < t[0]:
tokens.append((six.u(' '), token_types.WHITESPACE))
last_end = sum(t[:2])
tokens.append((t_norm, t_class))
return tokens
def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS, def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS,
@@ -105,20 +67,10 @@ def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS,
Usage: Usage:
normalized_tokens(u'St.-Barthélemy') normalized_tokens(u'St.-Barthélemy')
''' '''
normalized = normalize_string(s, string_options=string_options) s = safe_decode(s)
normalized_tokens = _normalize.normalized_tokens(s, string_options, token_options, whitespace)
# Tuples of (offset, len, type)
raw_tokens = tokenize_raw(normalized)
tokens = []
last_end = 0
if not whitespace:
tokens = [(_normalize.normalize_token(normalized, t, token_options),
token_types.from_id(t[-1])) for t in raw_tokens]
else:
tokens = normalize_tokens_whitespace(normalized, raw_tokens, token_options=token_options)
if strip_parentheticals: if strip_parentheticals:
return remove_parens(tokens) normalized_tokens = remove_parens(normalized_tokens)
else:
return tokens return [(s, token_types.from_id(token_type)) for s, token_type in normalized_tokens]

200
scripts/geodata/text/pynormalize.c
View File

@@ -1,7 +1,6 @@
#include <Python.h> #include <Python.h>
#include "src/normalize.h" #include <libpostal/libpostal.h>
#include "src/transliterate.h"
#if PY_MAJOR_VERSION >= 3 #if PY_MAJOR_VERSION >= 3
#define IS_PY3K #define IS_PY3K
@@ -19,9 +18,7 @@ struct module_state {
static struct module_state _state; static struct module_state _state;
#endif #endif
static PyObject *py_normalize_string(PyObject *self, PyObject *args)
static PyObject *py_normalize_string_utf8(PyObject *self, PyObject *args)
{ {
PyObject *arg1; PyObject *arg1;
uint64_t options; uint64_t options;
@@ -48,7 +45,7 @@ static PyObject *py_normalize_string_utf8(PyObject *self, PyObject *args)
if (str == NULL) { if (str == NULL) {
PyErr_SetString(PyExc_TypeError, PyErr_SetString(PyExc_TypeError,
"Parameter could not be utf-8 encoded"); "Parameter could not be utf-8 encoded");
goto exit_decref_unistr; goto exit_normalize_decref_unistr;
} }
char *input = PyBytes_AsString(str); char *input = PyBytes_AsString(str);
@@ -56,13 +53,13 @@ static PyObject *py_normalize_string_utf8(PyObject *self, PyObject *args)
#endif #endif
if (input == NULL) { if (input == NULL) {
goto exit_decref_str; goto exit_normalize_decref_str;
} }
char *normalized = normalize_string_utf8(input, options); char *normalized = libpostal_normalize_string(input, options);
if (normalized == NULL) { if (normalized == NULL) {
goto exit_decref_str; goto exit_normalize_decref_str;
} }
PyObject *result = PyUnicode_DecodeUTF8((const char *)normalized, strlen(normalized), "strict"); PyObject *result = PyUnicode_DecodeUTF8((const char *)normalized, strlen(normalized), "strict");
@@ -70,7 +67,7 @@ static PyObject *py_normalize_string_utf8(PyObject *self, PyObject *args)
if (result == NULL) { if (result == NULL) {
PyErr_SetString(PyExc_ValueError, PyErr_SetString(PyExc_ValueError,
"Result could not be utf-8 decoded"); "Result could not be utf-8 decoded");
goto exit_decref_str; goto exit_normalize_decref_str;
} }
#ifndef IS_PY3K #ifndef IS_PY3K
@@ -80,21 +77,26 @@ static PyObject *py_normalize_string_utf8(PyObject *self, PyObject *args)
return result; return result;
exit_decref_str: exit_normalize_decref_str:
#ifndef IS_PY3K #ifndef IS_PY3K
Py_XDECREF(str); Py_XDECREF(str);
#endif #endif
exit_decref_unistr: exit_normalize_decref_unistr:
Py_XDECREF(unistr); Py_XDECREF(unistr);
return 0; return 0;
} }
static PyObject *py_normalize_string_latin(PyObject *self, PyObject *args) static PyObject *py_normalized_tokens(PyObject *self, PyObject *args)
{ {
PyObject *arg1; PyObject *arg1;
uint64_t options; uint64_t string_options = LIBPOSTAL_NORMALIZE_DEFAULT_STRING_OPTIONS;
if (!PyArg_ParseTuple(args, "OK:normalize", &arg1, &options)) { uint64_t token_options = LIBPOSTAL_NORMALIZE_DEFAULT_TOKEN_OPTIONS;
uint32_t arg_whitespace = 0;
PyObject *result = NULL;
if (!PyArg_ParseTuple(args, "O|KKI:normalize", &arg1, &string_options, &token_options, &arg_whitespace)) {
return 0; return 0;
} }
@@ -117,7 +119,7 @@ static PyObject *py_normalize_string_latin(PyObject *self, PyObject *args)
if (str == NULL) { if (str == NULL) {
PyErr_SetString(PyExc_TypeError, PyErr_SetString(PyExc_TypeError,
"Parameter could not be utf-8 encoded"); "Parameter could not be utf-8 encoded");
goto exit_decref_unistr; goto exit_normalized_tokens_decref_str;
} }
char *input = PyBytes_AsString(str); char *input = PyBytes_AsString(str);
@@ -125,98 +127,46 @@ static PyObject *py_normalize_string_latin(PyObject *self, PyObject *args)
#endif #endif
if (input == NULL) { if (input == NULL) {
goto exit_decref_str; goto exit_normalized_tokens_decref_str;
} }
char *normalized = normalize_string_latin(input, strlen(input), options); bool whitespace = arg_whitespace;
PyObject *result = PyUnicode_DecodeUTF8((const char *)normalized, strlen(normalized), "strict"); size_t num_tokens;
free(normalized); libpostal_normalized_token_t *normalized_tokens = libpostal_normalized_tokens(input, string_options, token_options, whitespace, &num_tokens);
if (result == NULL) {
PyErr_SetString(PyExc_ValueError, if (normalized_tokens == NULL) {
"Result could not be utf-8 decoded"); goto exit_normalized_tokens_decref_str;
goto exit_decref_str;
} }
#ifndef IS_PY3K result = PyList_New((Py_ssize_t)num_tokens);
Py_XDECREF(str); if (!result) {
#endif goto exit_free_normalized_tokens;
Py_XDECREF(unistr);
return result;
exit_decref_str:
#ifndef IS_PY3K
Py_XDECREF(str);
#endif
exit_decref_unistr:
Py_XDECREF(unistr);
return 0;
}
static PyObject *py_normalize_token(PyObject *self, PyObject *args)
{
PyObject *s;
uint32_t offset;
uint32_t len;
uint16_t type;
uint64_t options;
if (!PyArg_ParseTuple(args, "O(IIH)K:normalize", &s, &offset, &len, &type, &options)) {
PyErr_SetString(PyExc_TypeError,
"Error parsing arguments");
return 0;
} }
token_t token = (token_t){(size_t)offset, (size_t)len, type}; for (size_t i = 0; i < num_tokens; i++) {
libpostal_normalized_token_t normalized_token = normalized_tokens[i];
PyObject *unistr = PyUnicode_FromObject(s); char *token_str = normalized_token.str;
if (unistr == NULL) { PyObject *py_token = PyUnicode_DecodeUTF8((const char *)token_str, strlen(token_str), "strict");
PyErr_SetString(PyExc_TypeError, if (py_token == NULL) {
"Parameter could not be converted to unicode in scanner"); Py_DECREF(result);
return 0; goto exit_free_normalized_tokens;
}
#ifdef IS_PY3K
// Python 3 encoding, supported by Python 3.3+
char *input = PyUnicode_AsUTF8(unistr);
#else
// Python 2 encoding
PyObject *str = PyUnicode_AsEncodedString(unistr, "utf-8", "strict");
if (str == NULL) {
PyErr_SetString(PyExc_ValueError,
"Parameter could not be utf-8 encoded");
goto exit_decref_unistr;
} }
char *input = PyBytes_AsString(str); PyObject *t = PyTuple_New(2);
PyObject *py_token_type = PyInt_FromLong(normalized_token.token.type);
#endif PyTuple_SetItem(t, 0, py_token);
PyTuple_SetItem(t, 1, py_token_type);
if (input == NULL) { // Note: PyList_SetItem steals a reference, so don't worry about DECREF
goto exit_decref_str; PyList_SetItem(result, (Py_ssize_t)i, t);
} }
char_array *token_buffer = char_array_new_size(token.len); for (size_t i = 0; i < num_tokens; i++) {
free(normalized_tokens[i].str);
add_normalized_token(token_buffer, input, token, options);
char *token_str = char_array_get_string(token_buffer);
PyObject *result = PyUnicode_DecodeUTF8((const char *)token_str, token_buffer->n - 1, "strict");
if (result == NULL) {
PyErr_SetString(PyExc_ValueError,
"Error decoding token");
char_array_destroy(token_buffer);
goto exit_decref_str;
} }
free(normalized_tokens);
char_array_destroy(token_buffer);
#ifndef IS_PY3K #ifndef IS_PY3K
Py_XDECREF(str); Py_XDECREF(str);
@@ -224,20 +174,24 @@ static PyObject *py_normalize_token(PyObject *self, PyObject *args)
Py_XDECREF(unistr); Py_XDECREF(unistr);
return result; return result;
exit_free_normalized_tokens:
exit_decref_str: for (size_t i = 0; i < num_tokens; i++) {
free(normalized_tokens[i].str);
}
free(normalized_tokens);
exit_normalized_tokens_decref_str:
#ifndef IS_PY3K #ifndef IS_PY3K
Py_XDECREF(str); Py_XDECREF(str);
#endif #endif
exit_decref_unistr: exit_normalized_tokens_decref_unistr:
Py_XDECREF(unistr); Py_XDECREF(unistr);
return 0; return 0;
} }
static PyMethodDef normalize_methods[] = { static PyMethodDef normalize_methods[] = {
{"normalize_string_utf8", (PyCFunction)py_normalize_string_utf8, METH_VARARGS, "normalize_string_utf8(input, options)"}, {"normalize_string", (PyCFunction)py_normalize_string, METH_VARARGS, "normalize_string(input, options)"},
{"normalize_string_latin", (PyCFunction)py_normalize_string_latin, METH_VARARGS, "normalize_string_latin(input, options)"}, {"normalized_tokens", (PyCFunction)py_normalized_tokens, METH_VARARGS, "normalize_token(input, string_options, token_options, whitespace)"},
{"normalize_token", (PyCFunction)py_normalize_token, METH_VARARGS, "normalize_token(input, options)"},
{NULL, NULL}, {NULL, NULL},
}; };
@@ -295,32 +249,40 @@ init_normalize(void) {
INITERROR; INITERROR;
} }
if (!transliteration_module_setup(NULL)) { if (!libpostal_setup()) {
PyErr_SetString(PyExc_RuntimeError, PyErr_SetString(PyExc_RuntimeError,
"Could not load transliterate module"); "Could not load libpostal");
Py_DECREF(module); Py_DECREF(module);
INITERROR; INITERROR;
} }
PyModule_AddObject(module, "NORMALIZE_STRING_LATIN_ASCII", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_LATIN_ASCII)); PyModule_AddObject(module, "NORMALIZE_STRING_LATIN_ASCII", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_STRING_LATIN_ASCII));
PyModule_AddObject(module, "NORMALIZE_STRING_TRANSLITERATE", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_TRANSLITERATE)); PyModule_AddObject(module, "NORMALIZE_STRING_TRANSLITERATE", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_STRING_TRANSLITERATE));
PyModule_AddObject(module, "NORMALIZE_STRING_STRIP_ACCENTS", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_STRIP_ACCENTS)); PyModule_AddObject(module, "NORMALIZE_STRING_STRIP_ACCENTS", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_STRING_STRIP_ACCENTS));
PyModule_AddObject(module, "NORMALIZE_STRING_DECOMPOSE", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_DECOMPOSE)); PyModule_AddObject(module, "NORMALIZE_STRING_DECOMPOSE", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_STRING_DECOMPOSE));
PyModule_AddObject(module, "NORMALIZE_STRING_COMPOSE", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_COMPOSE)); PyModule_AddObject(module, "NORMALIZE_STRING_COMPOSE", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_STRING_COMPOSE));
PyModule_AddObject(module, "NORMALIZE_STRING_LOWERCASE", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_LOWERCASE)); PyModule_AddObject(module, "NORMALIZE_STRING_LOWERCASE", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_STRING_LOWERCASE));
PyModule_AddObject(module, "NORMALIZE_STRING_TRIM", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_TRIM)); PyModule_AddObject(module, "NORMALIZE_STRING_TRIM", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_STRING_TRIM));
PyModule_AddObject(module, "NORMALIZE_STRING_REPLACE_HYPHENS", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_REPLACE_HYPHENS)); PyModule_AddObject(module, "NORMALIZE_STRING_REPLACE_HYPHENS", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_STRING_REPLACE_HYPHENS));
PyModule_AddObject(module, "NORMALIZE_STRING_SIMPLE_LATIN_ASCII", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_SIMPLE_LATIN_ASCII)); PyModule_AddObject(module, "NORMALIZE_STRING_SIMPLE_LATIN_ASCII", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_STRING_SIMPLE_LATIN_ASCII));
PyModule_AddObject(module, "NORMALIZE_TOKEN_REPLACE_HYPHENS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_REPLACE_HYPHENS)); PyModule_AddObject(module, "NORMALIZE_TOKEN_REPLACE_HYPHENS", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_HYPHENS));
PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_HYPHENS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_HYPHENS)); PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_HYPHENS", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_TOKEN_DELETE_HYPHENS));
PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_FINAL_PERIOD", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_FINAL_PERIOD)); PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_FINAL_PERIOD", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_TOKEN_DELETE_FINAL_PERIOD));
PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS)); PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS));
PyModule_AddObject(module, "NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES)); PyModule_AddObject(module, "NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES));
PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE)); PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE));
PyModule_AddObject(module, "NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC)); PyModule_AddObject(module, "NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC));
PyModule_AddObject(module, "NORMALIZE_TOKEN_REPLACE_DIGITS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_REPLACE_DIGITS)); PyModule_AddObject(module, "NORMALIZE_TOKEN_REPLACE_DIGITS", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_DIGITS));
PyModule_AddObject(module, "NORMALIZE_DEFAULT_STRING_OPTIONS", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_DEFAULT_STRING_OPTIONS));
PyModule_AddObject(module, "NORMALIZE_DEFAULT_TOKEN_OPTIONS", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_DEFAULT_TOKEN_OPTIONS));
PyModule_AddObject(module, "NORMALIZE_TOKEN_OPTIONS_DROP_PERIODS", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_TOKEN_OPTIONS_DROP_PERIODS));
PyModule_AddObject(module, "NORMALIZE_DEFAULT_TOKEN_OPTIONS_NUMERIC", PyLong_FromUnsignedLongLong(LIBPOSTAL_NORMALIZE_DEFAULT_TOKEN_OPTIONS_NUMERIC));
#if PY_MAJOR_VERSION >= 3 #if PY_MAJOR_VERSION >= 3

31
scripts/geodata/text/pytokenize.c
View File

@@ -1,6 +1,6 @@
#include <Python.h> #include <Python.h>
#include "src/scanner.h" #include <libpostal/libpostal.h>
#if PY_MAJOR_VERSION >= 3 #if PY_MAJOR_VERSION >= 3
#define IS_PY3K #define IS_PY3K
@@ -18,14 +18,17 @@ struct module_state {
static struct module_state _state; static struct module_state _state;
#endif #endif
static PyObject *py_tokenize(PyObject *self, PyObject *args) static PyObject *py_tokenize(PyObject *self, PyObject *args)
{ {
PyObject *arg1; PyObject *arg1;
if (!PyArg_ParseTuple(args, "O:tokenize", &arg1)) { uint32_t arg_whitespace = 0;
if (!PyArg_ParseTuple(args, "OI:tokenize", &arg1, &arg_whitespace)) {
return 0; return 0;
} }
bool whitespace = arg_whitespace;
PyObject *unistr = PyUnicode_FromObject(arg1); PyObject *unistr = PyUnicode_FromObject(arg1);
if (unistr == NULL) { if (unistr == NULL) {
PyErr_SetString(PyExc_TypeError, PyErr_SetString(PyExc_TypeError,
@@ -57,26 +60,28 @@ static PyObject *py_tokenize(PyObject *self, PyObject *args)
goto error_decref_str; goto error_decref_str;
} }
token_array *tokens = tokenize(input); size_t num_tokens;
libpostal_token_t *tokens = libpostal_tokenize(input, whitespace, &num_tokens);
if (tokens == NULL) { if (tokens == NULL) {
goto error_decref_str; goto error_decref_str;
} }
PyObject *result = PyTuple_New(tokens->n); PyObject *result = PyTuple_New(num_tokens);
if (!result) { if (!result) {
token_array_destroy(tokens); free(tokens);
goto error_decref_str; goto error_decref_str;
return 0; return 0;
} }
PyObject *tuple; PyObject *tuple;
token_t token; libpostal_token_t token;
for (size_t i = 0; i < tokens->n; i++) { for (size_t i = 0; i < num_tokens; i++) {
token = tokens->a[i]; token = tokens[i];
tuple = Py_BuildValue("III", token.offset, token.len, token.type); tuple = Py_BuildValue("III", token.offset, token.len, token.type);
if (PyTuple_SetItem(result, i, tuple) < 0) { if (PyTuple_SetItem(result, i, tuple) < 0) {
token_array_destroy(tokens); free(tokens);
goto error_decref_str; goto error_decref_str;
} }
} }
@@ -86,7 +91,7 @@ static PyObject *py_tokenize(PyObject *self, PyObject *args)
#endif #endif
Py_XDECREF(unistr); Py_XDECREF(unistr);
token_array_destroy(tokens); free(tokens);
return result; return result;
@@ -100,12 +105,10 @@ error_decref_unistr:
} }
static PyMethodDef tokenize_methods[] = { static PyMethodDef tokenize_methods[] = {
{"tokenize", (PyCFunction)py_tokenize, METH_VARARGS, "tokenize(text)"}, {"tokenize", (PyCFunction)py_tokenize, METH_VARARGS, "tokenize(text, whitespace)"},
{NULL, NULL}, {NULL, NULL},
}; };
#ifdef IS_PY3K #ifdef IS_PY3K
static int tokenize_traverse(PyObject *m, visitproc visit, void *arg) { static int tokenize_traverse(PyObject *m, visitproc visit, void *arg) {

7
scripts/geodata/text/tokenize.py
View File

@@ -3,12 +3,9 @@ from geodata.text import _tokenize
from geodata.text.token_types import token_types from geodata.text.token_types import token_types
def tokenize_raw(s):
return _tokenize.tokenize(safe_decode(s))
def tokenize(s, whitespace=False):
def tokenize(s):
u = safe_decode(s) u = safe_decode(s)
s = safe_encode(s) s = safe_encode(s)
return [(safe_decode(s[start:start + length]), token_types.from_id(token_type)) return [(safe_decode(s[start:start + length]), token_types.from_id(token_type))
for start, length, token_type in _tokenize.tokenize(u)] for start, length, token_type in _tokenize.tokenize(u, whitespace)]

44
scripts/setup.py
View File

@@ -2,9 +2,7 @@ import os
from setuptools import setup, Extension, find_packages from setuptools import setup, Extension, find_packages
this_dir = os.path.dirname(__file__) RESOURCES_DIR = 'resources'
PROJECT_DIR = os.path.join(this_dir, os.pardir)
SRC_DIR = os.path.join(PROJECT_DIR, 'src')
def main(): def main():
@@ -14,35 +12,29 @@ def main():
packages=find_packages(), packages=find_packages(),
ext_modules=[ ext_modules=[
Extension('geodata.text._tokenize', Extension('geodata.text._tokenize',
sources=[os.path.join(SRC_DIR, f) sources=['geodata/text/pytokenize.c'],
for f in ('scanner.c', libraries=['postal'],
'string_utils.c', include_dirs=['/usr/local/include'],
'tokens.c', library_dirs=['/usr/local/lib'],
'utf8proc/utf8proc.c', extra_compile_args=['-std=c99',
)
] + ['geodata/text/pytokenize.c'],
include_dirs=[PROJECT_DIR],
extra_compile_args=['-O0', '-std=gnu99',
'-Wno-unused-function'], '-Wno-unused-function'],
), ),
Extension('geodata.text._normalize', Extension('geodata.text._normalize',
sources=[os.path.join(SRC_DIR, f) sources=['geodata/text/pynormalize.c'],
for f in ('normalize.c', libraries=['postal'],
'string_utils.c', include_dirs=['/usr/local/include'],
'utf8proc/utf8proc.c', library_dirs=['/usr/local/lib'],
'tokens.c', extra_compile_args=['-std=c99',
'unicode_scripts.c',
'transliterate.c',
'file_utils.c',
'trie.c',
'trie_search.c',)
] + ['geodata/text/pynormalize.c'],
include_dirs=[PROJECT_DIR],
extra_compile_args=['-std=gnu99', '-DHAVE_CONFIG_H',
'-DLIBPOSTAL_DATA_DIR="{}"'.format(os.getenv('LIBPOSTAL_DATA_DIR', os.path.realpath(os.path.join(PROJECT_DIR, 'data')))),
'-Wno-unused-function'], '-Wno-unused-function'],
), ),
], ],
data_files=[
(os.path.join(RESOURCES_DIR, os.path.relpath(d, RESOURCES_DIR)), [os.path.join(d, filename) for filename in filenames])
for d, _, filenames in os.walk(RESOURCES_DIR)
],
package_data={
'geodata': ['**/*.sh']
},
include_package_data=True, include_package_data=True,
zip_safe=False, zip_safe=False,
url='http://mapzen.com', url='http://mapzen.com',

70
src/libpostal.c
View File

@@ -1137,6 +1137,76 @@ bool libpostal_setup_language_classifier_datadir(char *datadir) {
return true; return true;
} }
libpostal_token_t *libpostal_tokenize(char *input, bool whitespace, size_t *n) {
token_array *tokens = NULL;
if (!whitespace) {
tokens = tokenize(input);
} else {
tokens = tokenize_keep_whitespace(input);
}
if (tokens == NULL) {
return NULL;
}
libpostal_token_t *a = tokens->a;
*n = tokens->n;
free(tokens);
return a;
}
char *libpostal_normalize_string(char *str, uint64_t options) {
if (options & LIBPOSTAL_NORMALIZE_STRING_LATIN_ASCII) {
return normalize_string_latin(str, strlen(str), options);
} else {
return normalize_string_utf8(str, options);
}
}
libpostal_normalized_token_t *libpostal_normalized_tokens(char *input, uint64_t string_options, uint64_t token_options, bool whitespace, size_t *n) {
if (input == NULL) {
return NULL;
}
char *normalized = libpostal_normalize_string(input, string_options);
if (normalized == NULL) {
return NULL;
}
token_array *tokens = NULL;
if (!whitespace) {
tokens = tokenize(normalized);
} else {
tokens = tokenize_keep_whitespace(normalized);
}
if (tokens == NULL || tokens->a == NULL) {
free(normalized);
return NULL;
}
size_t num_tokens = tokens->n;
token_t *token_array = tokens->a;
char_array *normalized_token = char_array_new_size(strlen(normalized));
libpostal_normalized_token_t *result = malloc(sizeof(libpostal_normalized_token_t) * num_tokens);
for (size_t i = 0; i < num_tokens; i++) {
token_t token = token_array[i];
char_array_clear(normalized_token);
add_normalized_token(normalized_token, normalized, token, token_options);
char *token_str = strdup(char_array_get_string(normalized_token));
result[i] = (libpostal_normalized_token_t){token_str, token};
}
free(normalized);
token_array_destroy(tokens);
char_array_destroy(normalized_token);
*n = num_tokens;
return result;
}
bool libpostal_setup_language_classifier(void) { bool libpostal_setup_language_classifier(void) {
return libpostal_setup_language_classifier_datadir(NULL); return libpostal_setup_language_classifier_datadir(NULL);
} }

110
src/libpostal.h
View File

@@ -12,6 +12,67 @@ extern "C" {
#define LIBPOSTAL_MAX_LANGUAGE_LEN 4 #define LIBPOSTAL_MAX_LANGUAGE_LEN 4
// Doing these as #defines so we can duplicate the values exactly in Python
typedef enum {
LIBPOSTAL_TOKEN_TYPE_END = 0, // Null byte
// Word types
LIBPOSTAL_TOKEN_TYPE_WORD = 1, // Any letter-only word (includes all unicode letters)
LIBPOSTAL_TOKEN_TYPE_ABBREVIATION = 2, // Loose abbreviations (roughly anything containing a "." as we don't care about sentences in addresses)
LIBPOSTAL_TOKEN_TYPE_IDEOGRAPHIC_CHAR = 3, // For languages that don't separate on whitespace (e.g. Chinese, Japanese, Korean), separate by character
LIBPOSTAL_TOKEN_TYPE_HANGUL_SYLLABLE = 4, // Hangul syllable sequences which contain more than one codepoint
LIBPOSTAL_TOKEN_TYPE_ACRONYM = 5, // Specifically things like U.N. where we may delete internal periods
LIBPOSTAL_TOKEN_TYPE_PHRASE = 10, // Not part of the first stage tokenizer, but may be used after phrase parsing
// Special tokens
LIBPOSTAL_TOKEN_TYPE_EMAIL = 20, // Make sure emails are tokenized altogether
LIBPOSTAL_TOKEN_TYPE_URL = 21, // Make sure urls are tokenized altogether
LIBPOSTAL_TOKEN_TYPE_US_PHONE = 22, // US phone number (with or without country code)
LIBPOSTAL_TOKEN_TYPE_INTL_PHONE = 23, // A non-US phone number (must have country code)
// Numbers and numeric types
LIBPOSTAL_TOKEN_TYPE_NUMERIC = 50, // Any sequence containing a digit
LIBPOSTAL_TOKEN_TYPE_ORDINAL = 51, // 1st, 2nd, 1er, 1 etc.
LIBPOSTAL_TOKEN_TYPE_ROMAN_NUMERAL = 52, // II, III, VI, etc.
LIBPOSTAL_TOKEN_TYPE_IDEOGRAPHIC_NUMBER = 53, // All numeric ideographic characters, includes e.g. Han numbers and chars like "²"
// Punctuation types, may separate a phrase
LIBPOSTAL_TOKEN_TYPE_PERIOD = 100,
LIBPOSTAL_TOKEN_TYPE_EXCLAMATION = 101,
LIBPOSTAL_TOKEN_TYPE_QUESTION_MARK = 102,
LIBPOSTAL_TOKEN_TYPE_COMMA = 103,
LIBPOSTAL_TOKEN_TYPE_COLON = 104,
LIBPOSTAL_TOKEN_TYPE_SEMICOLON = 105,
LIBPOSTAL_TOKEN_TYPE_PLUS = 106,
LIBPOSTAL_TOKEN_TYPE_AMPERSAND = 107,
LIBPOSTAL_TOKEN_TYPE_AT_SIGN = 108,
LIBPOSTAL_TOKEN_TYPE_POUND = 109,
LIBPOSTAL_TOKEN_TYPE_ELLIPSIS = 110,
LIBPOSTAL_TOKEN_TYPE_DASH = 111,
LIBPOSTAL_TOKEN_TYPE_BREAKING_DASH = 112,
LIBPOSTAL_TOKEN_TYPE_HYPHEN = 113,
LIBPOSTAL_TOKEN_TYPE_PUNCT_OPEN = 114,
LIBPOSTAL_TOKEN_TYPE_PUNCT_CLOSE = 115,
LIBPOSTAL_TOKEN_TYPE_DOUBLE_QUOTE = 119,
LIBPOSTAL_TOKEN_TYPE_SINGLE_QUOTE = 120,
LIBPOSTAL_TOKEN_TYPE_OPEN_QUOTE = 121,
LIBPOSTAL_TOKEN_TYPE_CLOSE_QUOTE = 122,
LIBPOSTAL_TOKEN_TYPE_SLASH = 124,
LIBPOSTAL_TOKEN_TYPE_BACKSLASH = 125,
LIBPOSTAL_TOKEN_TYPE_GREATER_THAN = 126,
LIBPOSTAL_TOKEN_TYPE_LESS_THAN = 127,
// Non-letters and whitespace
LIBPOSTAL_TOKEN_TYPE_OTHER = 200,
LIBPOSTAL_TOKEN_TYPE_WHITESPACE = 300,
LIBPOSTAL_TOKEN_TYPE_NEWLINE = 301,
LIBPOSTAL_TOKEN_TYPE_INVALID_CHAR = 500
} libpostal_token_type_t;
/* /*
Address dictionaries Address dictionaries
*/ */
@@ -99,6 +160,55 @@ bool libpostal_setup_parser(void);
bool libpostal_setup_parser_datadir(char *datadir); bool libpostal_setup_parser_datadir(char *datadir);
void libpostal_teardown_parser(void); void libpostal_teardown_parser(void);
typedef struct libpostal_token {
size_t offset;
size_t len;
uint16_t type;
} libpostal_token_t;
libpostal_token_t *libpostal_tokenize(char *input, bool whitespace, size_t *n);
// Normalize string options
#define LIBPOSTAL_NORMALIZE_STRING_LATIN_ASCII 1 << 0
#define LIBPOSTAL_NORMALIZE_STRING_TRANSLITERATE 1 << 1
#define LIBPOSTAL_NORMALIZE_STRING_STRIP_ACCENTS 1 << 2
#define LIBPOSTAL_NORMALIZE_STRING_DECOMPOSE 1 << 3
#define LIBPOSTAL_NORMALIZE_STRING_LOWERCASE 1 << 4
#define LIBPOSTAL_NORMALIZE_STRING_TRIM 1 << 5
#define LIBPOSTAL_NORMALIZE_STRING_REPLACE_HYPHENS 1 << 6
#define LIBPOSTAL_NORMALIZE_STRING_COMPOSE 1 << 7
#define LIBPOSTAL_NORMALIZE_STRING_SIMPLE_LATIN_ASCII 1 << 8
#define LIBPOSTAL_NORMALIZE_STRING_REPLACE_NUMEX 1 << 9
// Normalize token options
#define LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_HYPHENS 1 << 0
#define LIBPOSTAL_NORMALIZE_TOKEN_DELETE_HYPHENS 1 << 1
#define LIBPOSTAL_NORMALIZE_TOKEN_DELETE_FINAL_PERIOD 1 << 2
#define LIBPOSTAL_NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS 1 << 3
#define LIBPOSTAL_NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES 1 << 4
#define LIBPOSTAL_NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE 1 << 5
#define LIBPOSTAL_NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC 1 << 6
#define LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_DIGITS 1 << 7
#define LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_NUMERIC_TOKEN_LETTERS 1 << 8
#define LIBPOSTAL_NORMALIZE_DEFAULT_STRING_OPTIONS (LIBPOSTAL_NORMALIZE_STRING_LATIN_ASCII | LIBPOSTAL_NORMALIZE_STRING_COMPOSE | LIBPOSTAL_NORMALIZE_STRING_TRIM | LIBPOSTAL_NORMALIZE_STRING_REPLACE_HYPHENS | LIBPOSTAL_NORMALIZE_STRING_STRIP_ACCENTS | LIBPOSTAL_NORMALIZE_STRING_LOWERCASE)
#define LIBPOSTAL_NORMALIZE_DEFAULT_TOKEN_OPTIONS (LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_HYPHENS | LIBPOSTAL_NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | LIBPOSTAL_NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS | LIBPOSTAL_NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES | LIBPOSTAL_NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE)
#define LIBPOSTAL_NORMALIZE_TOKEN_OPTIONS_DROP_PERIODS (LIBPOSTAL_NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | LIBPOSTAL_NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS)
#define LIBPOSTAL_NORMALIZE_DEFAULT_TOKEN_OPTIONS_NUMERIC (LIBPOSTAL_NORMALIZE_DEFAULT_TOKEN_OPTIONS | LIBPOSTAL_NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC)
char *libpostal_normalize_string(char *input, uint64_t options);
typedef struct libpostal_normalized_token {
char *str;
libpostal_token_t token;
} libpostal_normalized_token_t;
libpostal_normalized_token_t *libpostal_normalized_tokens(char *input, uint64_t string_options, uint64_t token_options, bool whitespace, size_t *n);
bool libpostal_setup_language_classifier(void); bool libpostal_setup_language_classifier(void);
bool libpostal_setup_language_classifier_datadir(char *datadir); bool libpostal_setup_language_classifier_datadir(char *datadir);
void libpostal_teardown_language_classifier(void); void libpostal_teardown_language_classifier(void);

38
src/normalize.h
View File

@@ -30,6 +30,7 @@ As well as normalizations for individual string tokens:
#include "constants.h" #include "constants.h"
#include "klib/khash.h" #include "klib/khash.h"
#include "libpostal.h"
#include "string_utils.h" #include "string_utils.h"
#include "utf8proc/utf8proc.h" #include "utf8proc/utf8proc.h"
#include "unicode_scripts.h" #include "unicode_scripts.h"
@@ -39,25 +40,26 @@ As well as normalizations for individual string tokens:
#include "tokens.h" #include "tokens.h"
#include "vector.h" #include "vector.h"
#define NORMALIZE_STRING_LATIN_ASCII 1 << 0 #define NORMALIZE_STRING_LATIN_ASCII LIBPOSTAL_NORMALIZE_STRING_LATIN_ASCII
#define NORMALIZE_STRING_TRANSLITERATE 1 << 1 #define NORMALIZE_STRING_TRANSLITERATE LIBPOSTAL_NORMALIZE_STRING_TRANSLITERATE
#define NORMALIZE_STRING_STRIP_ACCENTS 1 << 2 #define NORMALIZE_STRING_STRIP_ACCENTS LIBPOSTAL_NORMALIZE_STRING_STRIP_ACCENTS
#define NORMALIZE_STRING_DECOMPOSE 1 << 3 #define NORMALIZE_STRING_DECOMPOSE LIBPOSTAL_NORMALIZE_STRING_DECOMPOSE
#define NORMALIZE_STRING_LOWERCASE 1 << 4 #define NORMALIZE_STRING_LOWERCASE LIBPOSTAL_NORMALIZE_STRING_LOWERCASE
#define NORMALIZE_STRING_TRIM 1 << 5 #define NORMALIZE_STRING_TRIM LIBPOSTAL_NORMALIZE_STRING_TRIM
#define NORMALIZE_STRING_REPLACE_HYPHENS 1 << 6 #define NORMALIZE_STRING_REPLACE_HYPHENS LIBPOSTAL_NORMALIZE_STRING_REPLACE_HYPHENS
#define NORMALIZE_STRING_COMPOSE 1 << 7 #define NORMALIZE_STRING_COMPOSE LIBPOSTAL_NORMALIZE_STRING_COMPOSE
#define NORMALIZE_STRING_SIMPLE_LATIN_ASCII 1 << 8 #define NORMALIZE_STRING_SIMPLE_LATIN_ASCII LIBPOSTAL_NORMALIZE_STRING_SIMPLE_LATIN_ASCII
#define NORMALIZE_STRING_REPLACE_NUMEX 1 << 9 #define NORMALIZE_STRING_REPLACE_NUMEX LIBPOSTAL_NORMALIZE_STRING_REPLACE_NUMEX
#define NORMALIZE_TOKEN_REPLACE_HYPHENS 1 << 0 #define NORMALIZE_TOKEN_REPLACE_HYPHENS LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_HYPHENS
#define NORMALIZE_TOKEN_DELETE_HYPHENS 1 << 1 #define NORMALIZE_TOKEN_DELETE_HYPHENS LIBPOSTAL_NORMALIZE_TOKEN_DELETE_HYPHENS
#define NORMALIZE_TOKEN_DELETE_FINAL_PERIOD 1 << 2 #define NORMALIZE_TOKEN_DELETE_FINAL_PERIOD LIBPOSTAL_NORMALIZE_TOKEN_DELETE_FINAL_PERIOD
#define NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS 1 << 3 #define NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS LIBPOSTAL_NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS
#define NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES 1 << 4 #define NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES LIBPOSTAL_NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES
#define NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE 1 << 5 #define NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE LIBPOSTAL_NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE
#define NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC 1 << 6 #define NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC LIBPOSTAL_NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC
#define NORMALIZE_TOKEN_REPLACE_DIGITS 1 << 7 #define NORMALIZE_TOKEN_REPLACE_DIGITS LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_DIGITS
#define NORMALIZE_TOKEN_REPLACE_NUMERIC_TOKEN_LETTERS LIBPOSTAL_NORMALIZE_TOKEN_REPLACE_NUMERIC_TOKEN_LETTERS
// Replace digits with capital D e.g. 10013 => DDDDD, intended for use with lowercased strings // Replace digits with capital D e.g. 10013 => DDDDD, intended for use with lowercased strings
#define DIGIT_CHAR "D" #define DIGIT_CHAR "D"

94
src/token_types.h
View File

@@ -1,64 +1,60 @@
#ifndef TOKEN_TYPES_H #ifndef TOKEN_TYPES_H
#define TOKEN_TYPES_H #define TOKEN_TYPES_H
#include "libpostal.h"
// Doing these as #defines so we can duplicate the values exactly in Python // Doing these as #defines so we can duplicate the values exactly in Python
#define END 0 // Null byte
// Word types #define END LIBPOSTAL_TOKEN_TYPE_END
#define WORD 1 // Any letter-only word (includes all unicode letters)
#define ABBREVIATION 2 // Loose abbreviations (roughly anything containing a "." as we don't care about sentences in addresses)
#define IDEOGRAPHIC_CHAR 3 // For languages that don't separate on whitespace (e.g. Chinese, Japanese, Korean), separate by character
#define HANGUL_SYLLABLE 4 // Hangul syllable sequences which contain more than one codepoint
#define ACRONYM 5 // Specifically things like U.N. where we may delete internal periods
#define PHRASE 10 // Not part of the first stage tokenizer, but may be used after phrase parsing #define WORD LIBPOSTAL_TOKEN_TYPE_WORD
#define ABBREVIATION LIBPOSTAL_TOKEN_TYPE_ABBREVIATION
#define IDEOGRAPHIC_CHAR LIBPOSTAL_TOKEN_TYPE_IDEOGRAPHIC_CHAR
#define HANGUL_SYLLABLE LIBPOSTAL_TOKEN_TYPE_HANGUL_SYLLABLE
#define ACRONYM LIBPOSTAL_TOKEN_TYPE_ACRONYM
#define PHRASE LIBPOSTAL_TOKEN_TYPE_PHRASE
// Special tokens #define EMAIL LIBPOSTAL_TOKEN_TYPE_EMAIL
#define EMAIL 20 // Make sure emails are tokenized altogether #define URL LIBPOSTAL_TOKEN_TYPE_URL
#define URL 21 // Make sure urls are tokenized altogether #define US_PHONE LIBPOSTAL_TOKEN_TYPE_US_PHONE
#define US_PHONE 22 // US phone number (with or without country code) #define INTL_PHONE LIBPOSTAL_TOKEN_TYPE_INTL_PHONE
#define INTL_PHONE 23 // A non-US phone number (must have country code)
// Numbers and numeric types #define NUMERIC LIBPOSTAL_TOKEN_TYPE_NUMERIC
#define NUMERIC 50 // Any sequence containing a digit #define ORDINAL LIBPOSTAL_TOKEN_TYPE_ORDINAL
#define ORDINAL 51 // 1st, 2nd, 1er, 1 etc. #define ROMAN_NUMERAL LIBPOSTAL_TOKEN_TYPE_ROMAN_NUMERAL
#define ROMAN_NUMERAL 52 // II, III, VI, etc. #define IDEOGRAPHIC_NUMBER LIBPOSTAL_TOKEN_TYPE_IDEOGRAPHIC_NUMBER
#define IDEOGRAPHIC_NUMBER 53 // All numeric ideographic characters, includes e.g. Han numbers and chars like "²"
#define PERIOD LIBPOSTAL_TOKEN_TYPE_PERIOD
#define EXCLAMATION LIBPOSTAL_TOKEN_TYPE_EXCLAMATION
#define QUESTION_MARK LIBPOSTAL_TOKEN_TYPE_QUESTION_MARK
#define COMMA LIBPOSTAL_TOKEN_TYPE_COMMA
#define COLON LIBPOSTAL_TOKEN_TYPE_COLON
#define SEMICOLON LIBPOSTAL_TOKEN_TYPE_SEMICOLON
#define PLUS LIBPOSTAL_TOKEN_TYPE_PLUS
#define AMPERSAND LIBPOSTAL_TOKEN_TYPE_AMPERSAND
#define AT_SIGN LIBPOSTAL_TOKEN_TYPE_AT_SIGN
#define POUND LIBPOSTAL_TOKEN_TYPE_POUND
#define ELLIPSIS LIBPOSTAL_TOKEN_TYPE_ELLIPSIS
#define DASH LIBPOSTAL_TOKEN_TYPE_DASH
#define BREAKING_DASH LIBPOSTAL_TOKEN_TYPE_BREAKING_DASH
#define HYPHEN LIBPOSTAL_TOKEN_TYPE_HYPHEN
#define PUNCT_OPEN LIBPOSTAL_TOKEN_TYPE_PUNCT_OPEN
#define PUNCT_CLOSE LIBPOSTAL_TOKEN_TYPE_PUNCT_CLOSE
#define DOUBLE_QUOTE LIBPOSTAL_TOKEN_TYPE_DOUBLE_QUOTE
#define SINGLE_QUOTE LIBPOSTAL_TOKEN_TYPE_SINGLE_QUOTE
#define OPEN_QUOTE LIBPOSTAL_TOKEN_TYPE_OPEN_QUOTE
#define CLOSE_QUOTE LIBPOSTAL_TOKEN_TYPE_CLOSE_QUOTE
#define SLASH LIBPOSTAL_TOKEN_TYPE_SLASH
#define BACKSLASH LIBPOSTAL_TOKEN_TYPE_BACKSLASH
#define GREATER_THAN LIBPOSTAL_TOKEN_TYPE_GREATER_THAN
#define LESS_THAN LIBPOSTAL_TOKEN_TYPE_LESS_THAN
// Punctuation types, may separate a phrase #define OTHER LIBPOSTAL_TOKEN_TYPE_OTHER
#define PERIOD 100 #define WHITESPACE LIBPOSTAL_TOKEN_TYPE_WHITESPACE
#define EXCLAMATION 101 #define NEWLINE LIBPOSTAL_TOKEN_TYPE_NEWLINE
#define QUESTION_MARK 102
#define COMMA 103
#define COLON 104
#define SEMICOLON 105
#define PLUS 106
#define AMPERSAND 107
#define AT_SIGN 108
#define POUND 109
#define ELLIPSIS 110
#define DASH 111
#define BREAKING_DASH 112
#define HYPHEN 113
#define PUNCT_OPEN 114
#define PUNCT_CLOSE 115
#define DOUBLE_QUOTE 119
#define SINGLE_QUOTE 120
#define OPEN_QUOTE 121
#define CLOSE_QUOTE 122
#define SLASH 124
#define BACKSLASH 125
#define GREATER_THAN 126
#define LESS_THAN 127
// Non-letters and whitespace #define INVALID_CHAR LIBPOSTAL_TOKEN_TYPE_INVALID_CHAR
#define OTHER 200
#define WHITESPACE 300
#define NEWLINE 301
#define INVALID_CHAR 500
#define is_word_token(type) ((type) == WORD || (type) == ABBREVIATION || (type) == ACRONYM || (type) == IDEOGRAPHIC_CHAR || (type) == HANGUL_SYLLABLE) #define is_word_token(type) ((type) == WORD || (type) == ABBREVIATION || (type) == ACRONYM || (type) == IDEOGRAPHIC_CHAR || (type) == HANGUL_SYLLABLE)

6
src/tokens.h
View File

@@ -12,11 +12,7 @@
#include "token_types.h" #include "token_types.h"
#include "vector.h" #include "vector.h"
typedef struct token { typedef libpostal_token_t token_t;
size_t offset;
size_t len;
uint16_t type;
} token_t;
VECTOR_INIT(token_array, token_t) VECTOR_INIT(token_array, token_t)