[python/normalization] Adding Python bindings to the normalize module for use in OSM polygon matching

python/postal/text/normalize.py

@@ -0,0 +1,29 @@
+from postal.text import _normalize
+from postal.text import _tokenize
+
+from postal.text.encoding import safe_decode
+
+DEFAULT_STRING_OPTIONS = _normalize.NORMALIZE_STRING_LATIN_ASCII |  \
+    _normalize.NORMALIZE_STRING_DECOMPOSE | \
+    _normalize.NORMALIZE_STRING_TRIM | \
+    _normalize.NORMALIZE_STRING_LOWERCASE
+
+DEFAULT_TOKEN_OPTIONS = _normalize.NORMALIZE_TOKEN_REPLACE_HYPHENS | \
+    _normalize.NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | \
+    _normalize.NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS | \
+    _normalize.NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES | \
+    _normalize.NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE | \
+    _normalize.NORMALIZE_TOKEN_REPLACE_DIGITS
+
+
+def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS,
+                      token_options=DEFAULT_TOKEN_OPTIONS):
+    s = safe_decode(s)
+    if string_options & _normalize.NORMALIZE_STRING_LATIN_ASCII:
+        normalized = _normalize.normalize_string_latin(s, string_options)
+    else:
+        normalized = _normalize.normalize_string_utf8(s, string_options)
+
+    # Tuples of (offset, len, type)
+    tokens = _tokenize.tokenize(normalized)
+    return [_normalize.normalize_token(normalized, t, token_options) for t in tokens]

python/postal/text/pynormalize.c

@@ -0,0 +1,325 @@
+#include <Python.h>
+
+#include "pyencoding.h"
+
+#include "src/normalize.h"
+#include "src/transliterate.h"
+
+#if PY_MAJOR_VERSION >= 3
+#define IS_PY3K
+#endif
+
+struct module_state {
+    PyObject *error;
+};
+
+
+#ifdef IS_PY3K
+    #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m))
+#else
+    #define GETSTATE(m) (&_state)
+    static struct module_state _state;
+#endif
+
+
+
+static PyObject *py_normalize_string_utf8(PyObject *self, PyObject *args) 
+{
+    PyObject *arg1;
+    uint64_t options;
+    if (!PyArg_ParseTuple(args, "OK:normalize", &arg1, &options)) {
+        return 0;
+    }
+
+    PyObject *unistr = PyUnicode_FromObject(arg1);
+    if (unistr == NULL) {
+        PyErr_SetString(PyExc_TypeError,
+                        "Parameter could not be converted to unicode in scanner");
+        return 0;
+    }
+
+    #ifdef IS_PY3K
+        // Python 3 encoding, supported by Python 3.3+
+
+        char *input = PyUnicode_AsUTF8(unistr);
+
+    #else
+        // Python 2 encoding
+
+        PyObject *str = PyUnicode_AsEncodedString(unistr, "utf-8", "strict");
+        if (str == NULL) {
+            PyErr_SetString(PyExc_TypeError,
+                            "Parameter could not be utf-8 encoded");
+            goto error_decref_unistr;
+        }
+
+        char *input = PyBytes_AsString(str);
+
+    #endif
+
+    if (input == NULL) {
+        goto error_decref_str;
+    }
+
+    char *normalized = normalize_string_utf8(input, options);
+
+    PyObject *result = PyUnicode_DecodeUTF8((const char *)normalized, strlen(normalized), "strict");
+    free(normalized);
+    if (result == NULL) {
+            PyErr_SetString(PyExc_ValueError,
+                            "Result could not be utf-8 decoded");
+            goto error_decref_str;
+    }
+
+    #ifndef IS_PY3K
+    Py_XDECREF(str);
+    #endif
+    Py_XDECREF(unistr);
+
+    return result;
+
+error_decref_str:
+#ifndef IS_PY3K
+    Py_XDECREF(str);
+#endif
+error_decref_unistr:
+    Py_XDECREF(unistr);
+    return 0;
+}
+
+
+static PyObject *py_normalize_string_latin(PyObject *self, PyObject *args) 
+{
+    PyObject *arg1;
+    uint64_t options;
+    if (!PyArg_ParseTuple(args, "OK:normalize", &arg1, &options)) {
+        return 0;
+    }
+
+    PyObject *unistr = PyUnicode_FromObject(arg1);
+    if (unistr == NULL) {
+        PyErr_SetString(PyExc_TypeError,
+                        "Parameter could not be converted to unicode in scanner");
+        return 0;
+    }
+
+    #ifdef IS_PY3K
+        // Python 3 encoding, supported by Python 3.3+
+
+        char *input = PyUnicode_AsUTF8(unistr);
+
+    #else
+        // Python 2 encoding
+
+        PyObject *str = PyUnicode_AsEncodedString(unistr, "utf-8", "strict");
+        if (str == NULL) {
+            PyErr_SetString(PyExc_TypeError,
+                            "Parameter could not be utf-8 encoded");
+            goto error_decref_unistr;
+        }
+
+        char *input = PyBytes_AsString(str);
+
+    #endif
+
+    if (input == NULL) {
+        goto error_decref_str;
+    }
+
+    char *normalized = normalize_string_latin(input, strlen(input), options);
+
+    PyObject *result = PyUnicode_DecodeUTF8((const char *)normalized, strlen(normalized), "strict");
+    free(normalized);
+    if (result == NULL) {
+        PyErr_SetString(PyExc_ValueError,
+                        "Result could not be utf-8 decoded");
+        goto error_decref_str;
+    }
+
+    #ifndef IS_PY3K
+    Py_XDECREF(str);
+    #endif
+    Py_XDECREF(unistr);
+
+    return result;
+
+error_decref_str:
+#ifndef IS_PY3K
+    Py_XDECREF(str);
+#endif
+error_decref_unistr:
+    Py_XDECREF(unistr);
+    return 0;
+}
+
+
+
+static PyObject *py_normalize_token(PyObject *self, PyObject *args) 
+{
+    PyObject *s;
+
+    uint32_t offset;
+    uint32_t len;
+    uint16_t type;
+
+    uint64_t options;
+    if (!PyArg_ParseTuple(args, "O(IIH)K:normalize", &s, &offset, &len, &type, &options)) {
+        PyErr_SetString(PyExc_TypeError,
+                        "Error parsing arguments");
+        return 0;
+    }
+
+    token_t token = (token_t){(size_t)offset, (size_t)len, type};
+
+    PyObject *unistr = PyUnicode_FromObject(s);
+    if (unistr == NULL) {
+        PyErr_SetString(PyExc_TypeError,
+                        "Parameter could not be converted to unicode in scanner");
+        return 0;
+    }
+
+    #ifdef IS_PY3K
+        // Python 3 encoding, supported by Python 3.3+
+
+        char *input = PyUnicode_AsUTF8(unistr);
+
+    #else
+        // Python 2 encoding
+
+        PyObject *str = PyUnicode_AsEncodedString(unistr, "utf-8", "strict");
+        if (str == NULL) {
+            PyErr_SetString(PyExc_ValueError,
+                            "Parameter could not be utf-8 encoded");
+            goto error_decref_unistr;
+        }
+
+        char *input = PyBytes_AsString(str);
+
+    #endif
+
+    if (input == NULL) {
+        goto error_decref_str;
+    }
+
+    char_array *token_buffer = char_array_new_size(token.len);
+
+    add_normalized_token(token_buffer, input, token, options);
+    char *token_str = char_array_get_string(token_buffer);
+    PyObject *result = PyUnicode_DecodeUTF8((const char *)token_str, token_buffer->n - 1, "strict");
+
+    if (result == NULL) {
+        PyErr_SetString(PyExc_ValueError,
+                        "Error decoding token");
+        char_array_destroy(token_buffer);
+        goto error_decref_str;
+    }
+
+    char_array_destroy(token_buffer);
+
+    #ifndef IS_PY3K
+    Py_XDECREF(str);
+    #endif
+    Py_XDECREF(unistr);
+
+    return result;
+
+error_decref_str:
+#ifndef IS_PY3K
+    Py_XDECREF(str);
+#endif
+error_decref_unistr:
+    Py_XDECREF(unistr);
+    return 0;
+}
+
+static PyMethodDef normalize_methods[] = {
+    {"normalize_string_utf8", (PyCFunction)py_normalize_string_utf8, METH_VARARGS, "normalize_string_utf8(input, options)"},
+    {"normalize_string_latin", (PyCFunction)py_normalize_string_latin, METH_VARARGS, "normalize_string_latin(input, options)"},
+    {"normalize_token", (PyCFunction)py_normalize_token, METH_VARARGS, "normalize_token(input, options)"},
+    {NULL, NULL},
+};
+
+
+
+#ifdef IS_PY3K
+
+static int normalize_traverse(PyObject *m, visitproc visit, void *arg) {
+    Py_VISIT(GETSTATE(m)->error);
+    return 0;
+}
+
+static int normalize_clear(PyObject *m) {
+    Py_CLEAR(GETSTATE(m)->error);
+    return 0;
+}
+
+
+static struct PyModuleDef module_def = {
+        PyModuleDef_HEAD_INIT,
+        "_normalize",
+        NULL,
+        sizeof(struct module_state),
+        normalize_methods,
+        NULL,
+        normalize_traverse,
+        normalize_clear,
+        NULL
+};
+
+#define INITERROR return NULL
+
+PyObject *
+PyInit_normalize(void)
+#else
+#define INITERROR return
+
+void
+init_normalize(void) {
+#endif
+
+#ifdef IS_PY3K
+    PyObject *module = PyModule_Create(&module_def);
+#else
+    PyObject *module = Py_InitModule("_normalize", normalize_methods);
+#endif
+
+    if (module == NULL)
+        INITERROR;
+    struct module_state *st = GETSTATE(module);
+
+    st->error = PyErr_NewException("_normalize.Error", NULL, NULL);
+    if (st->error == NULL) {
+        Py_DECREF(module);
+        INITERROR;
+    }
+
+    if (!transliteration_module_setup(NULL)) {
+        PyErr_SetString(PyExc_RuntimeError,
+                        "Could not load transliterate module");
+        Py_DECREF(module);
+        INITERROR;
+    }
+
+
+    PyModule_AddObject(module, "NORMALIZE_STRING_LATIN_ASCII", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_LATIN_ASCII));
+    PyModule_AddObject(module, "NORMALIZE_STRING_TRANSLITERATE", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_TRANSLITERATE));
+    PyModule_AddObject(module, "NORMALIZE_STRING_STRIP_ACCENTS", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_STRIP_ACCENTS));
+    PyModule_AddObject(module, "NORMALIZE_STRING_DECOMPOSE", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_DECOMPOSE));
+    PyModule_AddObject(module, "NORMALIZE_STRING_LOWERCASE", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_LOWERCASE));
+    PyModule_AddObject(module, "NORMALIZE_STRING_TRIM", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_TRIM));
+
+
+    PyModule_AddObject(module, "NORMALIZE_TOKEN_REPLACE_HYPHENS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_REPLACE_HYPHENS));
+    PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_HYPHENS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_HYPHENS));
+    PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_FINAL_PERIOD", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_FINAL_PERIOD));
+    PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS));
+    PyModule_AddObject(module, "NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES));
+    PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE));
+    PyModule_AddObject(module, "NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC));
+    PyModule_AddObject(module, "NORMALIZE_TOKEN_REPLACE_DIGITS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_REPLACE_DIGITS));
+
+
+#if PY_MAJOR_VERSION >= 3
+    return module;
+#endif
+}

setup.py

@@ -14,17 +14,33 @@ def main():
         ],
         ext_modules=[
             Extension('postal.text._tokenize',
-                      sources=[
+                      sources=[os.path.join(SRC_DIR, f)
-                          os.path.join(SRC_DIR, 'scanner.c'),
+                               for f in ('scanner.c',
-                          os.path.join(SRC_DIR, 'string_utils.c'),
+                                         'string_utils.c',
-                          os.path.join(SRC_DIR, 'tokens.c'),
+                                         'tokens.c',
-                          os.path.join(SRC_DIR, 'utf8proc/utf8proc.c'),
+                                         'utf8proc/utf8proc.c',
-                          'python/postal/text/pytokenize.c',
+                                         )
-                      ],
+                               ] + ['python/postal/text/pytokenize.c'],
                       include_dirs=['.'],
-                      extra_compile_args=['-O0', '-std=c99'],
+                      extra_compile_args=['-O0', '-std=c99',
+                                          '-Wno-unused-function'],
+                      ),
+            Extension('postal.text._normalize',
+                      sources=[os.path.join(SRC_DIR, f)
+                               for f in ('normalize.c',
+                                         'string_utils.c',
+                                         'utf8proc/utf8proc.c',
+                                         'tokens.c',
+                                         'unicode_scripts.c',
+                                         'transliterate.c',
+                                         'file_utils.c',
+                                         'trie.c',
+                                         'trie_search.c',)
+                               ] + ['python/postal/text/pynormalize.c'],
+                      include_dirs=['.'],
+                      extra_compile_args=['-O0', '-std=c99', '-DHAVE_CONFIG_H',
+                                          '-Wno-unused-function'],
                       ),
-
         ],
         packages=find_packages('python'),
         package_dir={'': 'python'},