diff --git a/python/postal/text/normalize.py b/python/postal/text/normalize.py new file mode 100644 index 00000000..c2fed1a1 --- /dev/null +++ b/python/postal/text/normalize.py @@ -0,0 +1,29 @@ +from postal.text import _normalize +from postal.text import _tokenize + +from postal.text.encoding import safe_decode + +DEFAULT_STRING_OPTIONS = _normalize.NORMALIZE_STRING_LATIN_ASCII | \ + _normalize.NORMALIZE_STRING_DECOMPOSE | \ + _normalize.NORMALIZE_STRING_TRIM | \ + _normalize.NORMALIZE_STRING_LOWERCASE + +DEFAULT_TOKEN_OPTIONS = _normalize.NORMALIZE_TOKEN_REPLACE_HYPHENS | \ + _normalize.NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | \ + _normalize.NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS | \ + _normalize.NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES | \ + _normalize.NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE | \ + _normalize.NORMALIZE_TOKEN_REPLACE_DIGITS + + +def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS, + token_options=DEFAULT_TOKEN_OPTIONS): + s = safe_decode(s) + if string_options & _normalize.NORMALIZE_STRING_LATIN_ASCII: + normalized = _normalize.normalize_string_latin(s, string_options) + else: + normalized = _normalize.normalize_string_utf8(s, string_options) + + # Tuples of (offset, len, type) + tokens = _tokenize.tokenize(normalized) + return [_normalize.normalize_token(normalized, t, token_options) for t in tokens] diff --git a/python/postal/text/pynormalize.c b/python/postal/text/pynormalize.c new file mode 100644 index 00000000..e2e5c277 --- /dev/null +++ b/python/postal/text/pynormalize.c @@ -0,0 +1,325 @@ +#include + +#include "pyencoding.h" + +#include "src/normalize.h" +#include "src/transliterate.h" + +#if PY_MAJOR_VERSION >= 3 +#define IS_PY3K +#endif + +struct module_state { + PyObject *error; +}; + + +#ifdef IS_PY3K + #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) +#else + #define GETSTATE(m) (&_state) + static struct module_state _state; +#endif + + + +static PyObject *py_normalize_string_utf8(PyObject *self, PyObject *args) +{ + PyObject *arg1; + uint64_t options; + if (!PyArg_ParseTuple(args, "OK:normalize", &arg1, &options)) { + return 0; + } + + PyObject *unistr = PyUnicode_FromObject(arg1); + if (unistr == NULL) { + PyErr_SetString(PyExc_TypeError, + "Parameter could not be converted to unicode in scanner"); + return 0; + } + + #ifdef IS_PY3K + // Python 3 encoding, supported by Python 3.3+ + + char *input = PyUnicode_AsUTF8(unistr); + + #else + // Python 2 encoding + + PyObject *str = PyUnicode_AsEncodedString(unistr, "utf-8", "strict"); + if (str == NULL) { + PyErr_SetString(PyExc_TypeError, + "Parameter could not be utf-8 encoded"); + goto error_decref_unistr; + } + + char *input = PyBytes_AsString(str); + + #endif + + if (input == NULL) { + goto error_decref_str; + } + + char *normalized = normalize_string_utf8(input, options); + + PyObject *result = PyUnicode_DecodeUTF8((const char *)normalized, strlen(normalized), "strict"); + free(normalized); + if (result == NULL) { + PyErr_SetString(PyExc_ValueError, + "Result could not be utf-8 decoded"); + goto error_decref_str; + } + + #ifndef IS_PY3K + Py_XDECREF(str); + #endif + Py_XDECREF(unistr); + + return result; + +error_decref_str: +#ifndef IS_PY3K + Py_XDECREF(str); +#endif +error_decref_unistr: + Py_XDECREF(unistr); + return 0; +} + + +static PyObject *py_normalize_string_latin(PyObject *self, PyObject *args) +{ + PyObject *arg1; + uint64_t options; + if (!PyArg_ParseTuple(args, "OK:normalize", &arg1, &options)) { + return 0; + } + + PyObject *unistr = PyUnicode_FromObject(arg1); + if (unistr == NULL) { + PyErr_SetString(PyExc_TypeError, + "Parameter could not be converted to unicode in scanner"); + return 0; + } + + #ifdef IS_PY3K + // Python 3 encoding, supported by Python 3.3+ + + char *input = PyUnicode_AsUTF8(unistr); + + #else + // Python 2 encoding + + PyObject *str = PyUnicode_AsEncodedString(unistr, "utf-8", "strict"); + if (str == NULL) { + PyErr_SetString(PyExc_TypeError, + "Parameter could not be utf-8 encoded"); + goto error_decref_unistr; + } + + char *input = PyBytes_AsString(str); + + #endif + + if (input == NULL) { + goto error_decref_str; + } + + char *normalized = normalize_string_latin(input, strlen(input), options); + + PyObject *result = PyUnicode_DecodeUTF8((const char *)normalized, strlen(normalized), "strict"); + free(normalized); + if (result == NULL) { + PyErr_SetString(PyExc_ValueError, + "Result could not be utf-8 decoded"); + goto error_decref_str; + } + + #ifndef IS_PY3K + Py_XDECREF(str); + #endif + Py_XDECREF(unistr); + + return result; + +error_decref_str: +#ifndef IS_PY3K + Py_XDECREF(str); +#endif +error_decref_unistr: + Py_XDECREF(unistr); + return 0; +} + + + +static PyObject *py_normalize_token(PyObject *self, PyObject *args) +{ + PyObject *s; + + uint32_t offset; + uint32_t len; + uint16_t type; + + uint64_t options; + if (!PyArg_ParseTuple(args, "O(IIH)K:normalize", &s, &offset, &len, &type, &options)) { + PyErr_SetString(PyExc_TypeError, + "Error parsing arguments"); + return 0; + } + + token_t token = (token_t){(size_t)offset, (size_t)len, type}; + + PyObject *unistr = PyUnicode_FromObject(s); + if (unistr == NULL) { + PyErr_SetString(PyExc_TypeError, + "Parameter could not be converted to unicode in scanner"); + return 0; + } + + #ifdef IS_PY3K + // Python 3 encoding, supported by Python 3.3+ + + char *input = PyUnicode_AsUTF8(unistr); + + #else + // Python 2 encoding + + PyObject *str = PyUnicode_AsEncodedString(unistr, "utf-8", "strict"); + if (str == NULL) { + PyErr_SetString(PyExc_ValueError, + "Parameter could not be utf-8 encoded"); + goto error_decref_unistr; + } + + char *input = PyBytes_AsString(str); + + #endif + + if (input == NULL) { + goto error_decref_str; + } + + char_array *token_buffer = char_array_new_size(token.len); + + add_normalized_token(token_buffer, input, token, options); + char *token_str = char_array_get_string(token_buffer); + PyObject *result = PyUnicode_DecodeUTF8((const char *)token_str, token_buffer->n - 1, "strict"); + + if (result == NULL) { + PyErr_SetString(PyExc_ValueError, + "Error decoding token"); + char_array_destroy(token_buffer); + goto error_decref_str; + } + + char_array_destroy(token_buffer); + + #ifndef IS_PY3K + Py_XDECREF(str); + #endif + Py_XDECREF(unistr); + + return result; + +error_decref_str: +#ifndef IS_PY3K + Py_XDECREF(str); +#endif +error_decref_unistr: + Py_XDECREF(unistr); + return 0; +} + +static PyMethodDef normalize_methods[] = { + {"normalize_string_utf8", (PyCFunction)py_normalize_string_utf8, METH_VARARGS, "normalize_string_utf8(input, options)"}, + {"normalize_string_latin", (PyCFunction)py_normalize_string_latin, METH_VARARGS, "normalize_string_latin(input, options)"}, + {"normalize_token", (PyCFunction)py_normalize_token, METH_VARARGS, "normalize_token(input, options)"}, + {NULL, NULL}, +}; + + + +#ifdef IS_PY3K + +static int normalize_traverse(PyObject *m, visitproc visit, void *arg) { + Py_VISIT(GETSTATE(m)->error); + return 0; +} + +static int normalize_clear(PyObject *m) { + Py_CLEAR(GETSTATE(m)->error); + return 0; +} + + +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + "_normalize", + NULL, + sizeof(struct module_state), + normalize_methods, + NULL, + normalize_traverse, + normalize_clear, + NULL +}; + +#define INITERROR return NULL + +PyObject * +PyInit_normalize(void) +#else +#define INITERROR return + +void +init_normalize(void) { +#endif + +#ifdef IS_PY3K + PyObject *module = PyModule_Create(&module_def); +#else + PyObject *module = Py_InitModule("_normalize", normalize_methods); +#endif + + if (module == NULL) + INITERROR; + struct module_state *st = GETSTATE(module); + + st->error = PyErr_NewException("_normalize.Error", NULL, NULL); + if (st->error == NULL) { + Py_DECREF(module); + INITERROR; + } + + if (!transliteration_module_setup(NULL)) { + PyErr_SetString(PyExc_RuntimeError, + "Could not load transliterate module"); + Py_DECREF(module); + INITERROR; + } + + + PyModule_AddObject(module, "NORMALIZE_STRING_LATIN_ASCII", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_LATIN_ASCII)); + PyModule_AddObject(module, "NORMALIZE_STRING_TRANSLITERATE", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_TRANSLITERATE)); + PyModule_AddObject(module, "NORMALIZE_STRING_STRIP_ACCENTS", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_STRIP_ACCENTS)); + PyModule_AddObject(module, "NORMALIZE_STRING_DECOMPOSE", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_DECOMPOSE)); + PyModule_AddObject(module, "NORMALIZE_STRING_LOWERCASE", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_LOWERCASE)); + PyModule_AddObject(module, "NORMALIZE_STRING_TRIM", PyLong_FromUnsignedLongLong(NORMALIZE_STRING_TRIM)); + + + PyModule_AddObject(module, "NORMALIZE_TOKEN_REPLACE_HYPHENS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_REPLACE_HYPHENS)); + PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_HYPHENS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_HYPHENS)); + PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_FINAL_PERIOD", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_FINAL_PERIOD)); + PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS)); + PyModule_AddObject(module, "NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES)); + PyModule_AddObject(module, "NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE)); + PyModule_AddObject(module, "NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC)); + PyModule_AddObject(module, "NORMALIZE_TOKEN_REPLACE_DIGITS", PyLong_FromUnsignedLongLong(NORMALIZE_TOKEN_REPLACE_DIGITS)); + + +#if PY_MAJOR_VERSION >= 3 + return module; +#endif +} \ No newline at end of file diff --git a/setup.py b/setup.py index 7c6acb8f..a33b7410 100644 --- a/setup.py +++ b/setup.py @@ -14,17 +14,33 @@ def main(): ], ext_modules=[ Extension('postal.text._tokenize', - sources=[ - os.path.join(SRC_DIR, 'scanner.c'), - os.path.join(SRC_DIR, 'string_utils.c'), - os.path.join(SRC_DIR, 'tokens.c'), - os.path.join(SRC_DIR, 'utf8proc/utf8proc.c'), - 'python/postal/text/pytokenize.c', - ], + sources=[os.path.join(SRC_DIR, f) + for f in ('scanner.c', + 'string_utils.c', + 'tokens.c', + 'utf8proc/utf8proc.c', + ) + ] + ['python/postal/text/pytokenize.c'], include_dirs=['.'], - extra_compile_args=['-O0', '-std=c99'], + extra_compile_args=['-O0', '-std=c99', + '-Wno-unused-function'], + ), + Extension('postal.text._normalize', + sources=[os.path.join(SRC_DIR, f) + for f in ('normalize.c', + 'string_utils.c', + 'utf8proc/utf8proc.c', + 'tokens.c', + 'unicode_scripts.c', + 'transliterate.c', + 'file_utils.c', + 'trie.c', + 'trie_search.c',) + ] + ['python/postal/text/pynormalize.c'], + include_dirs=['.'], + extra_compile_args=['-O0', '-std=c99', '-DHAVE_CONFIG_H', + '-Wno-unused-function'], ), - ], packages=find_packages('python'), package_dir={'': 'python'},