tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[normalize] adding an option for string trimming in normalize

Browse Source
This commit is contained in:
Al
2015-07-27 01:59:14 -04:00
parent b94526a27b
commit 053b987d58
2 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/normalize.c
View File

@@ -11,6 +11,10 @@ char *normalize_string_utf8(char *str, uint64_t options) {
bool have_utf8proc_options = false; bool have_utf8proc_options = false;
if (options & NORMALIZE_STRING_TRIM) {
string_trim(str);
}
if (options & NORMALIZE_STRING_DECOMPOSE) { if (options & NORMALIZE_STRING_DECOMPOSE) {
have_utf8proc_options = true; have_utf8proc_options = true;
utf8proc_options |= UTF8PROC_OPTIONS_NFD; utf8proc_options |= UTF8PROC_OPTIONS_NFD;

1
src/normalize.h
View File

@@ -44,6 +44,7 @@ extern "C" {
#define NORMALIZE_STRING_STRIP_ACCENTS 1 << 2 #define NORMALIZE_STRING_STRIP_ACCENTS 1 << 2
#define NORMALIZE_STRING_DECOMPOSE 1 << 3 #define NORMALIZE_STRING_DECOMPOSE 1 << 3
#define NORMALIZE_STRING_LOWERCASE 1 << 4 #define NORMALIZE_STRING_LOWERCASE 1 << 4
#define NORMALIZE_STRING_TRIM 1 << 5
#define NORMALIZE_TOKEN_REPLACE_HYPHENS 1 << 0 #define NORMALIZE_TOKEN_REPLACE_HYPHENS 1 << 0
#define NORMALIZE_TOKEN_DELETE_HYPHENS 1 << 1 #define NORMALIZE_TOKEN_DELETE_HYPHENS 1 << 1