tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[normalize] Adding NORMALIZE_STRING_COMPOSE for NFC unicode normalization

Browse Source
This commit is contained in:
Al
2016-05-28 19:25:12 -04:00
parent 757c6147cb
commit 6c39c663ff
2 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/normalize.c
View File

@@ -19,6 +19,11 @@ char *normalize_string_utf8(char *str, uint64_t options) {
utf8proc_options |= UTF8PROC_OPTIONS_NFD; utf8proc_options |= UTF8PROC_OPTIONS_NFD;
} }
if (options & NORMALIZE_STRING_COMPOSE) {
have_utf8proc_options = true;
utf8proc_options |= UTF8PROC_OPTIONS_NFC;
}
if (options & NORMALIZE_STRING_STRIP_ACCENTS) { if (options & NORMALIZE_STRING_STRIP_ACCENTS) {
have_utf8proc_options = true; have_utf8proc_options = true;
utf8proc_options |= UTF8PROC_OPTIONS_STRIP_ACCENTS; utf8proc_options |= UTF8PROC_OPTIONS_STRIP_ACCENTS;

1
src/normalize.h
View File

@@ -45,6 +45,7 @@ As well as normalizations for individual string tokens:
#define NORMALIZE_STRING_LOWERCASE 1 << 4 #define NORMALIZE_STRING_LOWERCASE 1 << 4
#define NORMALIZE_STRING_TRIM 1 << 5 #define NORMALIZE_STRING_TRIM 1 << 5
#define NORMALIZE_STRING_REPLACE_HYPHENS 1 << 6 #define NORMALIZE_STRING_REPLACE_HYPHENS 1 << 6
#define NORMALIZE_STRING_COMPOSE 1 << 7
#define NORMALIZE_TOKEN_REPLACE_HYPHENS 1 << 0 #define NORMALIZE_TOKEN_REPLACE_HYPHENS 1 << 0
#define NORMALIZE_TOKEN_DELETE_HYPHENS 1 << 1 #define NORMALIZE_TOKEN_DELETE_HYPHENS 1 << 1