tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] normalize canonical strings (after expanding abbreviations, concatenated suffixes, etc.) with Latin-ASCII, Latin-ASCII-Simple or simple UTF-8 normalization depending on the options

Browse Source
This commit is contained in:
Al
2017-08-03 14:08:05 -06:00
parent ed011e50d5
commit 0c6af2b74c
2 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/normalize.c
View File

@@ -114,13 +114,19 @@ char *normalize_string_utf8(char *str, uint64_t options) {
char *normalize_string_latin_languages(char *str, size_t len, uint64_t options, size_t num_languages, char **languages) {
char *latin_transliterator = LATIN_ASCII;
char *transliterated = NULL;
char *latin_transliterator = NULL;
if (options & NORMALIZE_STRING_SIMPLE_LATIN_ASCII) {
latin_transliterator = LATIN_ASCII_SIMPLE;
} else if (options & NORMALIZE_STRING_LATIN_ASCII) {
latin_transliterator = LATIN_ASCII;
}
if (latin_transliterator != NULL) {
transliterated = transliterate(latin_transliterator, str, len);
}
char *transliterated = transliterate(latin_transliterator, str, len);
char *utf8_normalized;
if (transliterated == NULL) {
utf8_normalized = normalize_string_utf8_languages(str, options, num_languages, languages);