tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[unicode] Adding SCRIPT_INHERITED as a common script so diacritics like COMBING CEDILLA don't break the current script and produce false word breaks

Browse Source
This commit is contained in:
Al
2016-01-11 16:39:15 -05:00
parent 3260edcf18
commit 25ae5bed33
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/unicode_scripts.c
View File

@@ -13,6 +13,10 @@ inline script_languages_t get_script_languages(script_t script) {
return script_languages[script]; return script_languages[script];
} }
inline bool is_common_script(script_t script) {
return script == SCRIPT_COMMON || script == SCRIPT_INHERITED;
}
string_script_t get_string_script(char *str, size_t len) { string_script_t get_string_script(char *str, size_t len) {
int32_t ch; int32_t ch;
script_t last_script = SCRIPT_UNKNOWN; script_t last_script = SCRIPT_UNKNOWN;
@@ -32,18 +36,18 @@ string_script_t get_string_script(char *str, size_t len) {
script = get_char_script((uint32_t)ch); script = get_char_script((uint32_t)ch);
if (script == SCRIPT_COMMON && last_script != SCRIPT_UNKNOWN) { if (is_common_script(script) && last_script != SCRIPT_UNKNOWN) {
script = last_script; script = last_script;
} }
if (last_script != script && last_script != SCRIPT_UNKNOWN && last_script != SCRIPT_COMMON) { if (last_script != script && last_script != SCRIPT_UNKNOWN && !is_common_script(last_script)) {
if (script_len < len) { if (script_len < len) {
while (true) { while (true) {
char_len = utf8proc_iterate_reversed((const uint8_t *)str, idx, &ch); char_len = utf8proc_iterate_reversed((const uint8_t *)str, idx, &ch);
if (ch == 0) break; if (ch == 0) break;
script = get_char_script((uint32_t)ch); script = get_char_script((uint32_t)ch);
if (script != SCRIPT_COMMON) { if (!is_common_script(script)) {
break; break;
} }