tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[numex] Adding utf8_is_number_or_letter

Browse Source
This commit is contained in:
Al
2015-06-04 14:49:12 -04:00
parent ca746304e3
commit 528dd05983
2 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/string_utils.c
View File

@@ -154,6 +154,14 @@ inline bool utf8_is_number(int32_t ch) {
return cat == UTF8PROC_CATEGORY_ND || cat == UTF8PROC_CATEGORY_NL || cat == UTF8PROC_CATEGORY_NO; return cat == UTF8PROC_CATEGORY_ND || cat == UTF8PROC_CATEGORY_NL || cat == UTF8PROC_CATEGORY_NO;
} }
inline bool utf8_is_letter_or_number(int32_t ch) {
int cat = utf8proc_category(ch);
return cat == UTF8PROC_CATEGORY_LL || cat == UTF8PROC_CATEGORY_LU \
|| cat == UTF8PROC_CATEGORY_LT || cat == UTF8PROC_CATEGORY_LO \
|| cat == UTF8PROC_CATEGORY_LM || cat == UTF8PROC_CATEGORY_ND \
|| cat == UTF8PROC_CATEGORY_NL || cat == UTF8PROC_CATEGORY_NO;
}
inline bool utf8_is_hyphen(int32_t ch) { inline bool utf8_is_hyphen(int32_t ch) {
int cat = utf8proc_category(ch); int cat = utf8proc_category(ch);
return cat == UTF8PROC_CATEGORY_PD || c == 0x2212; return cat == UTF8PROC_CATEGORY_PD || c == 0x2212;

1
src/string_utils.h
View File

@@ -32,6 +32,7 @@ char *utf8_reversed_string(const char *s); // returns a copy, caller frees
ssize_t utf8proc_iterate_reversed(const uint8_t *str, ssize_t start, int32_t *dst); ssize_t utf8proc_iterate_reversed(const uint8_t *str, ssize_t start, int32_t *dst);
bool utf8_is_letter(int32_t ch); bool utf8_is_letter(int32_t ch);
bool utf8_is_number(int32_t ch); bool utf8_is_number(int32_t ch);
bool utf8_is_letter_or_number(int32_t ch);
bool utf8_is_hyphen(int32_t ch); bool utf8_is_hyphen(int32_t ch);
bool utf8_is_punctuation(int32_t ch); bool utf8_is_punctuation(int32_t ch);
bool utf8_is_symbol(int32_t ch); bool utf8_is_symbol(int32_t ch);