[tokenization] is_whitespace
This commit is contained in:
@@ -71,4 +71,6 @@
|
|||||||
|
|
||||||
#define is_special_token(type) ((type) == EMAIL || (type) == URL || (type) == US_PHONE || (type) == INTL_PHONE)
|
#define is_special_token(type) ((type) == EMAIL || (type) == URL || (type) == US_PHONE || (type) == INTL_PHONE)
|
||||||
|
|
||||||
|
#define is_whitespace(type) ((type) == WHITESPACE)
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
Reference in New Issue
Block a user