[normalize] Adding hyphen elimination as a string option (changes tokenization)

2015-10-27 13:32:47 -04:00
parent 3fe2365234
commit 40918812e2
6 changed files with 36 additions and 4 deletions
--- a/src/normalize.h
+++ b/src/normalize.h
@@ -44,6 +44,7 @@ As well as normalizations for individual string tokens:
 #define NORMALIZE_STRING_DECOMPOSE 1 << 3
 #define NORMALIZE_STRING_LOWERCASE 1 << 4
 #define NORMALIZE_STRING_TRIM 1 << 5
+#define NORMALIZE_STRING_REPLACE_HYPHENS 1 << 6

 #define NORMALIZE_TOKEN_REPLACE_HYPHENS 1 << 0
 #define NORMALIZE_TOKEN_DELETE_HYPHENS 1 << 1