[fix] terminate the char_array if input token is zero-length in add_normalized_token

src/normalize.c

@@ -388,7 +388,7 @@ void add_normalized_token(char_array *array, char *str, token_t token, uint64_t
 
     uint8_t *ptr = (uint8_t *)str + token.offset;
     size_t len = token.len;
-    if (token.len == 0) return;
+    if (token.len > 0) {
 
         bool alpha_numeric_split = false;
         char *append_if_not_numeric = NULL;
@@ -477,6 +477,7 @@ void add_normalized_token(char_array *array, char *str, token_t token, uint64_t
             last_was_letter = is_letter;
 
         }
+    }
 
     char_array_terminate(array);