[tokenization] skipping control characters in tokenization, comes up in OSM surprisingly

2015-10-04 18:25:44 -04:00
parent d6480d2902
commit aa39c45b87
2 changed files with 5 additions and 2 deletions
--- a/src/token_types.h
+++ b/src/token_types.h
@@ -60,6 +60,6 @@
 #define WHITESPACE 300
 #define NEWLINE 301

- 
+#define INVALID_CHAR 500 

 #endif