tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[tokenization] Adding a version which of tokenize which keeps whitespace tokens

Browse Source
This commit is contained in:
Al
2015-07-21 17:26:20 -04:00
parent 5d21cb1604
commit 71be52275d
3 changed files with 38 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/scanner.h
View File

@@ -5,6 +5,8 @@
extern "C" {
#endif
#include <stdbool.h>
#include "token_types.h"
#include "tokens.h"
@@ -16,7 +18,8 @@ uint16_t scan_token(scanner_t *s);
scanner_t scanner_from_string(const char *input, size_t len);
void tokenize_add_tokens(token_array *tokens, const char *input, size_t len);
void tokenize_add_tokens(token_array *tokens, const char *input, size_t len, bool keep_whitespace);
token_array *tokenize_keep_whitespace(const char *input);
token_array *tokenize(const char *input);