tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[tokenization] Adding a version which of tokenize which keeps whitespace tokens

Browse Source
This commit is contained in:
Al
2015-07-21 17:26:20 -04:00
parent 5d21cb1604
commit 71be52275d
3 changed files with 38 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/scanner.c
View File

@@ -347350,7 +347350,7 @@ inline scanner_t scanner_from_string(const char *input, size_t len) {
return scanner;
}
void tokenize_add_tokens(token_array *tokens, const char *input, size_t len) {
void tokenize_add_tokens(token_array *tokens, const char *input, size_t len, bool keep_whitespace) {
scanner_t scanner = scanner_from_string(input, len);
size_t token_start, token_length;
@@ -347360,8 +347360,10 @@ void tokenize_add_tokens(token_array *tokens, const char *input, size_t len) {
token_start = scanner.start - scanner.src;
token_length = scanner.cursor - scanner.start;
if (token_type != WHITESPACE) {
// Caller frees
if (token_type == WHITESPACE && !keep_whitespace) {
continue;
}
token_t token;
token.offset = token_start;
token.len = token_length;
@@ -347369,15 +347371,20 @@ void tokenize_add_tokens(token_array *tokens, const char *input, size_t len) {
token_array_push(tokens, token);
}
}
}
token_array *tokenize_keep_whitespace(const char *input) {
token_array *tokens = token_array_new();
tokenize_add_tokens(tokens, input, strlen(input), true);
return tokens;
}
token_array *tokenize(const char *input) {
token_array *tokens = token_array_new();
tokenize_add_tokens(tokens, input, strlen(input));
tokenize_add_tokens(tokens, input, strlen(input), false);
return tokens;
}

5
src/scanner.h
View File

@@ -5,6 +5,8 @@
extern "C" {
#endif
#include <stdbool.h>
#include "token_types.h"
#include "tokens.h"
@@ -16,7 +18,8 @@ uint16_t scan_token(scanner_t *s);
scanner_t scanner_from_string(const char *input, size_t len);
void tokenize_add_tokens(token_array *tokens, const char *input, size_t len);
void tokenize_add_tokens(token_array *tokens, const char *input, size_t len, bool keep_whitespace);
token_array *tokenize_keep_whitespace(const char *input);
token_array *tokenize(const char *input);

17
src/scanner.re
View File

@@ -219,7 +219,7 @@ inline scanner_t scanner_from_string(const char *input, size_t len) {
return scanner;
}
void tokenize_add_tokens(token_array *tokens, const char *input, size_t len) {
void tokenize_add_tokens(token_array *tokens, const char *input, size_t len, bool keep_whitespace) {
scanner_t scanner = scanner_from_string(input, len);
size_t token_start, token_length;
@@ -229,8 +229,10 @@ void tokenize_add_tokens(token_array *tokens, const char *input, size_t len) {
token_start = scanner.start - scanner.src;
token_length = scanner.cursor - scanner.start;
if (token_type != WHITESPACE) {
// Caller frees
if (token_type == WHITESPACE && !keep_whitespace) {
continue;
}
token_t token;
token.offset = token_start;
token.len = token_length;
@@ -238,15 +240,20 @@ void tokenize_add_tokens(token_array *tokens, const char *input, size_t len) {
token_array_push(tokens, token);
}
}
}
token_array *tokenize_keep_whitespace(const char *input) {
token_array *tokens = token_array_new();
tokenize_add_tokens(tokens, input, strlen(input), true);
return tokens;
}
token_array *tokenize(const char *input) {
token_array *tokens = token_array_new();
tokenize_add_tokens(tokens, input, strlen(input));
tokenize_add_tokens(tokens, input, strlen(input), false);
return tokens;
}