tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[utils] Adding source string to tokenized_string struct

Browse Source
This commit is contained in:
Al
2015-09-15 13:21:51 -04:00
parent b2f690b6f6
commit c47ff1b113
2 changed files with 21 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/tokens.h
View File

@@ -21,11 +21,13 @@ typedef struct token {
VECTOR_INIT(token_array, token_t)
typedef struct tokenized_string {
cstring_array *str;
char *str;
cstring_array *strings;
token_array *tokens;
} tokenized_string_t;
tokenized_string_t *tokenized_string_new(void);
tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens);
tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens);
void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position);
char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index);