tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[tokens] Adding a copy_tokens option for tokenized_string

Browse Source
This commit is contained in:
Al
2015-09-11 15:03:09 -07:00
parent 9ce658b7a3
commit 32c180528f
2 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/tokens.c
View File

@@ -20,10 +20,14 @@ void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_
}
tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens) {
tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens) {
tokenized_string_t *self = malloc(sizeof(tokenized_string_t));
self->str = cstring_array_new_size(strlen(src) + tokens->n);
self->tokens = tokens;
if (copy_tokens) {
self->tokens = token_array_new_copy(tokens);;
} else {
self->tokens = tokens;
}
token_t token;

2
src/tokens.h
View File

@@ -26,7 +26,7 @@ typedef struct tokenized_string {
} tokenized_string_t;
tokenized_string_t *tokenized_string_new(void);
tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens);
tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens);
void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position);
char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index);
void tokenized_string_destroy(tokenized_string_t *self);