diff --git a/src/tokens.c b/src/tokens.c index 976d2b06..f3646743 100644 --- a/src/tokens.c +++ b/src/tokens.c @@ -3,17 +3,27 @@ tokenized_string_t *tokenized_string_new(void) { tokenized_string_t *self = malloc(sizeof(tokenized_string_t)); - self->str = cstring_array_new(); + self->str = NULL; + self->strings = cstring_array_new(); self->tokens = token_array_new(); return self; } +tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens) { + tokenized_string_t *self = malloc(sizeof(tokenized_string_t)); + self->str = NULL; + self->strings = cstring_array_new_size(len); + self->tokens = token_array_new_size(len); + + return self; +} + void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position) { char *ptr = (char *) (src + position); - cstring_array_add_string_len(self->str, ptr, len); + cstring_array_add_string_len(self->strings, ptr, len); token_t token = (token_t){position, len, token_type}; token_array_push(self->tokens, token); @@ -22,7 +32,8 @@ void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_ tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens) { tokenized_string_t *self = malloc(sizeof(tokenized_string_t)); - self->str = cstring_array_new_size(strlen(src) + tokens->n); + self->str = src; + self->strings = cstring_array_new_size(strlen(src) + tokens->n); if (copy_tokens) { self->tokens = token_array_new_copy(tokens, tokens->n); } else { @@ -33,7 +44,7 @@ tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, for (int i = 0; i < tokens->n; i++) { token = tokens->a[i]; - cstring_array_add_string_len(self->str, src + token.offset, token.len); + cstring_array_add_string_len(self->strings, src + token.offset, token.len); } return self; } @@ -41,7 +52,7 @@ tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index) { if (index < self->tokens->n) { - return cstring_array_get_string(self->str, index); + return cstring_array_get_string(self->strings, index); } else { return NULL; } @@ -50,8 +61,8 @@ char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index) { void tokenized_string_destroy(tokenized_string_t *self) { if (!self) return; - if (self->str) - cstring_array_destroy(self->str); + if (self->strings) + cstring_array_destroy(self->strings); if (self->tokens) token_array_destroy(self->tokens); free(self); diff --git a/src/tokens.h b/src/tokens.h index 8ef44fe4..c8ab2d08 100644 --- a/src/tokens.h +++ b/src/tokens.h @@ -21,11 +21,13 @@ typedef struct token { VECTOR_INIT(token_array, token_t) typedef struct tokenized_string { - cstring_array *str; + char *str; + cstring_array *strings; token_array *tokens; } tokenized_string_t; tokenized_string_t *tokenized_string_new(void); +tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens); tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens); void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position); char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index);