tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[utils] Adding source string to tokenized_string struct

Browse Source
This commit is contained in:
Al
2015-09-15 13:21:51 -04:00
parent b2f690b6f6
commit c47ff1b113
2 changed files with 21 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/tokens.c
View File

@@ -3,17 +3,27 @@
tokenized_string_t *tokenized_string_new(void) {
tokenized_string_t *self = malloc(sizeof(tokenized_string_t));
self->str = cstring_array_new();
self->str = NULL;
self->strings = cstring_array_new();
self->tokens = token_array_new();
return self;
}
tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens) {
tokenized_string_t *self = malloc(sizeof(tokenized_string_t));
self->str = NULL;
self->strings = cstring_array_new_size(len);
self->tokens = token_array_new_size(len);
return self;
}
void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position) {
char *ptr = (char *) (src + position);
cstring_array_add_string_len(self->str, ptr, len);
cstring_array_add_string_len(self->strings, ptr, len);
token_t token = (token_t){position, len, token_type};
token_array_push(self->tokens, token);
@@ -22,7 +32,8 @@ void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_
tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens) {
tokenized_string_t *self = malloc(sizeof(tokenized_string_t));
self->str = cstring_array_new_size(strlen(src) + tokens->n);
self->str = src;
self->strings = cstring_array_new_size(strlen(src) + tokens->n);
if (copy_tokens) {
self->tokens = token_array_new_copy(tokens, tokens->n);
} else {
@@ -33,7 +44,7 @@ tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens,
for (int i = 0; i < tokens->n; i++) {
token = tokens->a[i];
cstring_array_add_string_len(self->str, src + token.offset, token.len);
cstring_array_add_string_len(self->strings, src + token.offset, token.len);
}
return self;
}
@@ -41,7 +52,7 @@ tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens,
char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index) {
if (index < self->tokens->n) {
return cstring_array_get_string(self->str, index);
return cstring_array_get_string(self->strings, index);
} else {
return NULL;
}
@@ -50,8 +61,8 @@ char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index) {
void tokenized_string_destroy(tokenized_string_t *self) {
if (!self)
return;
if (self->str)
cstring_array_destroy(self->str);
if (self->strings)
cstring_array_destroy(self->strings);
if (self->tokens)
token_array_destroy(self->tokens);
free(self);

4
src/tokens.h
View File

@@ -21,11 +21,13 @@ typedef struct token {
VECTOR_INIT(token_array, token_t)
typedef struct tokenized_string {
cstring_array *str;
char *str;
cstring_array *strings;
token_array *tokens;
} tokenized_string_t;
tokenized_string_t *tokenized_string_new(void);
tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens);
tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens);
void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position);
char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index);