tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[tokenization] tokenized string with source

Browse Source
This commit is contained in:
Al
2015-10-04 18:27:04 -04:00
parent aa39c45b87
commit 55a5a79b4b
2 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/tokens.c
View File

@@ -19,6 +19,12 @@ tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens) {
return self; return self;
} }
inline tokenized_string_t *tokenized_string_new_from_str_size(char *src, size_t len, size_t num_tokens) {
tokenized_string_t *self = tokenized_string_new_size(len, num_tokens);
self->str = src;
return self;
}
void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position) { void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position) {
char *ptr = (char *) (src + position); char *ptr = (char *) (src + position);

1
src/tokens.h
View File

@@ -28,6 +28,7 @@ typedef struct tokenized_string {
tokenized_string_t *tokenized_string_new(void); tokenized_string_t *tokenized_string_new(void);
tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens); tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens);
tokenized_string_t *tokenized_string_new_from_str_size(char *src, size_t len, size_t num_tokens);
tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens); tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens);
void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position); void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position);
char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index); char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index);