[tokenization] tokenized string with source

src/tokens.c

@@ -19,6 +19,12 @@ tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens) {
     return self;
 }
 
+inline tokenized_string_t *tokenized_string_new_from_str_size(char *src, size_t len, size_t num_tokens) {
+    tokenized_string_t *self = tokenized_string_new_size(len, num_tokens);
+    self->str = src;
+    return self;
+}
+
 
 void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position) {
     char *ptr = (char *) (src + position);