From de240d2b94556d7e7af066fcf6cacc59956a104b Mon Sep 17 00:00:00 2001 From: Al Date: Sun, 17 Jan 2016 20:51:43 -0500 Subject: [PATCH] [fix] tokenize_add_tokens respects specified length --- src/scanner.c | 6 +++++- src/scanner.re | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/scanner.c b/src/scanner.c index b53f8257..94af3497 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -440360,7 +440360,9 @@ void tokenize_add_tokens(token_array *tokens, const char *input, size_t len, boo size_t token_start, token_length; uint16_t token_type; - while ( ( token_type = scan_token(&scanner)) != END ) { + size_t consumed = 0; + + while (consumed < len && (token_type = scan_token(&scanner)) != END) { token_start = scanner.start - scanner.src; token_length = scanner.cursor - scanner.start; @@ -440374,6 +440376,8 @@ void tokenize_add_tokens(token_array *tokens, const char *input, size_t len, boo token.type = token_type; token_array_push(tokens, token); + + consumed += token_length; } } diff --git a/src/scanner.re b/src/scanner.re index 593c5a51..61013e1a 100644 --- a/src/scanner.re +++ b/src/scanner.re @@ -232,7 +232,9 @@ void tokenize_add_tokens(token_array *tokens, const char *input, size_t len, boo size_t token_start, token_length; uint16_t token_type; - while ( ( token_type = scan_token(&scanner)) != END ) { + size_t consumed = 0; + + while (consumed < len && (token_type = scan_token(&scanner)) != END) { token_start = scanner.start - scanner.src; token_length = scanner.cursor - scanner.start; @@ -246,6 +248,8 @@ void tokenize_add_tokens(token_array *tokens, const char *input, size_t len, boo token.type = token_type; token_array_push(tokens, token); + + consumed += token_length; } }