[tokenization] Exposing some of the scanner's methods in header for use in the Python scanner so it can avoid the additional allocation

src/scanner.h

@@ -8,6 +8,14 @@ extern "C" {
 #include "token_types.h"
 #include "tokens.h"
 
+typedef struct scanner {
+    unsigned char *src, *cursor, *start, *end;
+} scanner_t;
+
+int scan_token(scanner_t *s);
+
+inline scanner_t scanner_from_string(const char *input);
+
 tokenized_string_t *tokenize(const char *str);