From cee9da05d61b0472a04d9a319557a4e41057d908 Mon Sep 17 00:00:00 2001 From: Al Date: Wed, 28 Oct 2015 21:37:41 -0400 Subject: [PATCH] [fix] using tokenize_raw API --- python/postal/text/normalize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/postal/text/normalize.py b/python/postal/text/normalize.py index 0fbe0f4e..2ed758f0 100644 --- a/python/postal/text/normalize.py +++ b/python/postal/text/normalize.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from postal.text import _normalize -from postal.text import _tokenize +from postal.text.tokenize import tokenize_raw from postal.text.token_types import token_types from postal.text.encoding import safe_decode @@ -40,5 +40,5 @@ def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS, normalized = _normalize.normalize_string_utf8(s, string_options) # Tuples of (offset, len, type) - tokens = _tokenize.tokenize(normalized) + tokens = tokenize_raw(normalized) return [(_normalize.normalize_token(normalized, t, token_options), token_types.from_id(t[-1])) for t in tokens]