From b9ee3be8062f012c47c919cb0fefbc6f5a18fa1a Mon Sep 17 00:00:00 2001 From: Al Date: Thu, 5 May 2016 02:20:12 -0400 Subject: [PATCH] [phrases] Using simple string encoding/decoding for default serialize/deserialize in PhraseFilter base class --- scripts/geodata/text/phrases.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/geodata/text/phrases.py b/scripts/geodata/text/phrases.py index bd4864f9..f9e5e693 100644 --- a/scripts/geodata/text/phrases.py +++ b/scripts/geodata/text/phrases.py @@ -9,10 +9,13 @@ SENTINEL = None class PhraseFilter(object): def __init__(self, phrases): - self.trie = [(key, self.serialize(val)) for key, val in six.iteritems(phrases)] + if hasattr(phrases, 'items'): + phrases = six.iteritems(phrases) + vals = [(safe_decode(key), self.serialize(val)) for key, val in phrases] + self.trie = BytesTrie(vals) - serialize = safe_encode - deserialize = safe_decode + serialize = staticmethod(safe_encode) + deserialize = staticmethod(safe_decode) def filter(self, tokens): def return_item(item):