[phrases] Using safe_encode/safe_decode as default trie serializer/deserializer

scripts/geodata/address_expansions/gazetteers.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import ujson as json
 
 from collections import defaultdict, OrderedDict
 
@@ -27,6 +28,8 @@ POSSIBLE_ROMAN_NUMERALS = set(['i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii',
 
 
 class DictionaryPhraseFilter(PhraseFilter):
+    serialize = safe_encode
+    deserialize = safe_decode
 
     def __init__(self, *dictionaries):
         self.dictionaries = dictionaries

scripts/geodata/text/phrases.py

@@ -1,8 +1,8 @@
 import six
-import ujson as json
 
 from collections import *
 from marisa_trie import BytesTrie
+from geodata.encoding import safe_encode, safe_decode
 
 SENTINEL = None
 
@@ -11,8 +11,8 @@ class PhraseFilter(object):
     def __init__(self, phrases):
         self.trie = [(key, self.serialize(val)) for key, val in six.iteritems(phrases)]
 
-    serialize = json.dumps
+    serialize = safe_encode
-    deserialize = json.loads
+    deserialize = safe_decode
 
     def filter(self, tokens):
         def return_item(item):