tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[languages] Moving search_suffix and search_prefix into methods

Browse Source
This commit is contained in:
Al
2015-08-24 14:04:28 -04:00
parent c68f56e61d
commit 8bbcb60aee
1 changed files with 16 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/geodata/language_id/disambiguation.py
View File

@@ -97,6 +97,18 @@ class DictionaryPhraseFilter(PhraseFilter):
else: else:
return None, 0 return None, 0
def search_suffix(self, token):
suffix_search, suffix_len = self.search_substring(SUFFIX_KEY + token[::-1])
if suffix_len > 0:
suffix_len -= len(SUFFIX_KEY)
return suffix_search, suffix_len
def search_prefix(self, token):
prefix_search, prefix_len = self.search_substring(PREFIX_KEY + token)
if prefix_len > 0:
prefix_len -= len(PREFIX_KEY)
return prefix_search, prefix_len
def basic_filter(self, tokens): def basic_filter(self, tokens):
return super(DictionaryPhraseFilter, self).filter(tokens) return super(DictionaryPhraseFilter, self).filter(tokens)
@@ -105,13 +117,13 @@ class DictionaryPhraseFilter(PhraseFilter):
if c != token_types.PHRASE: if c != token_types.PHRASE:
token = t[1] token = t[1]
token_len = len(token) token_len = len(token)
suffix_search, suffix_len = self.search_substring(SUFFIX_KEY + token[::-1])
if suffix_search and self.trie.get(token[token_len - (suffix_len - len(SUFFIX_KEY)):].rstrip('.')): suffix_search, suffix_len = self.search_suffix(token)
if suffix_search and self.trie.get(token[(token_len - suffix_len):].rstrip('.')):
yield (token_types.PHRASE, [(c,) + t], suffix_search) yield (token_types.PHRASE, [(c,) + t], suffix_search)
continue continue
prefix_search, prefix_len = self.search_substring(PREFIX_KEY + token) prefix_search, prefix_len = self.search_prefix(token)
if prefix_search and self.trie.get(token[:(prefix_len - len(PREFIX_KEY))]): if prefix_search and self.trie.get(token[:prefix_len]):
yield (token_types.PHRASE, [(c,) + t], prefix_search) yield (token_types.PHRASE, [(c,) + t], prefix_search)
continue continue
yield c, t, data yield c, t, data