tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[openaddresses] adding clear_cache method to clear the LRU cache for point-in-polygon indices and using it in OpenAddresses import since it heavily reuses polygons and only for the current file

Browse Source
This commit is contained in:
Al
2016-10-22 20:28:59 -04:00
parent d51a1d6196
commit 63edd53fb3
2 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
scripts/geodata/openaddresses/formatter.py
View File

@@ -255,6 +255,9 @@ class OpenAddressesFormatter(object):
latitude_index = headers.index('LAT') latitude_index = headers.index('LAT')
longitude_index = headers.index('LON') longitude_index = headers.index('LON')
# Clear cached polygons
self.components.osm_admin_rtree.clear_cache()
for row in reader: for row in reader:
try: try:
latitude = float(row[latitude_index]) latitude = float(row[latitude_index])

7
scripts/geodata/polygons/index.py
View File

@@ -1,4 +1,5 @@
import fiona import fiona
import gc
import geohash import geohash
import os import os
import rtree import rtree
@@ -86,6 +87,12 @@ class PolygonIndex(object):
def setup(self): def setup(self):
pass pass
def clear_cache(self, garbage_collect=True):
if self.persistent_polygons and self.cache_size > 0:
self.polygons.clear()
if garbage_collect:
gc.collect()
def simplify_polygon(self, poly, simplify_tolerance=None, preserve_topology=None): def simplify_polygon(self, poly, simplify_tolerance=None, preserve_topology=None):
if simplify_tolerance is None: if simplify_tolerance is None:
simplify_tolerance = self.simplify_tolerance simplify_tolerance = self.simplify_tolerance