[openaddresses] adding Italy countrywide to a pre_release_downloads set so it can be used in libpostal without having been merged yet

resources/parser/data_sets/openaddresses.yaml

@@ -12,6 +12,10 @@ global:
     place_only_probability: 0.2
     place_and_postcode_probability: 0.1
 
+    pre_release_downloads:
+        # Italy countrywide with postcodes, must have
+        - "http://s3.amazonaws.com/data.openaddresses.io/runs/104280/it/countrywide.zip"
+
     fields: &default_fields
         - field_name: NUMBER
           component: house_number
@@ -557,6 +561,8 @@ countries:
               add_osm_boundaries: true
 
     it:
+        files:
+            - filename: countrywide.csv
         subdirs:
             # Trentino-Alto Adige/Südtirol
             32:

scripts/geodata/openaddresses/config.py

@@ -14,6 +14,7 @@ class OpenAddressesConfig(object):
 
         config = yaml.load(open(path))
         self.config = config['global']
+        self.pre_release_downloads = config.get('pre_release_downloads', [])
         self.country_configs = config['countries']
 
     @property

scripts/geodata/openaddresses/download_openaddresses.py

@@ -7,7 +7,7 @@ import sys
 import tempfile
 import yaml
 
-from six.moves.urllib_parse import urljoin, quote_plus
+from six.moves.urllib_parse import urljoin, quote_plus, unquote_plus
 
 this_dir = os.path.realpath(os.path.dirname(__file__))
 sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir)))
@@ -37,6 +37,17 @@ def download_and_unzip_file(url, out_dir):
     return success
 
 
+def download_pre_release_downloads(out_dir):
+    for url in openaddresses_config.pre_release_downloads:
+        print(six.u('doing pre_release {}').format(safe_decode(url)))
+
+        success = download_and_unzip_file(url, out_dir)
+        if not success:
+            print(six.u('ERR: could not download {}').format(source))
+            return False
+    return True
+
+
 def openaddresses_download_all_files(out_dir):
     temp_dir = tempfile.gettempdir()
 
@@ -50,6 +61,8 @@ def openaddresses_download_all_files(out_dir):
     source_index = headers.index('source')
     url_index = headers.index('processed')
 
+    download_pre_release_downloads(out_dir)
+
     for row in reader:
         source = row[source_index].rsplit('.')[0]
         processed = row[url_index]
@@ -74,6 +87,8 @@ def openaddresses_download_configured_files(out_dir):
 
         url = urljoin(OPENADDRESSES_LATEST_DIR, zip_url_path)
 
+        download_pre_release_downloads(out_dir)
+
         print(six.u('doing {}').format(safe_decode(source)))
         success = download_and_unzip_file(url, out_dir)
         if not success: