[wof] adding basic WhosOnFirst client for interaction with S3, downloading by ID, etc.

scripts/geodata/whosonfirst/client.py

@@ -0,0 +1,55 @@
+import boto3
+import os
+import six
+import ujson as json
+
+from geodata.encoding import safe_encode
+from geodata.file_utils import ensure_dir
+
+
+class WhosOnFirst(object):
+    WOF_S3_BUCKET = 'whosonfirst.mapzen.com'
+
+    def __init__(self, wof_dir, **s3_args):
+        self.s3 = boto3.client('s3')
+        self.wof_dir = wof_dir
+
+    @classmethod
+    def path_and_filename(cls, wof_id):
+        id_str = safe_encode(wof_id)
+        n = 3
+        parts = [id_str[i:i + n] for i in six.moves.xrange(0, len(id_str), n)]
+        filename = six.u('{}.geojson').format(wof_id)
+        return six.u('/').join(parts), filename
+
+    def local_path(self, wof_id):
+        s3_path, filename = self.path_and_filename(wof_id)
+        local_path = s3_path
+        if os.sep != six.u('/'):
+            local_path = s3_path.replace(six.u('/'), os.sep)
+        return os.path.join(self.wof_dir, local_path, filename)
+
+    def exists_locally(self, wof_id):
+        local_path = self.local_path(wof_id)
+        return os.path.exists(local_path)
+
+    def load(self, wof_id):
+        local_path = self.local_path(wof_id)
+        return json.load(open(local_path))
+
+    def download_file(self, wof_id):
+        s3_path, filename = self.path_and_filename(wof_id)
+
+        local_path = self.local_path(wof_id)
+        local_dir = os.path.dirname(local_path)
+
+        s3_key = six.u('/').join(('data', s3_path, filename))
+        try:
+            bucket = self.WOF_S3_BUCKET
+            self.s3.head_object(Bucket=bucket, Key=s3_key)
+            ensure_dir(local_dir)
+            if not os.path.exists(local_path):
+                self.s3.download_file(self.WOF_S3_BUCKET, s3_key, local_path)
+            return True
+        except Exception:
+            return False