tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[categories] Moving category configs to YAML files

Browse Source
This commit is contained in:
Al
2016-03-28 15:37:37 -04:00
parent 161012f9f4
commit ef055e10b6
1 changed files with 13 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
scripts/geodata/categories/scrape_nominatim_special_phrases.py
View File

@@ -23,6 +23,7 @@ import requests
import six import six
import sys import sys
import time import time
import yaml
this_dir = os.path.realpath(os.path.dirname(__file__)) this_dir = os.path.realpath(os.path.dirname(__file__))
sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir))) sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir)))
@@ -102,11 +103,19 @@ def scrape_all_nominatim_category_pages(url=NOMINATIM_SPECIAL_PHRASES_URL):
def main(url=NOMINATIM_SPECIAL_PHRASES_URL, output_dir=DEFAULT_CATEGORIES_DIR): def main(url=NOMINATIM_SPECIAL_PHRASES_URL, output_dir=DEFAULT_CATEGORIES_DIR):
languages = scrape_all_nominatim_category_pages(url=url) languages = scrape_all_nominatim_category_pages(url=url)
for lang, phrases in six.iteritems(languages): for lang, phrases in six.iteritems(languages):
filename = os.path.join(output_dir, '{}.csv'.format(lang.lower())) filename = os.path.join(output_dir, '{}.yml'.format(lang.lower()))
with open(filename, 'w') as f: with open(filename, 'w') as f:
writer = csv.writer(f) phrase_data = [
for phrase, key, value, is_plural in phrases: {
writer.writerow((safe_encode(phrase), key, value, str(int(is_plural)))) 'phrase': phrase,
'key': key,
'value': value,
'is_plural': is_plural
}
for phrase, key, value, is_plural in phrases
]
yaml.dump(phrase_data, f, allow_unicode=True, default_flow_style=False)
print('Done') print('Done')