tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[osm] Using tsv_no_quote writers in all OSM training data files

Browse Source
This commit is contained in:
Al
2015-08-13 18:40:41 -04:00
parent 9464670174
commit 46f2c68a69
1 changed files with 11 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/geodata/osm/osm_address_training_data.py
View File

@@ -24,6 +24,7 @@ from address_normalizer.text.tokenize import *
from geodata.i18n.languages import * from geodata.i18n.languages import *
from geodata.polygons.language_polys import * from geodata.polygons.language_polys import *
from geodata.csv_utils import *
from geodata.file_utils import * from geodata.file_utils import *
this_dir = os.path.realpath(os.path.dirname(__file__)) this_dir = os.path.realpath(os.path.dirname(__file__))
@@ -78,7 +79,7 @@ def parse_osm(filename, allowed_types=ALL_OSM_TAGS):
def write_osm_json(filename, out_filename): def write_osm_json(filename, out_filename):
out = open(out_filename, 'w') out = open(out_filename, 'w')
writer = csv.writer(out, delimiter='\t') writer = csv.writer(out, 'tsv_no_quote')
for key, attrs in parse_osm(filename): for key, attrs in parse_osm(filename):
writer.writerow((key, json.dumps(attrs))) writer.writerow((key, json.dumps(attrs)))
out.close() out.close()
@@ -343,7 +344,7 @@ def get_language_names(language_rtree, key, value, tag_prefix='name'):
def build_ways_training_data(language_rtree, infile, out_dir): def build_ways_training_data(language_rtree, infile, out_dir):
i = 0 i = 0
f = open(os.path.join(out_dir, WAYS_LANGUAGE_DATA_FILENAME), 'w') f = open(os.path.join(out_dir, WAYS_LANGUAGE_DATA_FILENAME), 'w')
writer = csv.writer(f, delimiter='\t') writer = csv.writer(f, 'tsv_no_quote')
for key, value in parse_osm(infile, allowed_types=WAYS_RELATIONS): for key, value in parse_osm(infile, allowed_types=WAYS_RELATIONS):
country, name_language = get_language_names(language_rtree, key, value, tag_prefix='name') country, name_language = get_language_names(language_rtree, key, value, tag_prefix='name')
@@ -371,10 +372,10 @@ def build_address_format_training_data(language_rtree, infile, out_dir):
formatter = AddressFormatter(splitter='\n') formatter = AddressFormatter(splitter='\n')
formatted_file = open(os.path.join(out_dir, ADDRESS_FORMAT_DATA_FILENAME), 'w') formatted_file = open(os.path.join(out_dir, ADDRESS_FORMAT_DATA_FILENAME), 'w')
formatted_writer = csv.writer(formatted_file, delimiter='\t') formatted_writer = csv.writer(formatted_file, 'tsv_no_quote')
formatted_tagged_file = open(os.path.join(out_dir, ADDRESS_FORMAT_DATA_TAGGED_FILENAME), 'w') formatted_tagged_file = open(os.path.join(out_dir, ADDRESS_FORMAT_DATA_TAGGED_FILENAME), 'w')
formatted_tagged_writer = csv.writer(formatted_tagged_file, delimiter='\t') formatted_tagged_writer = csv.writer(formatted_tagged_file, 'tsv_no_quote')
for key, value in parse_osm(infile): for key, value in parse_osm(infile):
try: try:
@@ -390,11 +391,11 @@ def build_address_format_training_data(language_rtree, infile, out_dir):
formatted_address_untagged = formatter.format_address(country, value, tag_components=False) formatted_address_untagged = formatter.format_address(country, value, tag_components=False)
if formatted_address_tagged is not None: if formatted_address_tagged is not None:
formatted_address_tagged = safe_encode(formatted_address_tagged.replace('\n', '\\n')) formatted_address_tagged = safe_encode(formatted_address_tagged.replace('\n', '\\n'))
formatted_tagged_writer.writerow((country, default_languages[0]['lang'], formatted_address_tagged)) formatted_tagged_writer.writerow((default_languages[0]['lang'], country, formatted_address_tagged))
if formatted_address_untagged is not None: if formatted_address_untagged is not None:
formatted_address_untagged = safe_encode(formatted_address_untagged.replace('\n', '\\n')) formatted_address_untagged = safe_encode(formatted_address_untagged.replace('\n', '\\n'))
formatted_writer.writerow((country, default_languages[0]['lang'], formatted_address_untagged)) formatted_writer.writerow((default_languages[0]['lang'], country, formatted_address_untagged))
if formatted_address_tagged is not None or formatted_address_untagged is not None: if formatted_address_tagged is not None or formatted_address_untagged is not None:
i += 1 i += 1
@@ -405,7 +406,7 @@ def build_address_format_training_data(language_rtree, infile, out_dir):
def build_address_training_data(langauge_rtree, infile, out_dir, format=False): def build_address_training_data(langauge_rtree, infile, out_dir, format=False):
i = 0 i = 0
f = open(os.path.join(out_dir, ADDRESS_LANGUAGE_DATA_FILENAME), 'w') f = open(os.path.join(out_dir, ADDRESS_LANGUAGE_DATA_FILENAME), 'w')
writer = csv.writer(f, delimiter='\t') writer = csv.writer(f, 'tsv_no_quote')
for key, value in parse_osm(infile): for key, value in parse_osm(infile):
country, street_language = get_language_names(language_rtree, key, value, tag_prefix='addr:street') country, street_language = get_language_names(language_rtree, key, value, tag_prefix='addr:street')
@@ -432,7 +433,7 @@ def build_venue_training_data(language_rtree, infile, out_dir):
i = 0 i = 0
f = open(os.path.join(out_dir, VENUE_LANGUAGE_DATA_FILENAME), 'w') f = open(os.path.join(out_dir, VENUE_LANGUAGE_DATA_FILENAME), 'w')
writer = csv.writer(f, delimiter='\t') writer = csv.writer(f, 'tsv_no_quote')
for key, value in parse_osm(infile): for key, value in parse_osm(infile):
country, name_language = get_language_names(language_rtree, key, value, tag_prefix='name') country, name_language = get_language_names(language_rtree, key, value, tag_prefix='name')
@@ -444,8 +445,9 @@ def build_venue_training_data(language_rtree, infile, out_dir):
continue continue
for k, v in name_language.iteritems(): for k, v in name_language.iteritems():
for s in v: for s in v:
s = s.strip()
if k in languages: if k in languages:
writer.writerow((k, country, safe_encode(venue_type), safe_encode(s))) writer.writerow((k, country, safe_encode(venue_type), safe_encode(s.replace('\t', ' '))))
if i % 1000 == 0 and i > 0: if i % 1000 == 0 and i > 0:
print 'did', i, 'venues' print 'did', i, 'venues'
i += 1 i += 1