tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[mv] csv_utils

Browse Source
This commit is contained in:
Al
2015-08-22 13:53:44 -04:00
parent cc43409b72
commit bd31dc99f2
1 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
scripts/geodata/csv_utils.py
View File

@@ -1,8 +1,16 @@
import csv import csv
import re
from encoding import safe_encode, safe_decode
newline_regex = re.compile('\r\n|\r|\n')
csv.register_dialect('tsv_no_quote', delimiter='\t', quoting=csv.QUOTE_NONE, quotechar='') csv.register_dialect('tsv_no_quote', delimiter='\t', quoting=csv.QUOTE_NONE, quotechar='')
def tsv_string(s):
return safe_encode(newline_regex.sub(u', ', safe_decode(s).strip()).replace(u'\t', u' '))
def unicode_csv_reader(filename, **kw): def unicode_csv_reader(filename, **kw):
for line in csv.reader(filename, **kw): for line in csv.reader(filename, **kw):
yield [unicode(c, 'utf-8') for c in line] yield [unicode(c, 'utf-8') for c in line]