tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #98 from uberbaud/posix_sh

Browse Source 
Use posix `sh` for systems without `bash`
This commit is contained in:
Al Barrentine
2016-07-27 18:44:11 -04:00
committed by GitHub
parent 65c4688f89 18c8e90eb3
commit 3e3950b37a
2 changed files with 14 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bootstrap.sh
View File

@@ -1,2 +1,2 @@
#!/usr/bin/env bash
#!/bin/sh
autoreconf -fi --warning=no-portability

33
src/libpostal_data
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env bash
#!/bin/sh
set -e
@@ -26,7 +26,7 @@ LIBPOSTAL_GEO_UPDATED_PATH=$LIBPOSTAL_DATA_DIR/last_updated_geo
LIBPOSTAL_PARSER_UPDATED_PATH=$LIBPOSTAL_DATA_DIR/last_updated_parser
LIBPOSTAL_LANG_CLASS_UPDATED_PATH=$LIBPOSTAL_DATA_DIR/last_updated_language_classifier
BASIC_MODULE_DIRS=(address_expansions numex transliteration)
BASIC_MODULE_DIRS="address_expansions numex transliteration"
GEODB_MODULE_DIR=geodb
PARSER_MODULE_DIR=address_parser
LANGUAGE_CLASSIFIER_MODULE_DIR=language_classifier
@@ -43,29 +43,22 @@ LARGE_FILE_SIZE=$((CHUNK_SIZE*2))
NUM_WORKERS=10
function kill_background_processes {
kill_background_processes() {
jobs -p | xargs kill;
exit
}
trap kill_background_processes SIGINT
trap kill_background_processes INT
function download_part() {
i=$1
offset=$2
max=$3
url=$4
part_filename=$5
echo "Downloading part $i: filename=$part_filename, offset=$offset, max=$max"
curl $url --silent -H"Range:bytes=$offset-$max" -o $part_filename
}
export -f download_part
PART_MSG='echo "Downloading part $1: filename=$5, offset=$2, max=$3"'
PART_CURL='curl $4 --silent -H"Range:bytes=$2-$3" -o $5'
DOWNLOAD_PART="$PART_MSG;$PART_CURL"
function download_multipart() {
download_multipart() {
url=$1
filename=$2
size=$3
num_workers=$4
num_chunks=$((size/CHUNK_SIZE))
echo "Downloading multipart: $url, size=$size, num_chunks=$num_chunks"
@@ -81,7 +74,7 @@ function download_multipart() {
fi;
printf "%s\0%s\0%s\0%s\0%s\0" "$i" "$offset" "$max" "$url" "$part_filename"
offset=$((offset+CHUNK_SIZE))
done | xargs -0 -n 5 -P $NUM_WORKERS bash -c 'download_part "$@"' --
done | xargs -0 -n 5 -P $NUM_WORKERS sh -c "$DOWNLOAD_PART" --
> $local_path
@@ -96,7 +89,7 @@ function download_multipart() {
}
function download_file() {
download_file() {
updated_path=$1
data_dir=$2
filename=$3
@@ -117,7 +110,7 @@ function download_file() {
content_length=$(curl -I $url 2> /dev/null | awk '/^Content-Length:/ { print $2 }' | tr -d '[[:space:]]')
if [ $content_length -ge $LARGE_FILE_SIZE ]; then
download_multipart $url $local_path $content_length $NUM_WORKERS
download_multipart $url $local_path $content_length
else
curl $url -o $local_path
fi
@@ -153,7 +146,7 @@ if [ $COMMAND = "download" ]; then
elif [ $COMMAND = "upload" ]; then
if [ $FILE = "base" ] || [ $FILE = "all" ]; then
tar -C $LIBPOSTAL_DATA_DIR -cvzf $LIBPOSTAL_DATA_DIR/$LIBPOSTAL_DATA_FILE ${BASIC_MODULE_DIRS[*]}
tar -C $LIBPOSTAL_DATA_DIR -cvzf $LIBPOSTAL_DATA_DIR/$LIBPOSTAL_DATA_FILE $BASIC_MODULE_DIRS
aws s3 cp --acl=public-read $LIBPOSTAL_DATA_DIR/$LIBPOSTAL_DATA_FILE $LIBPOSTAL_S3_KEY
fi