tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove call to seq which may not exist

Browse Source
This commit is contained in:
Tom Davis
2016-07-23 01:03:15 -04:00
parent e02c6adc85
commit 24e0314e71
1 changed files with 14 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/libpostal_data
View File

@@ -42,7 +42,7 @@ CHUNK_SIZE=$((64*$MB))
NUM_WORKERS=10
function kill_background_processes {
function kill_background_processes {
jobs -p | xargs kill;
exit
}
@@ -56,7 +56,7 @@ function download_part() {
url=$4
part_filename=$5
echo "Downloading part $i: filename=$part_filename, offset=$offset, max=$max"
curl $url --silent -H"Range:bytes=$offset-$max" -o $part_filename
curl $url --silent -H"Range:bytes=$offset-$max" -o $part_filename
}
export -f download_part
@@ -69,7 +69,9 @@ function download_multipart() {
num_chunks=$((size/CHUNK_SIZE))
echo "Downloading multipart: $url, size=$size, num_chunks=$num_chunks"
offset=0
for i in $(seq 1 $((num_chunks))); do
i=0
while [ $i -lt $num_chunks ]; do
i=$((i+1))
part_filename="$filename.$i"
if [ $i -lt $num_chunks ]; then
max=$((offset+CHUNK_SIZE-1));
@@ -78,11 +80,13 @@ function download_multipart() {
fi;
printf "%s\0%s\0%s\0%s\0%s\0" "$i" "$offset" "$max" "$url" "$part_filename"
offset=$((offset+CHUNK_SIZE))
done | xargs -0 -n 5 -P $NUM_WORKERS bash -c 'download_part "$@"' --
done | xargs -0 -n 5 -P $NUM_WORKERS bash -c 'download_part "$@"' --
> $local_path
for i in `seq 1 $((num_chunks))`; do
i=0
while [ $i -lt $num_chunks ]; do
i=$((i+1))
part_filename="$filename.$i"
cat $part_filename >> $local_path
rm $part_filename
@@ -116,7 +120,7 @@ function download_file() {
else
curl $url -o $local_path
fi
if date -ur . >/dev/null 2>&1; then
echo $(date -d "$(date -d "@$(date -ur $local_path +%s)") + 1 second") > $updated_path;
elif stat -f %Sm . >/dev/null 2>&1; then
@@ -135,13 +139,13 @@ if [ $COMMAND = "download" ]; then
if [ $FILE = "base" ] || [ $FILE = "all" ]; then
download_file $LIBPOSTAL_DATA_UPDATED_PATH $LIBPOSTAL_DATA_DIR $LIBPOSTAL_DATA_FILE "data file"
fi
if [ $FILE = "geodb" ] || [ $FILE = "all" ]; then
if [ $FILE = "geodb" ] || [ $FILE = "all" ]; then
download_file $LIBPOSTAL_GEO_UPDATED_PATH $LIBPOSTAL_DATA_DIR $LIBPOSTAL_GEODB_FILE "geodb data file"
fi
if [ $FILE = "parser" ] || [ $FILE = "all" ]; then
if [ $FILE = "parser" ] || [ $FILE = "all" ]; then
download_file $LIBPOSTAL_PARSER_UPDATED_PATH $LIBPOSTAL_DATA_DIR $LIBPOSTAL_PARSER_FILE "parser data file"
fi
if [ $FILE = "language_classifier" ] || [ $FILE = "all" ]; then
if [ $FILE = "language_classifier" ] || [ $FILE = "all" ]; then
download_file $LIBPOSTAL_LANG_CLASS_UPDATED_PATH $LIBPOSTAL_DATA_DIR $LIBPOSTAL_LANG_CLASS_FILE "language classifier data file"
fi
@@ -151,7 +155,7 @@ elif [ $COMMAND = "upload" ]; then
tar -C $LIBPOSTAL_DATA_DIR -cvzf $LIBPOSTAL_DATA_DIR/$LIBPOSTAL_DATA_FILE ${BASIC_MODULE_DIRS[*]}
aws s3 cp --acl=public-read $LIBPOSTAL_DATA_DIR/$LIBPOSTAL_DATA_FILE $LIBPOSTAL_S3_KEY
fi
if [ $FILE = "geodb" ] || [ $FILE = "all" ]; then
tar -C $LIBPOSTAL_DATA_DIR -cvzf $LIBPOSTAL_DATA_DIR/$LIBPOSTAL_GEODB_FILE $GEODB_MODULE_DIR
aws s3 cp --acl=public-read $LIBPOSTAL_DATA_DIR/$LIBPOSTAL_GEODB_FILE $LIBPOSTAL_S3_KEY