tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[build] fixing multi-part downloads, which no longer know the Content-Length so no longer know the byte ange for the last chunk. No longer needs to use any file dates/timestamps (compatibility 🎉), only the version numbers that are checked in to the repo

Browse Source
This commit is contained in:
Al
2019-02-12 13:32:58 -05:00
parent acebc951b6
commit fdb3b7e32e
1 changed files with 7 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/libpostal_data.in
View File

@@ -47,6 +47,7 @@ BASIC_MODULE_DIRS="address_expansions numex transliteration"
PARSER_MODULE_DIR=address_parser PARSER_MODULE_DIR=address_parser
LANGUAGE_CLASSIFIER_MODULE_DIR=language_classifier LANGUAGE_CLASSIFIER_MODULE_DIR=language_classifier
NUM_WORKERS=12
kill_background_processes() { kill_background_processes() {
jobs -p | xargs kill; jobs -p | xargs kill;
@@ -71,22 +72,18 @@ download_release_multipart() {
while [ $i -lt $num_chunks ]; do while [ $i -lt $num_chunks ]; do
i=$((i+1)) i=$((i+1))
part_filename="$filename.$i" part_filename="$filename.$i"
if [ $i -lt $num_chunks ]; then max=$((offset+CHUNK_SIZE-1));
max=$((offset+CHUNK_SIZE-1));
else
max="";
fi;
printf "%s\0%s\0%s\0%s\0%s\0" "$i" "$offset" "$max" "$url" "$part_filename" printf "%s\0%s\0%s\0%s\0%s\0" "$i" "$offset" "$max" "$url" "$part_filename"
offset=$((offset+CHUNK_SIZE)) offset=$((offset+CHUNK_SIZE))
done | xargs -0 -n 5 -P $NUM_WORKERS sh -c "$DOWNLOAD_PART" -- done | xargs -0 -n 5 -P $NUM_WORKERS sh -c "$DOWNLOAD_PART" --
> $local_path > $filename
i=0 i=0
while [ $i -lt $num_chunks ]; do while [ $i -lt $num_chunks ]; do
i=$((i+1)) i=$((i+1))
part_filename="$filename.$i" part_filename="$filename.$i"
cat $part_filename >> $local_path cat $part_filename >> $filename
rm $part_filename rm $part_filename
done; done;
@@ -103,14 +100,6 @@ download_release() {
shift 6 shift 6
subdirs=$@ subdirs=$@
echo $version_file_path
echo $version
echo $data_dir
echo $num_chunks
echo $filename
echo $name
echo $subdirs
local_path=$data_dir/$filename local_path=$data_dir/$filename
url=$LIBPOSTAL_BASE_URL/$version/$filename url=$LIBPOSTAL_BASE_URL/$version/$filename
@@ -118,7 +107,8 @@ download_release() {
if [ ! -e $version_file_path ]; then if [ ! -e $version_file_path ]; then
current_version="" current_version=""
else else
current_version="$(cat $version_path)" current_version="$(cat $version_file_path)"
fi; fi;
echo "Checking for new libpostal $name..." echo "Checking for new libpostal $name..."
@@ -132,16 +122,12 @@ download_release() {
curl -L $url --retry 3 --retry-delay 2 -o $local_path curl -L $url --retry 3 --retry-delay 2 -o $local_path
fi fi
if date -ud "@$(date -ur . +%s)" >/dev/null 2>&1; then
echo $(date -ud "$(date -ud "@$(date -ur $local_path +%s)") + 1 second") > $updated_path;
elif stat -f %Sm . >/dev/null 2>&1; then
echo $(date -ur $(stat -f %m $local_path) -v+1S) > $updated_path;
fi;
for subdir in $subdirs; do for subdir in $subdirs; do
rm -rf $data_dir/$subdir; rm -rf $data_dir/$subdir;
done done
tar -xvzf $local_path --no-same-owner -C $data_dir; tar -xvzf $local_path --no-same-owner -C $data_dir;
rm $local_path; rm $local_path;
echo "$version" > $version_file_path;
else else
echo "libpostal $name up to date" echo "libpostal $name up to date"
fi fi