62 lines
1.8 KiB
Bash
Executable File
62 lines
1.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
if [ "$#" -ne 2 ]; then
|
|
echo "Usage: ./libpostal_data [upload|download] data_dir"
|
|
fi
|
|
|
|
LIBPOSTAL_S3_BUCKET_NAME="libpostal"
|
|
LIBPOSTAL_S3_KEY="s3://$LIBPOSTAL_S3_BUCKET_NAME"
|
|
LIBPOSTAL_S3_BUCKET_URL="http://$LIBPOSTAL_S3_BUCKET_NAME.s3.amazonaws.com"
|
|
LIBPOSTAL_DATA_FILE="libpostal_data.tar.gz"
|
|
|
|
|
|
COMMAND=$1
|
|
LIBPOSTAL_DATA_DIR=$2
|
|
|
|
LIBPOSTAL_DATA_UPDATED_PATH=$LIBPOSTAL_DATA_DIR/last_updated
|
|
|
|
BASIC_MODULE_DIRS=(address_expansins numex transliteration)
|
|
|
|
EPOCH_DATE="Jan 1 00:00:00 1970"
|
|
|
|
download_file() {
|
|
updated_path=$1
|
|
data_dir=$2
|
|
filename=$3
|
|
name=$4
|
|
|
|
local_path=$data_dir/$filename
|
|
|
|
if [ ! -e $updated_path ]; then
|
|
echo "$EPOCH_DATE" > $updated_path;
|
|
fi;
|
|
|
|
echo "Checking for new libpostal $name..."
|
|
|
|
if [ $(curl $LIBPOSTAL_S3_BUCKET_URL/$filename -z "$(cat $updated_path)" --silent --remote-time -o $local_path -w %{http_code}) = "200" ]; then
|
|
echo "New libpostal $name available"
|
|
if date -r . >/dev/null 2>&1; then
|
|
echo $(date -d "$(date -d "@$(date -r $local_path +%s)") + 1 second") > $updated_path;
|
|
elif stat -f %Sm . >/dev/null 2>&1; then
|
|
echo $(date -r $(stat -f %m $local_path) -v+1S) > $updated_path;
|
|
fi;
|
|
tar -xvzf $local_path -C $data_dir;
|
|
rm $local_path;
|
|
else
|
|
echo "libpostal $name up to date"
|
|
fi
|
|
}
|
|
|
|
if [ $COMMAND = "download" ]; then
|
|
mkdir -p $LIBPOSTAL_DATA_DIR
|
|
|
|
download_file $LIBPOSTAL_DATA_UPDATED_PATH $LIBPOSTAL_DATA_DIR $LIBPOSTAL_DATA_FILE "data file"
|
|
|
|
elif [ $COMMAND = "upload" ]; then
|
|
tar -C $LIBPOSTAL_DATA_DIR -cvzf $LIBPOSTAL_DATA_DIR/$LIBPOSTAL_DATA_FILE ${BASIC_MODULE_DIRS[*]}
|
|
aws s3 cp --acl=public-read $LIBPOSTAL_DATA_DIR/$LIBPOSTAL_DATA_FILE $LIBPOSTAL_S3_KEY
|
|
else
|
|
echo "Invalid command: $COMMAND"
|
|
exit 1
|
|
fi
|