From 2991ffd193324a560a149e3e1cd99393b82e66bb Mon Sep 17 00:00:00 2001 From: Tom Davis Date: Sat, 23 Jul 2016 16:41:04 -0400 Subject: [PATCH] Don't call `download_multipart` for 1 chunk Previously, where a file was larger than `$LARGE_FILE_SIZE` but smaller than `$CHUNK_SIZE*2`, `download_multipart` would be called but would only download one (1) chunk that was the whole file. This fix keeps the same download performance as before but optimizes processing chunks out. --- src/libpostal_data | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libpostal_data b/src/libpostal_data index ac5f763a..27bcf30d 100755 --- a/src/libpostal_data +++ b/src/libpostal_data @@ -36,10 +36,11 @@ export LC_ALL=C EPOCH_DATE="Jan 1 00:00:00 1970" MB=$((1024*1024)) -LARGE_FILE_SIZE=$((100*$MB)) - CHUNK_SIZE=$((64*$MB)) +LARGE_FILE_SIZE=$((CHUNK_SIZE*2)) + + NUM_WORKERS=10 function kill_background_processes {