[parser] fixing chunked shuffle, making awk splitting work on Mac
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
#include "shuffle.h"
|
#include "shuffle.h"
|
||||||
|
|
||||||
#include <config.h>
|
#include <config.h>
|
||||||
|
|
||||||
#include "string_utils.h"
|
#include "string_utils.h"
|
||||||
|
|
||||||
// Run shuf/gshuf on a file in-place if the shuf command is available.
|
// Run shuf/gshuf on a file in-place if the shuf command is available.
|
||||||
@@ -56,10 +57,13 @@ bool shuffle_file_chunked(char *filename, size_t parts) {
|
|||||||
char *outfile = filename;
|
char *outfile = filename;
|
||||||
|
|
||||||
char_array *command = char_array_new();
|
char_array *command = char_array_new();
|
||||||
|
if (command == NULL) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
// Split the file randomly into $parts files
|
// Split the file randomly into $parts files
|
||||||
// Need to be assigned randomly, not just every nth line or it's not really a random permutation
|
// Need to be assigned randomly, not just every nth line or it's not really a random permutation
|
||||||
char_array_cat_printf(command, "awk -v parts=%zu -v filename=%s 'BEGIN{srand();} { print > filename\".\"int(rand() * parts) }'", parts, filename);
|
char_array_cat_printf(command, "awk -v parts=%zu 'BEGIN{srand();} { f = \"%s.\"int(rand() * parts); print > f }' %s", parts, filename, filename);
|
||||||
|
|
||||||
int ret = system(char_array_get_string(command));
|
int ret = system(char_array_get_string(command));
|
||||||
if (ret != EXIT_SUCCESS) {
|
if (ret != EXIT_SUCCESS) {
|
||||||
@@ -93,6 +97,7 @@ bool shuffle_file_chunked(char *filename, size_t parts) {
|
|||||||
goto exit_char_array_allocated;
|
goto exit_char_array_allocated;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
exit_char_array_allocated:
|
exit_char_array_allocated:
|
||||||
char_array_destroy(command);
|
char_array_destroy(command);
|
||||||
return ret == EXIT_SUCCESS;
|
return ret == EXIT_SUCCESS;
|
||||||
|
|||||||
Reference in New Issue
Block a user