tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated linenoise to be MSys2/MinGW compatible. Updated address_parser app to use the defined libpostal api and not include internal components directly. Removed windows src Makefile as it is now the same as the standard one.

Browse Source
This commit is contained in:
AeroXuk
2017-11-27 01:42:25 +00:00
parent bb5535602a
commit 69e0d5d963
5 changed files with 94 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/Makefile.am
View File

@@ -34,8 +34,8 @@ libpostal_CFLAGS = $(CFLAGS_O3)
bench_SOURCES = bench.c
bench_LDADD = libpostal.la libscanner.la $(CBLAS_LIBS)
bench_CFLAGS = $(CFLAGS_O3)
address_parser_SOURCES = address_parser_cli.c json_encode.c linenoise/linenoise.c address_parser.c address_parser_io.c averaged_perceptron.c crf.c crf_context.c sparse_matrix.c graph.c graph_builder.c float_utils.c averaged_perceptron_tagger.c address_dictionary.c normalize.c numex.c features.c unicode_scripts.c transliterate.c trie.c trie_search.c trie_utils.c string_utils.c tokens.c file_utils.c utf8proc/utf8proc.c ngrams.c language_classifier.c language_features.c logistic_regression.c logistic.c minibatch.c
address_parser_LDADD = libpostal.la libscanner.la $(CBLAS_LIBS)
address_parser_SOURCES = strndup.c address_parser_cli.c json_encode.c linenoise/linenoise.c string_utils.c utf8proc/utf8proc.c
address_parser_LDADD = libpostal.la $(CBLAS_LIBS)
address_parser_CFLAGS = $(CFLAGS_O3)
build_address_dictionary_SOURCES = strndup.c address_dictionary_builder.c address_dictionary.c file_utils.c string_utils.c trie.c trie_search.c utf8proc/utf8proc.c

37
src/address_parser_cli.c
View File

@@ -1,35 +1,15 @@
#include <stdio.h>
#include <stdlib.h>
#include "address_parser.h"
#include "averaged_perceptron_tagger.h"
#include "address_dictionary.h"
#include "collections.h"
#include "constants.h"
#include "file_utils.h"
#include "json_encode.h"
#include "libpostal.h"
#include "normalize.h"
#include "scanner.h"
#include "shuffle.h"
#include "tokens.h"
#include "linenoise/linenoise.h"
#include "log/log.h"
bool load_address_parser_dependencies(void) {
if (!address_dictionary_module_setup(NULL)) {
log_error("Could not load address dictionaries\n");
return false;
}
log_info("address dictionary module loaded\n");
return true;
}
#include "strndup.h"
int main(int argc, char **argv) {
char *address_parser_dir = LIBPOSTAL_ADDRESS_PARSER_DIR;
char *address_parser_dir = NULL;
char *history_file = "address_parser.history";
if (argc > 1) {
@@ -38,7 +18,7 @@ int main(int argc, char **argv) {
printf("Loading models...\n");
if (!libpostal_setup() || !address_parser_module_setup(address_parser_dir)) {
if (!libpostal_setup() || !libpostal_setup_parser_datadir(address_parser_dir)) {
exit(EXIT_FAILURE);
}
@@ -54,8 +34,6 @@ int main(int argc, char **argv) {
char *input = NULL;
address_parser_t *parser = get_address_parser();
while((input = linenoise("> ")) != NULL) {
if (input[0] != '\0') {
@@ -63,7 +41,7 @@ int main(int argc, char **argv) {
linenoiseHistorySave(history_file); /* Save the history on disk. */
}
if (strcmp(input, ".exit") == 0) {
if (strncmp(input, ".exit", 5) == 0) {
printf("Fin!\n");
free(input);
break;
@@ -95,7 +73,7 @@ int main(int argc, char **argv) {
cstring_array_destroy(command);
goto next_input;
} else if (string_starts_with(input, ".print_features")) {
} /*else if (string_starts_with(input, ".print_features")) {
size_t num_tokens = 0;
cstring_array *command = cstring_array_split(input, " ", 1, &num_tokens);
if (cstring_array_num_strings(command) > 1) {
@@ -111,13 +89,16 @@ int main(int argc, char **argv) {
cstring_array_destroy(command);
goto next_input;
} else if (strlen(input) == 0) {
}*/ else if (strlen(input) == 0) {
goto next_input;
}
libpostal_address_parser_response_t *parsed;
libpostal_address_parser_options_t options = libpostal_get_address_parser_default_options();
if (country != NULL) options.country = country;
if (language != NULL) options.language = language;
if ((parsed = libpostal_parse_address(input, options))) {
printf("\n");
printf("Result:\n\n");

69
src/linenoise/linenoise.c
View File

@@ -105,7 +105,6 @@
*
*/
#include <termios.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
@@ -114,7 +113,12 @@
#include <stdlib.h>
#include <ctype.h>
#include <sys/types.h>
#ifndef _WIN32
#include <termios.h>
#include <sys/ioctl.h>
#endif //_WIN32
#include <unistd.h>
#include "linenoise.h"
@@ -123,8 +127,10 @@
static char *unsupported_term[] = {"dumb","cons25","emacs",NULL};
static linenoiseCompletionCallback *completionCallback = NULL;
#ifndef _WIN32
static struct termios orig_termios; /* In order to restore at exit.*/
static int rawmode = 0; /* For atexit() function to check if restore is needed*/
#endif //_WIN32
static int mlmode = 0; /* Multi line mode. Default is single line. */
static int atexit_registered = 0; /* Register atexit just 1 time. */
static int history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN;
@@ -150,25 +156,25 @@ struct linenoiseState {
};
enum KEY_ACTION{
KEY_NULL = 0, /* NULL */
CTRL_A = 1, /* Ctrl+a */
CTRL_B = 2, /* Ctrl-b */
CTRL_C = 3, /* Ctrl-c */
CTRL_D = 4, /* Ctrl-d */
CTRL_E = 5, /* Ctrl-e */
CTRL_F = 6, /* Ctrl-f */
CTRL_H = 8, /* Ctrl-h */
TAB = 9, /* Tab */
CTRL_K = 11, /* Ctrl+k */
CTRL_L = 12, /* Ctrl+l */
ENTER = 13, /* Enter */
CTRL_N = 14, /* Ctrl-n */
CTRL_P = 16, /* Ctrl-p */
CTRL_T = 20, /* Ctrl-t */
CTRL_U = 21, /* Ctrl+u */
CTRL_W = 23, /* Ctrl+w */
ESC = 27, /* Escape */
BACKSPACE = 127 /* Backspace */
KEY_NULL = 0, /* NULL */
CTRL_A = 1, /* Ctrl+a */
CTRL_B = 2, /* Ctrl-b */
CTRL_C = 3, /* Ctrl-c */
CTRL_D = 4, /* Ctrl-d */
CTRL_E = 5, /* Ctrl-e */
CTRL_F = 6, /* Ctrl-f */
CTRL_H = 8, /* Ctrl-h */
TAB = 9, /* Tab */
CTRL_K = 11, /* Ctrl+k */
CTRL_L = 12, /* Ctrl+l */
ENTER = 13, /* Enter */
CTRL_N = 14, /* Ctrl-n */
CTRL_P = 16, /* Ctrl-p */
CTRL_T = 20, /* Ctrl-t */
CTRL_U = 21, /* Ctrl+u */
CTRL_W = 23, /* Ctrl+w */
ESC = 27, /* Escape */
BACKSPACE = 127 /* Backspace */
};
static void linenoiseAtExit(void);
@@ -207,7 +213,13 @@ static int isUnsupportedTerm(void) {
char *term = getenv("TERM");
int j;
if (term == NULL) return 0;
if (term == NULL) {
#ifdef _WIN32
return 1;
#else
return 0;
#endif // _WIN32
}
for (j = 0; unsupported_term[j]; j++)
if (!strcasecmp(term,unsupported_term[j])) return 1;
return 0;
@@ -215,6 +227,7 @@ static int isUnsupportedTerm(void) {
/* Raw mode: 1960 magic shit. */
static int enableRawMode(int fd) {
#ifndef _WIN32
struct termios raw;
if (!isatty(STDIN_FILENO)) goto fatal;
@@ -247,12 +260,17 @@ static int enableRawMode(int fd) {
fatal:
errno = ENOTTY;
return -1;
#else
return 0;
#endif //_WIN32
}
static void disableRawMode(int fd) {
#ifndef _WIN32
/* Don't even check the return value as it's too late. */
if (rawmode && tcsetattr(fd,TCSAFLUSH,&orig_termios) != -1)
rawmode = 0;
#endif //_WIN32
}
/* Use the ESC [6n escape sequence to query the horizontal cursor position
@@ -283,9 +301,13 @@ static int getCursorPosition(int ifd, int ofd) {
/* Try to get the number of columns in the current terminal, or assume 80
* if it fails. */
static int getColumns(int ifd, int ofd) {
#ifndef _WIN32
struct winsize ws;
if (ioctl(1, TIOCGWINSZ, &ws) == -1 || ws.ws_col == 0) {
#else
if(1) {
#endif //_WIN32
/* ioctl() failed. Try to query the terminal itself. */
int start, cols;
@@ -307,9 +329,12 @@ static int getColumns(int ifd, int ofd) {
}
}
return cols;
} else {
}
#ifndef _WIN32
else {
return ws.ws_col;
}
#endif //_WIN32
failed:
return 80;