tommy/libpostal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[tokenization] Adding url, email, US/international phone numbers, a separate type for ideographic numbers, more general quotes, paren types

Browse Source
This commit is contained in:
Al
2015-03-24 16:43:53 -04:00
parent 50187f28ce
commit 2d1c24a6e9
4 changed files with 298633 additions and 262454 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/token_types.h
View File

@@ -7,23 +7,26 @@ extern "C" {
// Doing these as #defines so we can duplicate the values exactly in Python
#define END 0 // Null byte
#define END 0 // Null byte
// Word types
#define WORD 1 // Any letter-only word (includes all unicode letters)
#define ABBREVIATION 2 // Loose abbreviations (ending in ".")
#define IDEOGRAM 3 // For languages that don't separate on whitespace (e.g. Chinese, Japanese, Korean), separate by character
#define PHRASE 4 // Not part of the first stage tokenizer, but may be used after phrase parsing
#define WORD 1 // Any letter-only word (includes all unicode letters)
#define ABBREVIATION 2 // Loose abbreviations (ending in ".")
#define IDEOGRAPHIC_CHAR 3 // For languages that don't separate on whitespace (e.g. Chinese, Japanese, Korean), separate by character
#define PHRASE 4 // Not part of the first stage tokenizer, but may be used after phrase parsing
// Special tokens
#define EMAIL 20 // Make sure emails are tokenized altogether
#define URL 21 // Make sure urls are tokenized altogether
#define US_PHONE 22 // US phone number (with or without country code)
#define INTL_PHONE 23 // A non-US phone number (must have country code)
// Numbers and numeric types
#define NUMBER 50 // All digits
#define NUMERIC 51 // Any sequence containing a digit
#define ORDINAL 52 // 1st, 2nd, etc.
#define NUMERIC_RANGE 53 // 2-3, Queens addresses, US ZIP+4 codes
#define ORDINAL_RANGE 54 // 1-2nd, 1st-2nd
#define ROMAN_NUMERAL 55 // II, III, VI, etc.
#define US_PHONE 56 // US phone number (with or without country code)
#define INTL_PHONE 57 // A non-US phone number (must have country code)
#define NUMERIC 50 // Any sequence containing a digit
#define ORDINAL 51 // 1st, 2nd, 1er, 1 etc.
#define ROMAN_NUMERAL 52 // II, III, VI, etc.
#define IDEOGRAPHIC_NUMBER 53 // All numeric ideographic characters, includes e.g. Han numbers and chars like "²"
// Punctuation types, may separate a phrase
#define PERIOD 100
@@ -40,16 +43,12 @@ extern "C" {
#define DASH 111
#define BREAKING_DASH 112
#define HYPHEN 113
#define LPAREN 114
#define RPAREN 115
#define LBSQUARE 116
#define RBSQUARE 117
#define DOUBLE_QUOTE 118
#define SINGLE_QUOTE 119
#define LEFT_DOUBLE_QUOTE 120
#define RIGHT_DOUBLE_QUOTE 121
#define LEFT_SINGLE_QUOTE 122
#define RIGHT_SINGLE_QUOTE 123
#define PUNCT_OPEN 114
#define PUNCT_CLOSE 115
#define DOUBLE_QUOTE 119
#define SINGLE_QUOTE 120
#define OPEN_QUOTE 121
#define CLOSE_QUOTE 122
#define SLASH 124
#define BACKSLASH 125
#define GREATER_THAN 126
@@ -58,8 +57,7 @@ extern "C" {
// Non-letters and whitespace
#define OTHER 200
#define WHITESPACE 300
#define NEWLINE 301
#define NEWLINE 301
#ifdef __cplusplus
}