51 lines
1.3 KiB
C
51 lines
1.3 KiB
C
#ifndef NORMALIZE_H
|
|
#define NORMALIZE_H
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
#include <stdlib.h>
|
|
#include <stdio.h>
|
|
#include <stdbool.h>
|
|
|
|
#include "constants.h"
|
|
#include "klib/khash.h"
|
|
#include "numex.h"
|
|
#include "scanner.h"
|
|
#include "string_utils.h"
|
|
#include "utf8proc/utf8proc.h"
|
|
#include "unicode_scripts.h"
|
|
#include "transliterate.h"
|
|
#include "trie.h"
|
|
#include "tokens.h"
|
|
#include "vector.h"
|
|
|
|
#define NORMALIZE_STRING_LATIN_ASCII 1 << 0
|
|
#define NORMALIZE_STRING_TRANSLITERATE 1 << 1
|
|
#define NORMALIZE_STRING_STRIP_ACCENTS 1 << 2
|
|
#define NORMALIZE_STRING_DECOMPOSE 1 << 3
|
|
#define NORMALIZE_STRING_LOWERCASE 1 << 4
|
|
|
|
#define NORMALIZE_TOKEN_REPLACE_HYPHENS 1 << 0
|
|
#define NORMALIZE_TOKEN_DELETE_HYPHENS 1 << 1
|
|
#define NORMALIZE_TOKEN_DELETE_FINAL_PERIOD 1 << 2
|
|
#define NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS 1 << 3
|
|
#define NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES 1 << 4
|
|
#define NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE 1 << 5
|
|
|
|
char *normalize_string_utf8(char *str, uint64_t options);
|
|
|
|
char *normalize_string_latin(char *str, size_t len, uint64_t options);
|
|
|
|
// Takes NORMALIZE_TOKEN_* options
|
|
void add_token_alternatives(string_tree_t *tree, char *str, token_t token, uint64_t options);
|
|
|
|
// Takes NORMALIZE_STRING_* options
|
|
string_tree_t *normalize_string(char *str, uint64_t options);
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif |