src/lib-fts/fts-tokenizer-generic-private.h

#ifndef FTS_TOKENIZER_GENERIC_PRIVATE_H
#define FTS_TOKENIZER_GENERIC_PRIVATE_H

extern const struct fts_tokenizer_vfuncs generic_tokenizer_vfuncs_simple;
extern const struct fts_tokenizer_vfuncs generic_tokenizer_vfuncs_tr29;

/* Word boundary letter type */
enum letter_type {
    LETTER_TYPE_NONE = 0,
    LETTER_TYPE_CR,
    LETTER_TYPE_LF,
    LETTER_TYPE_NEWLINE,
    LETTER_TYPE_EXTEND,
    LETTER_TYPE_REGIONAL_INDICATOR,
    LETTER_TYPE_FORMAT,
    LETTER_TYPE_KATAKANA,
    LETTER_TYPE_HEBREW_LETTER,
    LETTER_TYPE_ALETTER,
    LETTER_TYPE_SINGLE_QUOTE,
    LETTER_TYPE_DOUBLE_QUOTE,
    LETTER_TYPE_MIDNUMLET,
    LETTER_TYPE_MIDLETTER,
    LETTER_TYPE_MIDNUM,
    LETTER_TYPE_NUMERIC,
    LETTER_TYPE_EXTENDNUMLET,
    LETTER_TYPE_SOT,
    LETTER_TYPE_EOT,
    LETTER_TYPE_APOSTROPHE, /* Own modification to TR29 */
    LETTER_TYPE_OTHER /* WB14 "any" */
};

enum boundary_algorithm {
    BOUNDARY_ALGORITHM_NONE = 0,
    BOUNDARY_ALGORITHM_SIMPLE,
#define ALGORITHM_SIMPLE_NAME "simple"
    BOUNDARY_ALGORITHM_TR29
#define ALGORITHM_TR29_NAME "tr29"
};

struct generic_fts_tokenizer {
    struct fts_tokenizer tokenizer;
    unsigned int max_length;
    bool wb5a; /* TR29 rule for prefix separation
                  in e.g. French or Italian. */
    bool seen_wb5a;
    unichar_t prev_letter_c;
    unichar_t letter_c;
    enum boundary_algorithm algorithm;
    enum letter_type prev_letter;
    enum letter_type prev_prev_letter;
    size_t untruncated_length;
    buffer_t *token;
};

#endif