c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#ifndef FTS_TOKENIZER_PRIVATE_H
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#define FTS_TOKENIZER_PRIVATE_H
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#include "fts-tokenizer.h"
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#define FTS_TOKENIZER_CLASSES_NR 2
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenstruct fts_tokenizer_vfuncs {
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen int (*create)(const char *const *settings,
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen struct fts_tokenizer **tokenizer_r, const char **error_r);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen void (*destroy)(struct fts_tokenizer *tok);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
2730605833442b5ddcb261f90b8375fc98201e35Timo Sirainen void (*reset)(struct fts_tokenizer *tok);
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila int (*next)(struct fts_tokenizer *tok, const unsigned char *data,
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen size_t size, size_t *skip_r, const char **token_r,
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen const char **error_r);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen};
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenenum fts_tokenizer_parent_state {
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen FTS_TOKENIZER_PARENT_STATE_ADD_DATA = 0,
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen FTS_TOKENIZER_PARENT_STATE_NEXT_OUTPUT,
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen FTS_TOKENIZER_PARENT_STATE_FINALIZE
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen};
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenstruct fts_tokenizer {
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen const char *name;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen const struct fts_tokenizer_vfuncs *v;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen int refcount;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen struct fts_tokenizer *parent;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen buffer_t *parent_input;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen enum fts_tokenizer_parent_state parent_state;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen const unsigned char *prev_data;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen size_t prev_size;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen size_t prev_skip;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen bool prev_reply_finished;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen bool skip_parents; /* Return token as is, do not hand to parents. */
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen};
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenvoid fts_tokenizer_register(const struct fts_tokenizer *tok_class);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenvoid fts_tokenizer_unregister(const struct fts_tokenizer *tok_class);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#endif