00544ad37ece26b2c4f2210ed5e5295241d0db19Teemu Huovila#ifndef FTS_TOKENIZER_COMMON_H

00544ad37ece26b2c4f2210ed5e5295241d0db19Teemu Huovila#define FTS_TOKENIZER_COMMON_H

00544ad37ece26b2c4f2210ed5e5295241d0db19Teemu Huovilafts_tokenizer_delete_trailing_partial_char(const unsigned char *data,

00544ad37ece26b2c4f2210ed5e5295241d0db19Teemu Huovila size_t *len);

507ea0bc5b25efb4c96033a19dec66689a50ebd0Baofeng Wangfts_tokenizer_delete_trailing_invalid_char(const unsigned char *data,

507ea0bc5b25efb4c96033a19dec66689a50ebd0Baofeng Wang size_t *len);

00544ad37ece26b2c4f2210ed5e5295241d0db19Teemu Huovila#endif