src/lib-fts/fts-tokenizer.c

bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2014-2018 Dovecot authors, see the included COPYING file */
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#include "lib.h"
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#include "array.h"
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#include "istream.h"
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#include "str.h"
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#include "strfuncs.h"
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#include "fts-tokenizer.h"
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen#include "fts-tokenizer-private.h"
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
a46abae27d4b00f77e5932ed8f7595c054fe0280Timo Sirainenstatic ARRAY(const struct fts_tokenizer *) fts_tokenizer_classes;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovilavoid fts_tokenizers_init(void)
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila{
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila    if (!array_is_created(&fts_tokenizer_classes)) {
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila        fts_tokenizer_register(fts_tokenizer_generic);
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila        fts_tokenizer_register(fts_tokenizer_email_address);
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila    }
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila}
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovilavoid fts_tokenizers_deinit(void)
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila{
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila    if (array_is_created(&fts_tokenizer_classes))
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila        array_free(&fts_tokenizer_classes);
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila}
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen/* private */
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenvoid fts_tokenizer_register(const struct fts_tokenizer *tok_class)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen{
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    if (!array_is_created(&fts_tokenizer_classes))
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        i_array_init(&fts_tokenizer_classes, FTS_TOKENIZER_CLASSES_NR);
a46abae27d4b00f77e5932ed8f7595c054fe0280Timo Sirainen    array_append(&fts_tokenizer_classes, &tok_class, 1);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen}
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen/* private */
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenvoid fts_tokenizer_unregister(const struct fts_tokenizer *tok_class)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen{
a46abae27d4b00f77e5932ed8f7595c054fe0280Timo Sirainen    const struct fts_tokenizer *const *tp;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    unsigned int idx;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    array_foreach(&fts_tokenizer_classes, tp) {
a46abae27d4b00f77e5932ed8f7595c054fe0280Timo Sirainen        if (strcmp((*tp)->name, tok_class->name) == 0) {
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen            idx = array_foreach_idx(&fts_tokenizer_classes, tp);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen            array_delete(&fts_tokenizer_classes, idx, 1);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen            if (array_count(&fts_tokenizer_classes) == 0)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen                array_free(&fts_tokenizer_classes);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen            return;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        }
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    }
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    i_unreached();
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen}
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenconst struct fts_tokenizer *fts_tokenizer_find(const char *name)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen{
a46abae27d4b00f77e5932ed8f7595c054fe0280Timo Sirainen    const struct fts_tokenizer *const *tp;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    array_foreach(&fts_tokenizer_classes, tp) {
a46abae27d4b00f77e5932ed8f7595c054fe0280Timo Sirainen        if (strcmp((*tp)->name, name) == 0)
a46abae27d4b00f77e5932ed8f7595c054fe0280Timo Sirainen            return *tp;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    }
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    return NULL;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen}
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovilaconst char *fts_tokenizer_name(const struct fts_tokenizer *tok)
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila{
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila    return tok->name;
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila}
4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainenstatic void fts_tokenizer_self_reset(struct fts_tokenizer *tok)
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainen{
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainen    tok->prev_data = NULL;
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainen    tok->prev_size = 0;
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainen    tok->prev_skip = 0;
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainen    tok->prev_reply_finished = TRUE;
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainen}
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenint fts_tokenizer_create(const struct fts_tokenizer *tok_class,
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen             struct fts_tokenizer *parent,
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen             const char *const *settings,
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen             struct fts_tokenizer **tokenizer_r,
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen             const char **error_r)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen{
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    struct fts_tokenizer *tok;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    const char *empty_settings = NULL;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    i_assert(settings == NULL || str_array_length(settings) % 2 == 0);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    if (settings == NULL)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        settings = &empty_settings;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    if (tok_class->v->create(settings, &tok, error_r) < 0) {
085a87d32bef6e3d565b4f9f157eccc96929a193Phil Carmody        *tokenizer_r = NULL;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        return -1;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    }
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    tok->refcount = 1;
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainen    fts_tokenizer_self_reset(tok);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    if (parent != NULL) {
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        fts_tokenizer_ref(parent);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        tok->parent = parent;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        tok->parent_input = buffer_create_dynamic(default_pool, 128);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    }
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    *tokenizer_r = tok;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    return 0;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen}
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenvoid fts_tokenizer_ref(struct fts_tokenizer *tok)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen{
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    i_assert(tok->refcount > 0);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    tok->refcount++;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen}
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenvoid fts_tokenizer_unref(struct fts_tokenizer **_tok)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen{
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    struct fts_tokenizer *tok = *_tok;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    i_assert(tok->refcount > 0);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    *_tok = NULL;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    if (--tok->refcount > 0)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        return;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
6307d76096764e66bddc63d4a3e5a1aa19cc528fJosef 'Jeff' Sipek    buffer_free(&tok->parent_input);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    if (tok->parent != NULL)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        fts_tokenizer_unref(&tok->parent);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    tok->v->destroy(tok);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen}
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovilastatic int
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainenfts_tokenizer_next_self(struct fts_tokenizer *tok,
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila                        const unsigned char *data, size_t size,
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen                        const char **token_r, const char **error_r)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen{
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila    int ret = 0;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    size_t skip = 0;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    i_assert(tok->prev_reply_finished ||
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen         (data == tok->prev_data && size == tok->prev_size));
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    if (tok->prev_reply_finished) {
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        /* whole new data */
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen        ret = tok->v->next(tok, data, size, &skip, token_r, error_r);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    } else {
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        /* continuing previous data */
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        i_assert(tok->prev_skip <= size);
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila        ret = tok->v->next(tok, data + tok->prev_skip,
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen                   size - tok->prev_skip, &skip,
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen                   token_r, error_r);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    }
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila    if (ret > 0) {
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        i_assert(skip <= size - tok->prev_skip);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        tok->prev_data = data;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        tok->prev_size = size;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        tok->prev_skip = tok->prev_skip + skip;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        tok->prev_reply_finished = FALSE;
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila    } else if (ret == 0) {
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        /* we need a new data block */
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainen        fts_tokenizer_self_reset(tok);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    }
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila    return ret;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen}
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
2730605833442b5ddcb261f90b8375fc98201e35Timo Sirainenvoid fts_tokenizer_reset(struct fts_tokenizer *tok)
2730605833442b5ddcb261f90b8375fc98201e35Timo Sirainen{
2730605833442b5ddcb261f90b8375fc98201e35Timo Sirainen    tok->v->reset(tok);
12952c18d10fa83be65059471139c2fdc8a00c3dTimo Sirainen    fts_tokenizer_self_reset(tok);
2730605833442b5ddcb261f90b8375fc98201e35Timo Sirainen}
2730605833442b5ddcb261f90b8375fc98201e35Timo Sirainen
2bb1ef0b669901fb91ff961e7fb074439ef769abTimo Sirainenint fts_tokenizer_next(struct fts_tokenizer *tok,
2bb1ef0b669901fb91ff961e7fb074439ef769abTimo Sirainen               const unsigned char *data, size_t size,
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen               const char **token_r, const char **error_r)
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen{
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila    int ret;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    switch (tok->parent_state) {
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    case FTS_TOKENIZER_PARENT_STATE_ADD_DATA:
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen        ret = fts_tokenizer_next_self(tok, data, size, token_r, error_r);
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila        if (ret <= 0 || tok->parent == NULL || tok->skip_parents)
9bbc62421e4000b04563eb5f93272fc51b893fb2Timo Sirainen            break;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        buffer_set_used_size(tok->parent_input, 0);
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila        buffer_append(tok->parent_input, *token_r, strlen(*token_r));
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        tok->parent_state++;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        /* fall through */
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    case FTS_TOKENIZER_PARENT_STATE_NEXT_OUTPUT:
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila        ret = fts_tokenizer_next(tok->parent, tok->parent_input->data,
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen                                 tok->parent_input->used, token_r, error_r);
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila        if (ret != 0)
9bbc62421e4000b04563eb5f93272fc51b893fb2Timo Sirainen            break;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        tok->parent_state++;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        /* fall through */
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    case FTS_TOKENIZER_PARENT_STATE_FINALIZE:
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen        ret = fts_tokenizer_next(tok->parent, NULL, 0, token_r, error_r);
3dc5a231160859c9627157dc53a94d5e4494fe9fTeemu Huovila        if (ret != 0)
9bbc62421e4000b04563eb5f93272fc51b893fb2Timo Sirainen            break;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        /* we're finished sending this token to parent tokenizer.
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen           see if our own tokenizer has more tokens available */
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        tok->parent_state = FTS_TOKENIZER_PARENT_STATE_ADD_DATA;
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen        return fts_tokenizer_next(tok, data, size, token_r, error_r);
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    default:
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen        i_unreached();
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen    }
9bbc62421e4000b04563eb5f93272fc51b893fb2Timo Sirainen    /* we must not be returning empty tokens */
9bbc62421e4000b04563eb5f93272fc51b893fb2Timo Sirainen    i_assert(ret <= 0 || (*token_r)[0] != '\0');
9bbc62421e4000b04563eb5f93272fc51b893fb2Timo Sirainen    return ret;
c865b0e9c65fd77f7b2ab6f8616d3def5501ecb3Timo Sirainen}
16dd1fd16f0c6dbd4a057327370b432684e301ecTimo Sirainen
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainenint fts_tokenizer_final(struct fts_tokenizer *tok, const char **token_r,
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen            const char **error_r)
16dd1fd16f0c6dbd4a057327370b432684e301ecTimo Sirainen{
8b1a9a4d63b0abccdf7cb1acb8359d5396dd657bTimo Sirainen    return fts_tokenizer_next(tok, NULL, 0, token_r, error_r);
16dd1fd16f0c6dbd4a057327370b432684e301ecTimo Sirainen}