#include "lib.h"

#include "sha2.h"

#include "hex-binary.h"

#include "test-common.h"

#include "fts-tokenizer.h"

#include "fts-tokenizer-private.h"

#include <stdlib.h>

static void test_fts_tokenizer_generic_only(void)

{

static const unsigned char input[] =

"hello world\r\nAnd there\twas: text "

"galore, and longlonglongabcdefghijklmnopqrstuvwxyz more.\n\n (\"Hello world\")last ";

static const char *const expected_output[] = {

"hello", "world", "And",

"there", "was", "text", "galore",

"and", "longlonglongabcdefghijklmnopqr",

"more", "Hello", "world", "last", NULL

};

const struct fts_tokenizer *tok_class;

struct fts_tokenizer *tok;

const char * const *eopp = expected_output;

const char *token, *error;

test_begin("fts tokenizer generic simple");

tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);

test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0);

while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {

test_assert(strcmp(token, *eopp) == 0);

eopp++;

}

while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) {

test_assert(strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

test_end();

}

static void test_fts_tokenizer_generic_unicode_whitespace(void)

{

/* with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and

static const unsigned char input[] =

"hello\xEF\xBC\x81world\r\nAnd\xE2\x80\x80there\twas: text "

"galore\xE2\x81\x9F""and\xE2\x81\x9Amore.\n\n";

static const char *const expected_output[] = {

"hello", "world", "And",

"there", "was", "text", "galore",

"and", "more", NULL

};

const struct fts_tokenizer *tok_class;

struct fts_tokenizer *tok;

const char * const *eopp = expected_output;

const char *token, *error;

test_begin("fts tokenizer generic simple with Unicode whitespace");

fts_tokenizer_register(fts_tokenizer_generic);

tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);

test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0);

while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {

test_assert(strcmp(token, *eopp) == 0);

eopp++;

}

while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) {

test_assert(strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unregister(fts_tokenizer_generic);

test_end();

}

static void test_fts_tokenizer_char_generic_only(void)

{

static const unsigned char input[] =

"abc@example.com, "

"Bar Baz <bar@example.org>, "

"foo@domain";

static const char *const expected_output[] = {

"abc", "example", "com", "Bar", "Baz",

"bar", "example", "org", "foo", "domain", NULL

};

struct fts_tokenizer *tok;

const char * const *eopp = expected_output;

const char *token, *error;

unsigned int i;

int ret;

test_begin("fts tokenizer generic simple input one character at a time");

fts_tokenizer_register(fts_tokenizer_generic);

test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0);

for (i = 0; i <= sizeof(input)-1; ) {

ret = i < sizeof(input)-1 ?

fts_tokenizer_next(tok, &input[i], 1, &token) :

fts_tokenizer_next(tok, NULL, 0, &token);

if (ret == 0) {

i++;

continue;

}

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unregister(fts_tokenizer_generic);

test_end();

}

const char *const tr29_settings[] = {"algorithm", "tr29", NULL};

static void test_fts_tokenizer_generic_tr29_only(void)

{

static const unsigned char input[] =

"hello world\r\n\nAnd there\twas: text "

"galore, and more.\n\n (\"Hello world\")3.14 3,14 last"

" longlonglongabcdefghijklmnopqrstuvwxyz 1.";

static const char *const expected_output[] = {

"hello", "world", "And",

"there", "was", "text", "galore",

"and", "more", "Hello", "world", "3.14",

"3,14", "last", "longlonglongabcdefghijklmnopqr", "1", NULL

};

const struct fts_tokenizer *tok_class;

struct fts_tokenizer *tok;

const char * const *eopp = expected_output;

const char *token, *error;

test_begin("fts tokenizer generic TR29");

fts_tokenizer_register(fts_tokenizer_generic);

tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);

test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0);

while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {

test_assert(strcmp(token, *eopp) == 0);

eopp++;

}

while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) {

test_assert(strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unregister(fts_tokenizer_generic);

test_end();

}

static void test_fts_tokenizer_generic_tr29_unicode_whitespace(void)

{

/* with Unicode(utf8) U+2000(e2 80 80) and U+205A(e2 81 9a) and U+205F(e2

static const unsigned char input[] =

"hello world\r\nAnd\xE2\x80\x80there\twas: text "

"galore\xE2\x81\x9F""and\xE2\x81\x9Amore.\n\n";

static const char *const expected_output[] = {

"hello", "world", "And",

"there", "was", "text", "galore",

"and", "more", NULL

};

const struct fts_tokenizer *tok_class;

struct fts_tokenizer *tok;

const char * const *eopp = expected_output;

const char *token, *error;

test_begin("fts tokenizer generic TR29 with Unicode whitespace");

fts_tokenizer_register(fts_tokenizer_generic);

tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);

test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0);

while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {

test_assert(strcmp(token, *eopp) == 0);

eopp++;

}

while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) {

test_assert(strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unregister(fts_tokenizer_generic);

test_end();

}

static void test_fts_tokenizer_generic_tr29_midnumlet_end(void)

{

/* u+FF0E is EF BC 8E */

static const unsigned char input[] =

"hello world\xEF\xBC\x8E";

static const char *const expected_output[] = {

"hello", "world", NULL

};

const struct fts_tokenizer *tok_class;

struct fts_tokenizer *tok;

const char * const *eopp = expected_output;

const char *token, *error;

test_begin("fts tokenizer generic TR29 with MinNumLet U+FF0E at end");

fts_tokenizer_register(fts_tokenizer_generic);

tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);

test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0);

while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) {

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unregister(fts_tokenizer_generic);

test_end();

}

static void test_fts_tokenizer_char_generic_tr29_only(void)

{

static const unsigned char input[] =

"abc@example.com, "

"Bar Baz <bar@example.org>, "

"foo@domain";

static const char *const expected_output[] = {

"abc", "example.com", "Bar", "Baz",

"bar", "example.org", "foo", "domain", NULL

};

struct fts_tokenizer *tok;

const char * const *eopp = expected_output;

const char *token, *error;

unsigned int i;

int ret;

test_begin("fts tokenizer generic TR29 input one character at a time");

fts_tokenizer_register(fts_tokenizer_generic);

test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0);

for (i = 0; i <= sizeof(input)-1; ) {

ret = i < sizeof(input)-1 ?

fts_tokenizer_next(tok, &input[i], 1, &token) :

fts_tokenizer_next(tok, NULL, 0, &token);

if (ret == 0) {

i++;

continue;

}

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unregister(fts_tokenizer_generic);

test_end();

}

static void test_fts_tokenizer_line_address_only(void)

{

static const char *const input[] = {

"abc@example.com",

" Bar Baz <bar@example.org>",

"foo@domain",

" moro foo@domain Bar Baz <bar@example.org>"

};

static const char *const expected_output[] = {

"abc@example.com", "bar@example.org",

"foo@domain", "foo@domain", "bar@example.org", NULL

};

struct fts_tokenizer *tok;

const char * const *eopp = expected_output;

const char *token, *error;

unsigned int i;

int ret;

test_begin("fts tokenizer email address only, input one line at a time");

fts_tokenizer_register(fts_tokenizer_email_address);

test_assert(fts_tokenizer_create(fts_tokenizer_email_address, NULL, NULL, &tok, &error) == 0);

for (i = 0; i <= N_ELEMENTS(input);) {

ret = i < N_ELEMENTS(input) ?

fts_tokenizer_next(tok, (unsigned char *)input[i],

strlen(input[i]), &token) :

fts_tokenizer_next(tok, NULL, 0, &token);

if (ret == 0) {

i++;

continue;

}

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unregister(fts_tokenizer_email_address);

test_end();

}

static void test_fts_tokenizer_char_address_only(void)

{

static const unsigned char input[] =

"@invalid invalid@ abc@example.com, "

"Bar Baz <bar@example.org>, "

"foo@domain";

static const char *const expected_output[] = {

"abc@example.com", "bar@example.org",

"foo@domain", NULL

};

struct fts_tokenizer *tok;

const char * const *eopp = expected_output;

const char *token, *error;

unsigned int i;

int ret;

test_begin("fts tokenizer email address only, input one character at a time");

fts_tokenizer_register(fts_tokenizer_email_address);

test_assert(fts_tokenizer_create(fts_tokenizer_email_address, NULL, NULL, &tok, &error) == 0);

for (i = 0; i <= sizeof(input)-1; ) {

ret = i < sizeof(input)-1 ?

fts_tokenizer_next(tok, &input[i], 1, &token) :

fts_tokenizer_next(tok, NULL, 0, &token);

if (ret == 0) {

i++;

continue;

}

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unregister(fts_tokenizer_email_address);

test_end();

}

static void test_fts_tokenizer_rand_address_only(void)

{

static const unsigned char input[] =

"@invalid invalid@ Abc Dfg <abc.dfg@example.com>, "

"Foo Bar (comment)foo.bar@host.example.org foo ";

static const char *const expected_output[] = {

"abc.dfg@example.com",

"foo.bar@host.example.org",

NULL

};

struct fts_tokenizer *tok;

const char * const *eopp = expected_output;

const char *token, *error;

unsigned int i, step, step_max = 10;

int ret;

test_begin("fts tokenizer email address, input random length");

fts_tokenizer_register(fts_tokenizer_email_address);

test_assert(fts_tokenizer_create(fts_tokenizer_email_address, NULL,

NULL, &tok, &error) == 0);

step = rand() % step_max + 1;

for (i = 0; i <= sizeof(input)-1; ) {

ret = i < sizeof(input)-1 ?

fts_tokenizer_next(tok, &input[i], step, &token) :

fts_tokenizer_next(tok, NULL, 0, &token);

if (ret == 0) {

i += step;

step = rand() % step_max + 1;

step = I_MIN(step, sizeof(input) - i);

continue;

}

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unregister(fts_tokenizer_email_address);

test_end();

}

static void test_fts_tokenizer_address_char(void)

{

static const unsigned char input[] =

"@invalid invalid@ abc@example.com, "

"Bar Baz <bar@example.org>, "

"foo@domain";

static const char *const expected_output[] = {

"invalid", "invalid", "abc", "example", "com", "abc@example.com", "Bar", "Baz",

"bar", "example", "org", "bar@example.org",

"foo", "domain", "foo@domain", NULL

};

struct fts_tokenizer *tok, *gen_tok;

const char * const *eopp = expected_output;

const char *token, *error;

unsigned int i;

int ret;

test_begin("fts tokenizer email address + parent, input one character at a time");

test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0);

test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, NULL, &tok, &error) == 0);

for (i = 0; i <= sizeof(input)-1; ) {

ret = i < sizeof(input)-1 ?

fts_tokenizer_next(tok, &input[i], 1, &token) :

fts_tokenizer_next(tok, NULL, 0, &token);

if (ret == 0) {

i++;

continue;

}

test_assert(*eopp != NULL);

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unref(&gen_tok);

test_end();

}

static void test_fts_tokenizer_address_line(void)

{

static const char *const input[] = {

"@invalid invalid@ abc@example.com, ",

"Bar Baz <bar@example.org>, ",

"foo@domain, ",

"foo@domain Bar Baz <bar@example.org>, "

};

static const char *const expected_output[] = {

"invalid", "invalid", "abc", "example", "com", "abc@example.com", "Bar", "Baz",

"bar", "example", "org", "bar@example.org",

"foo", "domain", "foo@domain",

"foo", "domain", "foo@domain", "Bar", "Baz",

"bar", "example", "org", "bar@example.org", NULL

};

struct fts_tokenizer *tok, *gen_tok;

const char * const *eopp = expected_output;

const char *token, *error;

unsigned int i;

int ret;

test_begin("fts tokenizer email address + parent, input one line at a time");

test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0);

test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, NULL, &tok, &error) == 0);

for (i = 0; i <= N_ELEMENTS(input);) {

ret = i < N_ELEMENTS(input) ?

fts_tokenizer_next(tok, (unsigned char *)input[i],

strlen(input[i]), &token) :

fts_tokenizer_next(tok, NULL, 0, &token);

if (ret == 0) {

i++;

continue;

}

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unref(&gen_tok);

test_end();

}

static void test_fts_tokenizer_address_rand(void)

{

static const unsigned char input[] =

"@invalid invalid@ abc@example.com, "

"Bar Baz <bar@example.org>, "

"foo@domain";

static const char *const expected_output[] = {

"invalid", "invalid", "abc", "example", "com", "abc@example.com", "Bar", "Baz",

"bar", "example", "org", "bar@example.org",

"foo", "domain", "foo@domain", NULL

};

struct fts_tokenizer *tok, *gen_tok;

const char * const *eopp = expected_output;

const char *token, *error;

unsigned int i, step, step_max = 10;

int ret;

test_begin("fts tokenizer email address + parent, input random length");

fts_tokenizer_register(fts_tokenizer_generic);

fts_tokenizer_register(fts_tokenizer_email_address);

test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0);

test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, NULL, &tok, &error) == 0);

//srand(1424142100); /* had a bug */

step = rand() % step_max + 1;

for (i = 0; i <= sizeof(input)-1; ) {

ret = i < sizeof(input)-1 ?

fts_tokenizer_next(tok, &input[i], step, &token) :

fts_tokenizer_next(tok, NULL, 0, &token);

if (ret == 0) {

i += step;

step = rand() % step_max + 1;

step = I_MIN(step, sizeof(input) - i);

continue;

}

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

fts_tokenizer_unref(&tok);

fts_tokenizer_unref(&gen_tok);

fts_tokenizer_unregister(fts_tokenizer_generic);

fts_tokenizer_unregister(fts_tokenizer_email_address);

test_end();

}

static void test_fts_tokenizer_address_search(void)

{

static const unsigned char input[] =

"@invalid invalid@ abc@example.com, "

"Bar Baz <bar@example.org>, "

"foo@domain";

static const char *const expected_output[] = {

"invalid", "invalid", "abc@example.com", "Bar", "Baz",

"bar@example.org", "foo@domain", NULL

};

static const char *const settings[] = { "search", "" };

struct fts_tokenizer *tok, *gen_tok;

const char * const *eopp = expected_output;

const char *token, *error;

unsigned int i;

int ret;

test_begin("fts tokenizer search email address + parent, input one character at a time");

test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0);

test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, settings, &tok, &error) == 0);

for (i = 0; i <= sizeof(input)-1; ) {

ret = i < sizeof(input)-1 ?

fts_tokenizer_next(tok, &input[i], 1, &token) :

fts_tokenizer_next(tok, NULL, 0, &token);

if (ret == 0) {

i++;

continue;

}

test_assert(*eopp != NULL);

test_assert(null_strcmp(token, *eopp) == 0);

eopp++;

}

test_assert(*eopp == NULL);

/* make sure state is forgotten at EOF */

test_assert(fts_tokenizer_next(tok, (const void *)"foo", 3, &token) == 0);

test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0 &&

strcmp(token, "foo") == 0);

test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0);

test_assert(fts_tokenizer_next(tok, (const void *)"bar@baz", 7, &token) == 0);

test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0 &&

strcmp(token, "bar@baz") == 0);

test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0);

test_assert(fts_tokenizer_next(tok, (const void *)"foo@", 4, &token) == 0);

test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0 &&

strcmp(token, "foo") == 0);

test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0);

/* test reset explicitly */

test_assert(fts_tokenizer_next(tok, (const void *)"a", 1, &token) == 0);

fts_tokenizer_reset(tok);

test_assert(fts_tokenizer_next(tok, (const void *)"b@c", 3, &token) == 0);

test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0 &&

strcmp(token, "b@c") == 0);

test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0);

fts_tokenizer_unref(&tok);

fts_tokenizer_unref(&gen_tok);

test_end();

}

int main(void)

{

static void (*test_functions[])(void) = {

test_fts_tokenizer_generic_only,

test_fts_tokenizer_generic_unicode_whitespace,

test_fts_tokenizer_char_generic_only,

test_fts_tokenizer_generic_tr29_only,

test_fts_tokenizer_generic_tr29_unicode_whitespace,

test_fts_tokenizer_char_generic_tr29_only,

test_fts_tokenizer_generic_tr29_midnumlet_end,

test_fts_tokenizer_char_address_only,

test_fts_tokenizer_line_address_only,

test_fts_tokenizer_rand_address_only,

test_fts_tokenizer_address_char,

test_fts_tokenizer_address_line,

test_fts_tokenizer_address_rand,

test_fts_tokenizer_address_search,

NULL

};

int ret;

fts_tokenizers_init();

ret = test_run(test_functions);

fts_tokenizers_deinit();

return ret;

}