test-fts-filter.c revision bf698b98d3a3a1eced66cc682c449f23bf2b67d0
bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi#include "lib.h"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi#include "sha2.h"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi#include "str.h"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi#include "unichar.h"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi#include "test-common.h"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi#include "fts-language.h"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi#include "fts-filter.h"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi#include <stdio.h>
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic const char *const stopword_settings[] = {"stopwords_dir", TEST_STOPWORDS_DIR, NULL};
4fbe0d10901a80b27aacc9d9e6848e30e5fe727dAki Tuomistatic struct fts_language english_language = { .name = "en" };
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_find(void)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filter find");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_find("stopwords") == fts_filter_stopwords);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_find("snowball") == fts_filter_stemmer_snowball);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_find("normalizer-icu") == fts_filter_normalizer_icu);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_find("lowercase") == fts_filter_lowercase);
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_end();
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi}
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomistatic void test_fts_filter_lowercase(void)
f207fb0f2fd3113aa9a9f7911fb12d94dce19dffAki Tuomi{
f207fb0f2fd3113aa9a9f7911fb12d94dce19dffAki Tuomi struct {
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi const char *input;
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi const char *output;
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi } tests[] = {
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi { "foo", "foo" },
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi { "FOO", "foo" },
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi { "fOo", "foo" }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi };
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *filter;
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi const char *error;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *token;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi unsigned int i;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filter lowercase");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_lowercase, NULL, &english_language, NULL, &filter, &error) == 0);
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi for (i = 0; i < N_ELEMENTS(tests); i++) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi token = tests[i].input;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert_idx(fts_filter_filter(filter, &token, &error) > 0 &&
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi strcmp(token, tests[i].output) == 0, 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi fts_filter_unref(&filter);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_stopwords_eng(void)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *filter;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *error;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi int ret;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *input[] = {"an", "elephant", "and", "a", "bear",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "drive", "by", "for", "no", "reason",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "they", "will", "not", "sing", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *output[] = {NULL, "elephant", NULL, NULL, "bear",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "drive", NULL, NULL, NULL, "reason",
17541ea25593c656060199715051db2c1eef221dAki Tuomi NULL, NULL, NULL, "sing"};
17541ea25593c656060199715051db2c1eef221dAki Tuomi const char **ip, **op;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *token;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filter stopwords, English");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_stopwords, NULL, &english_language, stopword_settings, &filter, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi ip = input;
17541ea25593c656060199715051db2c1eef221dAki Tuomi op = output;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi while (*ip != NULL) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi token = *ip;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi ret = fts_filter_filter(filter, &token, &error);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi if (ret <= 0) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(ret == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(*op == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi } else {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(*op != NULL);
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi test_assert(strcmp(*ip, token) == 0);
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi op++;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi ip++;
7c9ae3d919ba59af5be3193a80ece4871a0d700cAki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi fts_filter_unref(&filter);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(filter == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_stopwords_fin(void)
be5773cb4d6edae8a5d9f300c3c7375cdd33826eJosef 'Jeff' Sipek{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const struct fts_language finnish = { .name = "fi" };
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *filter;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *error;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi int ret;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *input[] = {"olla", "vaiko", "eik\xC3\xB6", "olla",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "kenest\xC3\xA4", "ja", "joista", "jonka",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "testi", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *output[] = {NULL, "vaiko", "eik\xC3\xB6", NULL, NULL,
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi NULL, NULL, NULL, "testi"};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *input2[] =
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi {"kuka", "kenet", "keneen", "testi", "eiv\xC3\xA4t", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *output2[] = {NULL, NULL, NULL, "testi", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char **ip, **op;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *token;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filter stopwords, Finnish");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_stopwords, NULL, &finnish, stopword_settings, &filter, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek ip = input;
5f1d689131a75c39f064cbd4202373e7edf78f18Josef 'Jeff' Sipek op = output;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi while (*ip != NULL) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi token = *ip;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi ret = fts_filter_filter(filter, &token, &error);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi if (ret <= 0) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(ret == 0);
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi test_assert(*op == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi } else {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(*op != NULL);
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi test_assert(strcmp(*ip, token) == 0);
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi op++;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi ip++;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
be5773cb4d6edae8a5d9f300c3c7375cdd33826eJosef 'Jeff' Sipek fts_filter_unref(&filter);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(filter == NULL);
5f1d689131a75c39f064cbd4202373e7edf78f18Josef 'Jeff' Sipek
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_stopwords, NULL, &finnish, stopword_settings, &filter, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi ip = input2;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi op = output2;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi while (*ip != NULL) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi token = *ip;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi ret = fts_filter_filter(filter, &token, &error);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi if (ret <= 0) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(ret == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(*op == NULL);
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi } else {
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi test_assert(*op != NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(strcmp(*ip, token) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi op++;
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi ip++;
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi }
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi fts_filter_unref(&filter);
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_assert(filter == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomistatic void test_fts_filter_stopwords_fra(void)
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi{
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi const struct fts_language french = { .name = "fr" };
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi struct fts_filter *filter;
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi const char *error;
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi int ret;
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi const char *input[] = {"e\xC3\xBBt", "soyez", "soi", "peut", "que",
0d1b8b6bec79746c5d89d57dd8c1688946bd9237Josef 'Jeff' Sipek "quelconque", "\xC3\xA9t\xC3\xA9",
5f1d689131a75c39f064cbd4202373e7edf78f18Josef 'Jeff' Sipek "l\xE2\x80\x99""av\xC3\xA8nement",
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi NULL};
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi const char *output[] = {NULL, NULL, NULL, "peut", NULL,
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi "quelconque", NULL,
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi "l\xE2\x80\x99""av\xC3\xA8nement",};
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi const char **ip, **op;
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi const char *token;
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi test_begin("fts filter stopwords, French");
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi test_assert(fts_filter_create(fts_filter_stopwords, NULL, &french, stopword_settings, &filter, &error) == 0);
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi ip = input;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi op = output;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi while (*ip != NULL) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi token = *ip;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi ret = fts_filter_filter(filter, &token, &error);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi if (ret <= 0) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(ret == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(*op == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi } else {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(*op != NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(strcmp(*ip, token) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi op++;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi ip++;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi fts_filter_unref(&filter);
23a7a07d4d7cab538509ceef21a280dcc243362fStephan Bosch test_assert(filter == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_stopwords_fail_lazy_init(void)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const struct fts_language unknown = { .name = "bebobidoop" };
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *filter = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *error = NULL, *token = "foobar";
23a7a07d4d7cab538509ceef21a280dcc243362fStephan Bosch
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filter stopwords, fail filter() (lazy init)");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_stopwords, NULL, &unknown, stopword_settings, &filter, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(filter != NULL && error == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_filter(filter, &token, &error) < 0 && error != NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi#ifdef HAVE_FTS_STEMMER
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_stemmer_snowball_stem_english(void)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *stemmer;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *error;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *token = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const tokens[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "dries" ,"friendlies", "All", "human", "beings", "are",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "born", "free", "and", "equal", "in", "dignity", "and",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "rights", "They", "are", "endowed", "with", "reason", "and",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "conscience", "and", "should", "act", "towards", "one",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "another", "in", "a", "spirit", "of", "brotherhood", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const bases[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "dri" ,"friend", "All", "human", "be", "are", "born", "free",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "and", "equal", "in", "digniti", "and", "right", "They", "are",
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi "endow", "with", "reason", "and", "conscienc", "and", "should",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "act", "toward", "one", "anoth", "in", "a", "spirit", "of",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "brotherhood", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const *tpp;
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi const char * const *bpp;
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi test_begin("fts filter stem English");
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi test_assert(fts_filter_create(fts_filter_stemmer_snowball, NULL, &english_language, NULL, &stemmer, &error) == 0);
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi bpp = bases;
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi for (tpp=tokens; *tpp != NULL; tpp++) {
66761ea6a8f00006f28743b4bd2f5339f78c10e0Timo Sirainen token = *tpp;
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi test_assert(fts_filter_filter(stemmer, &token, &error) > 0);
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi test_assert(token != NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(null_strcmp(token, *bpp) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi bpp++;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi fts_filter_unref(&stemmer);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(stemmer == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_stemmer_snowball_stem_french(void)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *stemmer;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *error;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_language language = { .name = "fr" };
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *token = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const tokens[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "Tous", "les", "\xC3\xAAtres", "humains", "naissent",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "libres", "et", "\xC3\xA9gaux", "en", "dignit\xC3\xA9",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "et", "en", "droits", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const bases[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "Tous" ,"le", "\xC3\xAAtre", "humain", "naissent", "libr", "et",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "\xC3\xA9gal", "en", "dignit", "et", "en", "droit", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const *tpp;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const *bpp;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
e6ba63471b1451b48e1185492c33d7bf58be884dAki Tuomi test_begin("fts filter stem French");
e6ba63471b1451b48e1185492c33d7bf58be884dAki Tuomi test_assert(fts_filter_create(fts_filter_stemmer_snowball, NULL, &language, NULL, &stemmer, &error) == 0);
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch bpp = bases;
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch for (tpp=tokens; *tpp != NULL; tpp++) {
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch token = *tpp;
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch test_assert(fts_filter_filter(stemmer, &token, &error) > 0);
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch test_assert(token != NULL);
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch test_assert(null_strcmp(token, *bpp) == 0);
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch bpp++;
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch }
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi fts_filter_unref(&stemmer);
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch test_assert(stemmer == NULL);
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch test_end();
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi}
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_stopwords_stemmer_eng(void)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi int ret;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *stemmer;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *filter;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *error;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *token = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const tokens[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "dries" ,"friendlies", "All", "human", "beings", "are",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "born", "free", "and", "equal", "in", "dignity", "and",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "rights", "They", "are", "endowed", "with", "reason", "and",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "conscience", "and", "should", "act", "towards", "one",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "another", "in", "a", "spirit", "of", "brotherhood", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const bases[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "dri" ,"friend", "All", "human", "be", NULL, "born", "free",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi NULL, "equal", NULL, "digniti", NULL, "right", "They", NULL,
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "endow", NULL, "reason", NULL, "conscienc", NULL, "should",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "act", "toward", "one", "anoth", NULL, NULL, "spirit", NULL,
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "brotherhood", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const *tpp;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const *bpp;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filters stopwords and stemming chained, English");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi test_assert(fts_filter_create(fts_filter_stopwords, NULL, &english_language, stopword_settings, &filter, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_stemmer_snowball, filter, &english_language, NULL, &stemmer, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi bpp = bases;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi for (tpp=tokens; *tpp != NULL; tpp++) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi token = *tpp;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi ret = fts_filter_filter(stemmer, &token, &error);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(ret >= 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi if (ret == 0)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(*bpp == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi else {
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi test_assert(*bpp != NULL);
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi test_assert(null_strcmp(*bpp, token) == 0);
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi }
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi bpp++;
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi }
66761ea6a8f00006f28743b4bd2f5339f78c10e0Timo Sirainen fts_filter_unref(&stemmer);
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi fts_filter_unref(&filter);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(stemmer == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(filter == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
b99e1062d1fb964a3545f2b5ef87588e09185ef8Stephan Bosch#endif
b99e1062d1fb964a3545f2b5ef87588e09185ef8Stephan Bosch
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi#ifdef HAVE_LIBICU
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_normalizer_swedish_short(void)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *norm = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *input[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "Vem",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "Å",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "ÅÄÖ",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "Vem kan segla f\xC3\xB6rutan vind?\n"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "\xC3\x85\xC3\x84\xC3\x96\xC3\xB6\xC3\xA4\xC3\xA5"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi };
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *expected_output[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "vem",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "a",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "aao",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "vem kan segla forutan vind?\naaooaa"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi };
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const settings[] =
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC", NULL};
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch const char *error = NULL;
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch const char *token = NULL;
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch unsigned int i;
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch test_begin("fts filter normalizer Swedish short text");
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch T_BEGIN {
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch for (i = 0; i < N_ELEMENTS(input); i++) {
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch token = input[i];
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i);
d9b6d24bfa1354757fe9bc68205c0323e40abce3Stephan Bosch test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi fts_filter_unref(&norm);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi } T_END;
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch test_assert(norm == NULL);
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch test_end();
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch}
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Boschstatic void test_fts_filter_normalizer_swedish_short_default_id(void)
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch{
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch struct fts_filter *norm = NULL;
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch const char *input[] = {
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch "Vem",
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch "Å",
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch "ÅÄÖ",
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch "Vem kan segla f\xC3\xB6rutan vind?\n"
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch "\xC3\x85\xC3\x84\xC3\x96\xC3\xB6\xC3\xA4\xC3\xA5"
824ca2373b872449b6b6ce29707c02c0d810eb74Stephan Bosch };
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi const char *expected_output[] = {
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi "vem",
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi "a",
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi "aao",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "vem kan segla forutan vind?\naaooaa"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi };
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *error = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *token = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi unsigned int i;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filter normalizer Swedish short text using default ID");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi T_BEGIN {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, NULL, &norm, &error) == 0);
23a7a07d4d7cab538509ceef21a280dcc243362fStephan Bosch for (i = 0; i < N_ELEMENTS(input); i++) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi token = input[i];
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi fts_filter_unref(&norm);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi } T_END;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(norm == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
17541ea25593c656060199715051db2c1eef221dAki Tuomi
0d1b8b6bec79746c5d89d57dd8c1688946bd9237Josef 'Jeff' Sipek/* UDHRDIR comes from Automake AM_CPPFLAGS */
17541ea25593c656060199715051db2c1eef221dAki Tuomi#define UDHR_FRA_NAME "/udhr_fra.txt"
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_normalizer_french(void)
17541ea25593c656060199715051db2c1eef221dAki Tuomi{
17541ea25593c656060199715051db2c1eef221dAki Tuomi struct fts_filter *norm = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi FILE *input;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const settings[] =
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi char buf[4096] = {0};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *error = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *tokens;
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi unsigned char sha512_digest[SHA512_RESULTLEN];
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct sha512_ctx ctx;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const unsigned char correct_digest[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi 0x78, 0x1e, 0xb9, 0x04, 0xa4, 0x92, 0xca, 0x88,
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi 0x1e, 0xef, 0x7b, 0xc8, 0x3e, 0x4a, 0xa8, 0xdb,
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi 0x9c, 0xd4, 0x42, 0x5c, 0x64, 0x81, 0x06, 0xd5,
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi 0x72, 0x93, 0x38, 0x0c, 0x09, 0xce, 0xbe, 0xdf,
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi 0x65, 0xff, 0x36, 0x35, 0x05, 0x77, 0xcc, 0xc6,
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi 0xff, 0x44, 0x2c, 0x31, 0x10, 0x00, 0xf6, 0x8d,
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi 0x15, 0x25, 0x1e, 0x54, 0x67, 0x2a, 0x5b, 0xc1,
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi 0xdb, 0x84, 0xc5, 0x0d, 0x43, 0x7e, 0x8c, 0x70};
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi const char *udhr_path;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filter normalizer French UDHR");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi T_BEGIN {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi udhr_path = t_strconcat(UDHRDIR, UDHR_FRA_NAME, NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi input = fopen(udhr_path, "r");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(input != NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi sha512_init(&ctx);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi while (NULL != fgets(buf, sizeof(buf), input)) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi tokens = buf;
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi if (fts_filter_filter(norm, &tokens, &error) != 1){
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi break;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek sha512_loop(&ctx, tokens, strlen(tokens));
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi fclose(input);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi sha512_result(&ctx, sha512_digest);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(memcmp(sha512_digest, correct_digest,
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi sizeof(sha512_digest)) == 0);
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi fts_filter_unref(&norm);
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi } T_END;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(norm == NULL);
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_normalizer_empty(void)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi /* test just a couple of these */
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi static const char *empty_tokens[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "\xCC\x80", /* U+0300 */
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi "\xF3\xA0\x87\xAF", /* U+E01EF */
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi "\xCC\x80\xF3\xA0\x87\xAF" /* U+0300 U+E01EF */
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi };
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek const char * const settings[] =
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *norm;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *error;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi unsigned int i;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filter normalizer empty tokens");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi for (i = 0; i < N_ELEMENTS(empty_tokens); i++) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *token = empty_tokens[i];
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert_idx(fts_filter_filter(norm, &token, &error) == 0, i);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi fts_filter_unref(&norm);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_normalizer_baddata(void)
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi{
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi const char * const settings[] =
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *norm;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *token, *error;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi string_t *str;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi unsigned int i;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filter normalizer bad data");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi str = t_str_new(128);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi for (i = 1; i < 0x1ffff; i++) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi str_truncate(str, 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi uni_ucs4_to_utf8_c(i, str);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi token = str_c(str);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi T_BEGIN {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert_idx(fts_filter_filter(norm, &token, &error) >= 0, i);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi } T_END;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi }
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi str_truncate(str, 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi uni_ucs4_to_utf8_c(0x7fffffff, str);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi token = str_c(str);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_filter(norm, &token, &error) >= 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi fts_filter_unref(&norm);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomistatic void test_fts_filter_normalizer_invalid_id(void)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *norm = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *settings[] =
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi {"id", "Any-One-Out-There; DKFN; [: Nonspacing Mark :] Remove",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char *error = NULL, *token = "foo";
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_begin("fts filter normalizer invalid id");
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(error == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_filter(norm, &token, &error) < 0 && error != NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi#ifdef HAVE_FTS_STEMMER
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomistatic void test_fts_filter_normalizer_stopwords_stemmer_eng(void)
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi{
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi int ret;
e4e8301f2b8ee44abcd901db4e27f150de7c4739Timo Sirainen struct fts_filter *normalizer;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *stemmer;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi struct fts_filter *filter;
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi const char *error;
a371ea8bd48d45548cd7aa16d4f5aeb38ba48c91Aki Tuomi const char * const id_settings[] =
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi //{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC", NULL};
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi {"id", "Lower", NULL};
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi const char *token = NULL;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const tokens[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "dries" ,"friendlies", "All", "human", "beings", "are",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "born", "free", "and", "equal", "in", "dignity", "and",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "rights", "They", "are", "endowed", "with", "reason", "and",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "conscience", "and", "should", "act", "towards", "one",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "another", "in", "a", "spirit", "of", "brotherhood", "ABCFoo",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi NULL};
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi const char * const bases[] = {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "dri" ,"friend", "all", "human", "be", NULL, "born", "free",
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi NULL, "equal", NULL, "digniti", NULL, "right", NULL, NULL,
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "endow", NULL, "reason", NULL, "conscienc", NULL, "should",
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi "act", "toward", "one", "anoth", NULL, NULL, "spirit", NULL,
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi "brotherhood", "abcfoo", NULL};
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi const char * const *tpp;
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi const char * const *bpp;
03c6532fe7aa3ddae23c99ff6bec78d8dd2e8165Aki Tuomi
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi test_begin("fts filters normalizer, stopwords and stemming chained, English");
1be27c35ea17fccd83c54e2acc66eb8c44d1a8feAki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, id_settings, &normalizer, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_stopwords, normalizer, &english_language, stopword_settings, &filter, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(fts_filter_create(fts_filter_stemmer_snowball, filter, &english_language, NULL, &stemmer, &error) == 0);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi bpp = bases;
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi for (tpp = tokens; *tpp != NULL; tpp++) {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi token = *tpp;
f207fb0f2fd3113aa9a9f7911fb12d94dce19dffAki Tuomi ret = fts_filter_filter(stemmer, &token, &error);
f207fb0f2fd3113aa9a9f7911fb12d94dce19dffAki Tuomi if (ret <= 0) {
f207fb0f2fd3113aa9a9f7911fb12d94dce19dffAki Tuomi test_assert(ret == 0);
f207fb0f2fd3113aa9a9f7911fb12d94dce19dffAki Tuomi test_assert(*bpp == NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi } else {
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(*bpp != NULL);
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi test_assert(strcmp(*bpp, token) == 0);
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi }
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi bpp++;
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek }
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek fts_filter_unref(&stemmer);
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek fts_filter_unref(&filter);
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek fts_filter_unref(&normalizer);
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek test_assert(stemmer == NULL);
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek test_assert(filter == NULL);
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi test_assert(normalizer == NULL);
5f1d689131a75c39f064cbd4202373e7edf78f18Josef 'Jeff' Sipek test_end();
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi}
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi#endif
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi#endif
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi
097dbdf543bc5d1689c5570f5faaec1e864e3a87Aki Tuomi/* TODO: Functions to test 1. ref-unref pairs 2. multiple registers +
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi an unregister + find */
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomi
4036c1ca99d2c517f68a5b67a419ae7fdfd45300Aki Tuomiint main(void)
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi{
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi static void (*test_functions[])(void) = {
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_find,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_lowercase,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_stopwords_eng,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_stopwords_fin,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_stopwords_fra,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_stopwords_fail_lazy_init,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi#ifdef HAVE_FTS_STEMMER
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_stemmer_snowball_stem_english,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_stemmer_snowball_stem_french,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_stopwords_stemmer_eng,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi#endif
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi#ifdef HAVE_LIBICU
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_normalizer_swedish_short,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_normalizer_swedish_short_default_id,
204afc1f4f37a4f1cb53ff44b993a661cc45bf5dAki Tuomi test_fts_filter_normalizer_french,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_normalizer_empty,
98bc2ecdbfd4f2f20c3a5e96ae445072fbe22223Aki Tuomi test_fts_filter_normalizer_baddata,
4a197212360f75bfc89254bfd5bc4a31151fe4b4Timo Sirainen test_fts_filter_normalizer_invalid_id,
4a197212360f75bfc89254bfd5bc4a31151fe4b4Timo Sirainen#ifdef HAVE_FTS_STEMMER
4a197212360f75bfc89254bfd5bc4a31151fe4b4Timo Sirainen test_fts_filter_normalizer_stopwords_stemmer_eng,
4a197212360f75bfc89254bfd5bc4a31151fe4b4Timo Sirainen#endif
80521bcdd28b22818480a6e6e1ae84230e19c1baAki Tuomi#endif
80521bcdd28b22818480a6e6e1ae84230e19c1baAki Tuomi NULL
4a197212360f75bfc89254bfd5bc4a31151fe4b4Timo Sirainen };
80521bcdd28b22818480a6e6e1ae84230e19c1baAki Tuomi int ret;
80521bcdd28b22818480a6e6e1ae84230e19c1baAki Tuomi
80521bcdd28b22818480a6e6e1ae84230e19c1baAki Tuomi fts_filters_init();
4a197212360f75bfc89254bfd5bc4a31151fe4b4Timo Sirainen ret = test_run(test_functions);
80521bcdd28b22818480a6e6e1ae84230e19c1baAki Tuomi fts_filters_deinit();
4a197212360f75bfc89254bfd5bc4a31151fe4b4Timo Sirainen return ret;
80521bcdd28b22818480a6e6e1ae84230e19c1baAki Tuomi}
80521bcdd28b22818480a6e6e1ae84230e19c1baAki Tuomi