fts-backend-solr-old.c revision c215ca02d468b0e542523df1ed18e5f2d7e63968
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2006-2013 Dovecot authors, see the included COPYING file */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include "lib.h"
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen#include "array.h"
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include "str.h"
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include "hash.h"
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include "strescape.h"
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include "unichar.h"
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include "http-url.h"
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include "imap-utf7.h"
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include "mail-storage-private.h"
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen#include "mailbox-list-private.h"
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include "mail-search.h"
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include "fts-api.h"
1d738cce754bc64bbc66d3355ebdaf3f6eac55f1Timo Sirainen#include "solr-connection.h"
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen#include "fts-solr-plugin.h"
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen#include <ctype.h>
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen#define SOLR_CMDBUF_SIZE (1024*64)
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen#define SOLR_MAX_MULTI_ROWS 100000
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainenstruct solr_fts_backend {
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen struct fts_backend backend;
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen struct solr_connection *solr_conn;
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen char *id_username, *id_namespace;
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen struct mail_namespace *default_ns;
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen};
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainenstruct solr_fts_backend_update_context {
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen struct fts_backend_update_context ctx;
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen struct mailbox *cur_box;
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen char *id_box_name;
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen struct solr_connection_post *post;
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen uint32_t prev_uid, uid_validity;
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen string_t *cmd, *hdr;
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen bool headers_open;
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen bool body_open;
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen bool documents_added;
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen};
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainenstatic bool is_valid_xml_char(unichar_t chr)
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen{
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen /* Valid characters in XML:
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] |
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen [#x10000-#x10FFFF]
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen This function gets called only for #x80 and higher */
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen if (chr > 0xd7ff && chr < 0xe000)
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen return FALSE;
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen if (chr > 0xfffd && chr < 0x10000)
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen return FALSE;
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen return chr < 0x10ffff;
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen}
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainenstatic void
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainenxml_encode_data(string_t *dest, const unsigned char *data, unsigned int len)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen unichar_t chr;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen unsigned int i;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen for (i = 0; i < len; i++) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen switch (data[i]) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen case '&':
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(dest, "&amp;");
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen break;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen case '<':
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(dest, "&lt;");
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen break;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen case '>':
b397665e90fa0fc7c6a9156fdd6cf28b571e8e39Timo Sirainen str_append(dest, "&gt;");
b397665e90fa0fc7c6a9156fdd6cf28b571e8e39Timo Sirainen break;
b397665e90fa0fc7c6a9156fdd6cf28b571e8e39Timo Sirainen case '\t':
b397665e90fa0fc7c6a9156fdd6cf28b571e8e39Timo Sirainen case '\n':
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen case '\r':
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* exceptions to the following control char check */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append_c(dest, data[i]);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen break;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen default:
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (data[i] < 32) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* SOLR doesn't like control characters.
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen replace them with spaces. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append_c(dest, ' ');
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen } else if (data[i] >= 0x80) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* make sure the character is valid for XML
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen so we don't get XML parser errors */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen unsigned int char_len =
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen uni_utf8_char_bytes(data[i]);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (i + char_len <= len &&
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen uni_utf8_get_char_n(data + i, char_len, &chr) == 1 &&
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen is_valid_xml_char(chr))
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_append_n(dest, data + i, char_len);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_append_n(dest, utf8_replacement_char,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen UTF8_REPLACEMENT_CHAR_LEN);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen i += char_len - 1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen } else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_append_c(dest, data[i]);
da4376093d4e1b26b14ea1e945689fb7056fe0a0Timo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen break;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstatic void xml_encode(string_t *dest, const char *str)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen xml_encode_data(dest, (const unsigned char *)str, strlen(str));
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstatic const char *solr_escape_id_str(const char *str)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen string_t *tmp;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen const char *p;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen for (p = str; *p != '\0'; p++) {
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen if (*p == '/' || *p == '!')
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen break;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (*p == '\0')
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return str;
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen tmp = t_str_new(64);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen for (p = str; *p != '\0'; p++) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen switch (*p) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen case '/':
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(tmp, "!\\");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen break;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen case '!':
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(tmp, "!!");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen default:
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append_c(tmp, *p);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen }
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen return str_c(tmp);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainenstatic void solr_quote(string_t *dest, const char *str)
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen{
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen str_append_c(dest, '"');
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen str_append(dest, str_escape(str));
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen str_append_c(dest, '"');
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen}
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainenstatic void solr_quote_http(string_t *dest, const char *str)
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen{
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen str_append(dest, "%22");
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen http_url_escape_param(dest, str);
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen str_append(dest, "%22");
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenstatic void fts_solr_set_default_ns(struct solr_fts_backend *backend)
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen{
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen struct mail_namespace *ns = backend->backend.ns;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(ns->user);
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen const struct fts_solr_settings *set = &fuser->set;
0bf3eac1110a902e7ec7e695c64e8e46c114e623Timo Sirainen const char *str;
0bf3eac1110a902e7ec7e695c64e8e46c114e623Timo Sirainen
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen if (backend->default_ns != NULL)
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen return;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen if (set->default_ns_prefix != NULL) {
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen backend->default_ns =
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen mail_namespace_find_prefix(ns->user->namespaces,
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen set->default_ns_prefix);
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen if (backend->default_ns == NULL) {
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen i_error("fts_solr: default_ns setting points to "
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen "nonexistent namespace");
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen }
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen }
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen if (backend->default_ns == NULL) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen backend->default_ns =
f153a2cec0319f549388d28f8cfd4d50229d1132Timo Sirainen mail_namespace_find_inbox(ns->user->namespaces);
f153a2cec0319f549388d28f8cfd4d50229d1132Timo Sirainen }
f153a2cec0319f549388d28f8cfd4d50229d1132Timo Sirainen while (backend->default_ns->alias_for != NULL)
f153a2cec0319f549388d28f8cfd4d50229d1132Timo Sirainen backend->default_ns = backend->default_ns->alias_for;
f153a2cec0319f549388d28f8cfd4d50229d1132Timo Sirainen
dffa503fd4ce31334346e539496084c80a2d8d37Timo Sirainen if (ns != backend->default_ns) {
dffa503fd4ce31334346e539496084c80a2d8d37Timo Sirainen str = solr_escape_id_str(ns->prefix);
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen backend->id_namespace = i_strdup(str);
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen }
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen}
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainenstatic void fts_box_name_get_root(struct mail_namespace **ns, const char **name)
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen{
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen struct mail_namespace *orig_ns = *ns;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen while ((*ns)->alias_for != NULL)
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen *ns = (*ns)->alias_for;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (**name == '\0' && *ns != orig_ns &&
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen ((*ns)->flags & NAMESPACE_FLAG_INBOX_USER) != 0) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* ugly workaround to allow selecting INBOX from a Maildir/
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen when it's not in the inbox=yes namespace. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen *name = "INBOX";
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstatic const char *
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainenfts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r)
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen{
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen struct mail_namespace *ns = mailbox_get_namespace(box);
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen const char *name;
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen if (t_imap_utf8_to_utf7(box->name, &name) < 0)
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen i_unreached();
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_box_name_get_root(&ns, &name);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen *ns_r = ns;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return name;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstatic struct fts_backend *fts_backend_solr_alloc(void)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen struct solr_fts_backend *backend;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
57dd2621d4230bc2e06c69a4b9f8cf8f0ff2bc93Timo Sirainen backend = i_new(struct solr_fts_backend, 1);
57dd2621d4230bc2e06c69a4b9f8cf8f0ff2bc93Timo Sirainen backend->backend = fts_backend_solr_old;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return &backend->backend;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstatic int
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenfts_backend_solr_init(struct fts_backend *_backend, const char **error_r)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
40ad2c4902e9d83557f2e8a4bff3d98fea2c8aa1Timo Sirainen struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_backend->ns->user);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen const char *str;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (fuser == NULL) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen *error_r = "Invalid fts_solr setting";
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen return -1;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen }
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen if (solr_connection_init(fuser->set.url, fuser->set.debug,
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen &backend->solr_conn, error_r) < 0)
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen return -1;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen str = solr_escape_id_str(_backend->ns->user->username);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen backend->id_username = i_strdup(str);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen return 0;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainenstatic void fts_backend_solr_deinit(struct fts_backend *_backend)
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen{
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
b6612c334604eeb27e1ca2bd804ac66dcbc2eaadTimo Sirainen
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen i_free(backend->id_namespace);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_free(backend->id_username);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_free(backend);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
1ff03d2244dd08effadfecec365bec28793a1aa0Timo Sirainen
1ff03d2244dd08effadfecec365bec28793a1aa0Timo Sirainenstatic void
1ff03d2244dd08effadfecec365bec28793a1aa0Timo Sirainensolr_add_ns_query(string_t *str, struct solr_fts_backend *backend,
1ff03d2244dd08effadfecec365bec28793a1aa0Timo Sirainen struct mail_namespace *ns, bool neg)
1ff03d2244dd08effadfecec365bec28793a1aa0Timo Sirainen{
1ff03d2244dd08effadfecec365bec28793a1aa0Timo Sirainen while (ns->alias_for != NULL)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ns = ns->alias_for;
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen if (ns == backend->default_ns || *ns->prefix == '\0') {
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen if (!neg)
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen str_append(str, " -ns:[* TO *]");
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen else
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen str_append(str, " +ns:[* TO *]");
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen } else {
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen if (!neg)
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen str_append(str, " +ns:");
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen else
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen str_append(str, " -ns:");
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen solr_quote(str, ns->prefix);
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen }
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen}
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainenstatic void
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainensolr_add_ns_query_http(string_t *str, struct solr_fts_backend *backend,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct mail_namespace *ns)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen string_t *tmp;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen tmp = t_str_new(64);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen solr_add_ns_query(tmp, backend, ns, FALSE);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen http_url_escape_param(str, str_c(tmp));
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenfts_backend_solr_get_last_uid_fallback(struct solr_fts_backend *backend,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct mailbox *box,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen uint32_t *last_uid_r)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct mail_namespace *ns;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct mailbox_status status;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct solr_result **results;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen const struct seq_range *uidvals;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen const char *box_name;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen unsigned int count;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen string_t *str;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen pool_t pool;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen int ret = 0;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str = t_str_new(256);
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen str_append(str, "fl=uid&rows=1&sort=uid+desc&q=");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen box_name = fts_box_get_root(box, &ns);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_printfa(str, "uidv:%u+box:", status.uidvalidity);
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen solr_quote_http(str, box_name);
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen solr_add_ns_query_http(str, backend, ns);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(str, "+user:");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen solr_quote_http(str, ns->user->username);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen pool = pool_alloconly_create("solr last uid lookup", 1024);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (solr_connection_select(backend->solr_conn, str_c(str),
d9fda7e3a0fa5551547ac3e3054b837fc77f4bfbTimo Sirainen pool, &results) < 0)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ret = -1;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen else if (results[0] == NULL) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* no UIDs */
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen *last_uid_r = 0;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen } else {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen uidvals = array_get(&results[0]->uids, &count);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen i_assert(count > 0);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen *last_uid_r = uidvals[0].seq1;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen } else {
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen i_error("fts_solr: Last UID lookup returned multiple rows");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ret = -1;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen pool_unref(&pool);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return ret;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstatic int
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenfts_backend_solr_get_last_uid(struct fts_backend *_backend,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct mailbox *box, uint32_t *last_uid_r)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct solr_fts_backend *backend =
b66d803de86bfb411165b3465b0d9ef64ecfe2a1Timo Sirainen (struct solr_fts_backend *)_backend;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct fts_index_header hdr;
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (fts_index_get_header(box, &hdr)) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen *last_uid_r = hdr.last_indexed_uid;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return 0;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* either nothing has been indexed, or the index was corrupted.
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen do it the slow way. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (fts_backend_solr_get_last_uid_fallback(backend, box, last_uid_r) < 0)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return -1;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_index_set_last_uid(box, *last_uid_r);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return 0;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainenstatic struct fts_backend_update_context *
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenfts_backend_solr_update_init(struct fts_backend *_backend)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct solr_fts_backend *backend =
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen (struct solr_fts_backend *)_backend;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct solr_fts_backend_update_context *ctx;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ctx = i_new(struct solr_fts_backend_update_context, 1);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ctx->ctx.backend = _backend;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ctx->hdr = str_new(default_pool, 4096);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_solr_set_default_ns(backend);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return &ctx->ctx;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
1deffbbcd7affd2ec89284f3c644cc73db0a2b90Timo Sirainen
1deffbbcd7affd2ec89284f3c644cc73db0a2b90Timo Sirainenstatic void xml_encode_id(struct solr_fts_backend_update_context *ctx,
1deffbbcd7affd2ec89284f3c644cc73db0a2b90Timo Sirainen string_t *str, uint32_t uid)
1deffbbcd7affd2ec89284f3c644cc73db0a2b90Timo Sirainen{
1deffbbcd7affd2ec89284f3c644cc73db0a2b90Timo Sirainen struct solr_fts_backend *backend =
1deffbbcd7affd2ec89284f3c644cc73db0a2b90Timo Sirainen (struct solr_fts_backend *)ctx->ctx.backend;
1deffbbcd7affd2ec89284f3c644cc73db0a2b90Timo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (uid != 0)
97cb20eb77d486ef67eac50567e3080faca025c1Timo Sirainen str_printfa(str, "%u/", uid);
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen else
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(str, "L/");
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (backend->id_namespace != NULL) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen xml_encode(str, backend->id_namespace);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append_c(str, '/');
d9fda7e3a0fa5551547ac3e3054b837fc77f4bfbTimo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_printfa(str, "%u/", ctx->uid_validity);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen xml_encode(str, backend->id_username);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append_c(str, '/');
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen xml_encode(str, ctx->id_box_name);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstatic void
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenfts_backend_solr_add_doc_prefix(struct solr_fts_backend_update_context *ctx,
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen uint32_t uid)
94cb0544ba0b6e249f20b76ad481bad27a633632Timo Sirainen{
94cb0544ba0b6e249f20b76ad481bad27a633632Timo Sirainen struct solr_fts_backend *backend =
94cb0544ba0b6e249f20b76ad481bad27a633632Timo Sirainen (struct solr_fts_backend *)ctx->ctx.backend;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct mailbox *box = ctx->cur_box;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct mail_namespace *ns;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen const char *box_name;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
2f8da04d700cc23fcd6630226a4866e828b761bdTimo Sirainen ctx->documents_added = TRUE;
2f8da04d700cc23fcd6630226a4866e828b761bdTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_printfa(ctx->cmd, "<doc>"
d9fda7e3a0fa5551547ac3e3054b837fc77f4bfbTimo Sirainen "<field name=\"uid\">%u</field>"
d9fda7e3a0fa5551547ac3e3054b837fc77f4bfbTimo Sirainen "<field name=\"uidv\">%u</field>",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen uid, ctx->uid_validity);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen box_name = fts_box_get_root(box, &ns);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
07e80e04c8876b6bf3f95266f48b41e1a681e445Timo Sirainen if (ns != backend->default_ns) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(ctx->cmd, "<field name=\"ns\">");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen xml_encode(ctx->cmd, ns->prefix);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(ctx->cmd, "</field>");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen }
d9fda7e3a0fa5551547ac3e3054b837fc77f4bfbTimo Sirainen str_append(ctx->cmd, "<field name=\"box\">");
4eccf5310af7bed72cd6f7a559a93165c516e514Timo Sirainen xml_encode(ctx->cmd, box_name);
2f8da04d700cc23fcd6630226a4866e828b761bdTimo Sirainen str_append(ctx->cmd, "</field><field name=\"user\">");
2f8da04d700cc23fcd6630226a4866e828b761bdTimo Sirainen xml_encode(ctx->cmd, ns->user->username);
d9fda7e3a0fa5551547ac3e3054b837fc77f4bfbTimo Sirainen str_append(ctx->cmd, "</field>");
d9fda7e3a0fa5551547ac3e3054b837fc77f4bfbTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainenstatic int
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenfts_backed_solr_build_commit(struct solr_fts_backend_update_context *ctx)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (ctx->post == NULL)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return 0;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(ctx->cmd, "</doc></add>");
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen solr_connection_post_more(ctx->post, str_data(ctx->cmd),
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen str_len(ctx->cmd));
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen return solr_connection_post_end(ctx->post);
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen}
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainenstatic int
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainenfts_backend_solr_update_deinit(struct fts_backend_update_context *_ctx)
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen{
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen struct solr_fts_backend_update_context *ctx =
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen (struct solr_fts_backend_update_context *)_ctx;
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen struct solr_fts_backend *backend =
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen (struct solr_fts_backend *)_ctx->backend;
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen const char *str;
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen int ret;
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen ret = fts_backed_solr_build_commit(ctx);
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen /* commit and wait until the documents we just indexed are
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen visible to the following search */
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen str = t_strdup_printf("<commit waitFlush=\"false\" "
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen "waitSearcher=\"%s\"/>",
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen ctx->documents_added ? "true" : "false");
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen if (solr_connection_post(backend->solr_conn, str) < 0)
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen ret = -1;
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen str_free(&ctx->cmd);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen str_free(&ctx->hdr);
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen i_free(ctx->id_box_name);
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen i_free(ctx);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen return ret;
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen}
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainenstatic void
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainenfts_backend_solr_update_set_mailbox(struct fts_backend_update_context *_ctx,
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen struct mailbox *box)
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen{
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen struct solr_fts_backend_update_context *ctx =
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen (struct solr_fts_backend_update_context *)_ctx;
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen struct mailbox_status status;
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen struct mail_namespace *ns;
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen if (ctx->prev_uid != 0) {
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen fts_index_set_last_uid(ctx->cur_box, ctx->prev_uid);
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen ctx->prev_uid = 0;
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen }
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen ctx->cur_box = box;
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen ctx->uid_validity = 0;
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen i_free_and_null(ctx->id_box_name);
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen if (box != NULL) {
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen ctx->id_box_name = i_strdup(fts_box_get_root(box, &ns));
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen ctx->uid_validity = status.uidvalidity;
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen }
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen}
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainenstatic void
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainenfts_backend_solr_update_expunge(struct fts_backend_update_context *_ctx,
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen uint32_t uid)
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen{
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen struct solr_fts_backend_update_context *ctx =
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen (struct solr_fts_backend_update_context *)_ctx;
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen struct solr_fts_backend *backend =
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen (struct solr_fts_backend *)_ctx->backend;
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen T_BEGIN {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen string_t *cmd;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen cmd = t_str_new(256);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(cmd, "<delete><id>");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen xml_encode_id(ctx, cmd, uid);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(cmd, "</id></delete>");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen (void)solr_connection_post(backend->solr_conn, str_c(cmd));
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen } T_END;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen}
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainenstatic void
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainenfts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen uint32_t uid)
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen{
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen struct solr_fts_backend *backend =
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen (struct solr_fts_backend *)ctx->ctx.backend;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen if (ctx->post == NULL) {
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen i_assert(ctx->prev_uid == 0);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen ctx->post = solr_connection_post_begin(backend->solr_conn);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen str_append(ctx->cmd, "<add>");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen } else {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ctx->headers_open = FALSE;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (ctx->body_open) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ctx->body_open = FALSE;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(ctx->cmd, "</field>");
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen }
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen str_append(ctx->cmd, "<field name=\"hdr\">");
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen str_append_str(ctx->cmd, ctx->hdr);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen str_append(ctx->cmd, "</field>");
40ad2c4902e9d83557f2e8a4bff3d98fea2c8aa1Timo Sirainen str_truncate(ctx->hdr, 0);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen str_append(ctx->cmd, "</doc>");
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen }
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen ctx->prev_uid = uid;
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen fts_backend_solr_add_doc_prefix(ctx, uid);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen str_printfa(ctx->cmd, "<field name=\"id\">");
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen xml_encode_id(ctx, ctx->cmd, uid);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen str_append(ctx->cmd, "</field>");
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstatic bool
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenfts_backend_solr_update_set_build_key(struct fts_backend_update_context *_ctx,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen const struct fts_backend_build_key *key)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct solr_fts_backend_update_context *ctx =
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen (struct solr_fts_backend_update_context *)_ctx;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (key->uid != ctx->prev_uid)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_backend_solr_uid_changed(ctx, key->uid);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen switch (key->type) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen case FTS_BACKEND_BUILD_KEY_HDR:
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen case FTS_BACKEND_BUILD_KEY_MIME_HDR:
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen xml_encode(ctx->hdr, key->hdr_name);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen str_append(ctx->hdr, ": ");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ctx->headers_open = TRUE;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen break;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen case FTS_BACKEND_BUILD_KEY_BODY_PART:
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ctx->headers_open = FALSE;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (!ctx->body_open) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ctx->body_open = TRUE;
94aa90d2d17a7aebcda5a4193a62e80ddbb169b7Timo Sirainen str_append(ctx->cmd, "<field name=\"body\">");
94aa90d2d17a7aebcda5a4193a62e80ddbb169b7Timo Sirainen }
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen break;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY:
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen i_unreached();
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen return TRUE;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstatic void
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenfts_backend_solr_update_unset_build_key(struct fts_backend_update_context *_ctx)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct solr_fts_backend_update_context *ctx =
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen (struct solr_fts_backend_update_context *)_ctx;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (ctx->headers_open)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append_c(ctx->hdr, '\n');
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(ctx->body_open);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_append_c(ctx->cmd, '\n');
8f0e6d627f3646e559ac5224c306839669d1a5e0Timo Sirainen }
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen}
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainenstatic int
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainenfts_backend_solr_update_build_more(struct fts_backend_update_context *_ctx,
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen const unsigned char *data, size_t size)
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen{
8f0e6d627f3646e559ac5224c306839669d1a5e0Timo Sirainen struct solr_fts_backend_update_context *ctx =
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen (struct solr_fts_backend_update_context *)_ctx;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen xml_encode_data(ctx->cmd, data, size);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if (str_len(ctx->cmd) > SOLR_CMDBUF_SIZE-128) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen solr_connection_post_more(ctx->post, str_data(ctx->cmd),
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_len(ctx->cmd));
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_truncate(ctx->cmd, 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int fts_backend_solr_refresh(struct fts_backend *backend ATTR_UNUSED)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
85da8c055280cd45553b6b335e9fb226d6e2801eTimo Sirainen}
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic bool
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainensolr_add_definite_query(string_t *str, struct mail_search_arg *arg)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen switch (arg->type) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen case SEARCH_TEXT: {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (arg->match_not)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_append_c(str, '-');
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_append(str, "(hdr:");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen solr_quote_http(str, arg->value.str);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_append(str, "+OR+body:");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen solr_quote_http(str, arg->value.str);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_append(str, ")");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen case SEARCH_BODY:
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (arg->match_not)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_append_c(str, '-');
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_append(str, "body:");
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen solr_quote_http(str, arg->value.str);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen break;
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen default:
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen return FALSE;
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen }
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen return TRUE;
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen}
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainenstatic bool
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainensolr_add_definite_query_args(string_t *str, struct mail_search_arg *arg,
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen bool and_args)
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen{
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen unsigned int last_len;
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen last_len = str_len(str);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen for (; arg != NULL; arg = arg->next) {
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen if (solr_add_definite_query(str, arg)) {
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen arg->match_always = TRUE;
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen last_len = str_len(str);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen if (and_args)
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen str_append(str, "+AND+");
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen else
5921647936e3d9a2dc60c0e49c66dd33efca234dTimo Sirainen str_append(str, "+OR+");
5921647936e3d9a2dc60c0e49c66dd33efca234dTimo Sirainen }
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen }
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen if (str_len(str) == last_len)
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen return FALSE;
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen str_truncate(str, last_len);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen return TRUE;
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen}
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainenstatic int
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainenfts_backend_solr_lookup(struct fts_backend *_backend, struct mailbox *box,
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen struct mail_search_arg *args, bool and_args,
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen struct fts_result *result)
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen{
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen struct solr_fts_backend *backend =
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen (struct solr_fts_backend *)_backend;
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen struct mail_namespace *ns;
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen struct mailbox_status status;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen string_t *str;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen const char *box_name;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen pool_t pool;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen struct solr_result **results;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen int ret;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_solr_set_default_ns(backend);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen mailbox_get_open_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT,
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen &status);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
1bea995196e46157e495a78b8f93780c576b3ef8Timo Sirainen str = t_str_new(256);
1bea995196e46157e495a78b8f93780c576b3ef8Timo Sirainen str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=",
1bea995196e46157e495a78b8f93780c576b3ef8Timo Sirainen status.uidnext);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
bd503f12eb667df389a99162f567bd8785798f55Timo Sirainen if (!solr_add_definite_query_args(str, args, and_args)) {
bd503f12eb667df389a99162f567bd8785798f55Timo Sirainen /* can't search this query */
bd503f12eb667df389a99162f567bd8785798f55Timo Sirainen return 0;
bd503f12eb667df389a99162f567bd8785798f55Timo Sirainen }
bd503f12eb667df389a99162f567bd8785798f55Timo Sirainen
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen /* use a separate filter query for selecting the mailbox. it shouldn't
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen affect the score and there could be some caching benefits too. */
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen str_append(str, "&fq=%2Buser:");
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen solr_quote_http(str, box->storage->user->username);
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen box_name = fts_box_get_root(box, &ns);
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen str_printfa(str, "+%%2Buidv:%u+%%2Bbox:", status.uidvalidity);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen solr_quote_http(str, box_name);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen solr_add_ns_query_http(str, backend, ns);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen pool = pool_alloconly_create("fts solr search", 1024);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = solr_connection_select(backend->solr_conn, str_c(str),
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen pool, &results);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret == 0 && results[0] != NULL) {
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen array_append_array(&result->definite_uids, &results[0]->uids);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen array_append_array(&result->scores, &results[0]->scores);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen }
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen result->scores_sorted = TRUE;
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen pool_unref(&pool);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen return ret;
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic char *
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmailbox_get_id(struct solr_fts_backend *backend, struct mail_namespace *ns,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const char *mailbox, uint32_t uidvalidity)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen string_t *str = t_str_new(64);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_printfa(str, "%u\001", uidvalidity);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(str, mailbox);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (ns != backend->default_ns)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen str_printfa(str, "\001%s", ns->prefix);
bd503f12eb667df389a99162f567bd8785798f55Timo Sirainen return str_c_modifiable(str);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainenstatic int
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainensolr_search_multi(struct solr_fts_backend *backend, string_t *str,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mailbox *const boxes[],
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct fts_multi_result *result)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen struct solr_result **solr_results;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct fts_result *fts_result;
347acd14d8da653ce3757b3e29981326502bed6bTimo Sirainen ARRAY(struct fts_result) fts_results;
347acd14d8da653ce3757b3e29981326502bed6bTimo Sirainen struct mail_namespace *ns;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mailbox_status status;
6cb2c6ecddcdbeac9e6c73a292244747e12a793eTimo Sirainen HASH_TABLE(char *, struct mailbox *) mailboxes;
8cd0a1a2200e65cd134d03fe3f93ec02f1746359Timo Sirainen struct mailbox *box;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const char *box_name;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen char *box_id;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unsigned int i, len;
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen /* use a separate filter query for selecting the mailbox. it shouldn't
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen affect the score and there could be some caching benefits too. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_append(str, "&fq=%2Buser:");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (backend->backend.ns->owner != NULL)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen solr_quote_http(str, backend->backend.ns->owner->username);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen else
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen str_append(str, "%22%22");
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen hash_table_create(&mailboxes, default_pool, 0, str_hash, strcmp);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen str_append(str, "%2B(");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen len = str_len(str);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen for (i = 0; boxes[i] != NULL; i++) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (str_len(str) != len)
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen str_append(str, "+OR+");
bf8f4f90cb5e5f32c2611ba3425557964b9c47fcTimo Sirainen
bf8f4f90cb5e5f32c2611ba3425557964b9c47fcTimo Sirainen box_name = fts_box_get_root(boxes[i], &ns);
bf8f4f90cb5e5f32c2611ba3425557964b9c47fcTimo Sirainen mailbox_get_open_status(boxes[i], STATUS_UIDVALIDITY, &status);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen str_printfa(str, "%%2B(%%2Buidv:%u+%%2Bbox:", status.uidvalidity);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen solr_quote_http(str, box_name);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen solr_add_ns_query_http(str, backend, ns);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen str_append_c(str, ')');
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen box_id = mailbox_get_id(backend, ns, box_name, status.uidvalidity);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen hash_table_insert(mailboxes, box_id, boxes[i]);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen }
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen str_append_c(str, ')');
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen if (solr_connection_select(backend->solr_conn, str_c(str),
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen result->pool, &solr_results) < 0) {
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen hash_table_destroy(&mailboxes);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return -1;
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen }
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen p_array_init(&fts_results, result->pool, 32);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen for (i = 0; solr_results[i] != NULL; i++) {
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen box = hash_table_lookup(mailboxes, solr_results[i]->box_id);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen if (box == NULL) {
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen i_warning("fts_solr: Lookup returned unexpected mailbox "
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen "with id=%s", solr_results[i]->box_id);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen continue;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen fts_result = array_append_space(&fts_results);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen fts_result->box = box;
1bea995196e46157e495a78b8f93780c576b3ef8Timo Sirainen fts_result->definite_uids = solr_results[i]->uids;
629600d9a85e8025c15a5eaeb80329e116e022c9Timo Sirainen fts_result->scores = solr_results[i]->scores;
629600d9a85e8025c15a5eaeb80329e116e022c9Timo Sirainen fts_result->scores_sorted = TRUE;
629600d9a85e8025c15a5eaeb80329e116e022c9Timo Sirainen }
629600d9a85e8025c15a5eaeb80329e116e022c9Timo Sirainen array_append_zero(&fts_results);
629600d9a85e8025c15a5eaeb80329e116e022c9Timo Sirainen result->box_results = array_idx_modifiable(&fts_results, 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hash_table_destroy(&mailboxes);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstatic int
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenfts_backend_solr_lookup_multi(struct fts_backend *_backend,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct mailbox *const boxes[],
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct mail_search_arg *args, bool and_args,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct fts_multi_result *result)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen{
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen struct solr_fts_backend *backend =
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen (struct solr_fts_backend *)_backend;
39087f589d24f3072f220c2ed4528ee323f129ffTimo Sirainen string_t *str;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_solr_set_default_ns(backend);
9716b2665ee3938d3dfe64bda44d7c3ae3b55d30Timo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str = t_str_new(256);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen str_printfa(str, "fl=ns,box,uidv,uid,score&rows=%u&sort=box+asc,uid+asc&q=",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen SOLR_MAX_MULTI_ROWS);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (solr_add_definite_query_args(str, args, and_args)) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (solr_search_multi(backend, str, boxes, result) < 0)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return -1;
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen }
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen /* FIXME: maybe_uids could be handled also with some more work.. */
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen return 0;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen}
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenstruct fts_backend fts_backend_solr_old = {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen .name = "solr_old",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen .flags = 0,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_backend_solr_alloc,
1bea995196e46157e495a78b8f93780c576b3ef8Timo Sirainen fts_backend_solr_init,
1bea995196e46157e495a78b8f93780c576b3ef8Timo Sirainen fts_backend_solr_deinit,
1bea995196e46157e495a78b8f93780c576b3ef8Timo Sirainen fts_backend_solr_get_last_uid,
e1ad2d13f5f154ee975eed5c8e7f1a7823082685Timo Sirainen fts_backend_solr_update_init,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_backend_solr_update_deinit,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_backend_solr_update_set_mailbox,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen fts_backend_solr_update_expunge,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen fts_backend_solr_update_set_build_key,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_backend_solr_update_unset_build_key,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_backend_solr_update_build_more,
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen fts_backend_solr_refresh,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen NULL,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_backend_solr_optimize,
e1ad2d13f5f154ee975eed5c8e7f1a7823082685Timo Sirainen fts_backend_default_can_lookup,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_backend_solr_lookup,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen fts_backend_solr_lookup_multi,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen NULL
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen }
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen};
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen