fts-backend-solr.c revision 6b09a3b269f4b10364c9a77f6614dbe3d306b79d
2e37d45867d081db150ab78dad303b9077aea24fTimo Sirainen/* Copyright (c) 2006-2008 Dovecot authors, see the included COPYING file */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "lib.h"
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "array.h"
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "str.h"
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "mail-storage-private.h"
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "solr-connection.h"
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "fts-solr-plugin.h"
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include <stdlib.h>
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include <curl/curl.h>
7a7d2aa11e46195e2d92d6c337d7e78052a5ce67Timo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#define SOLR_CMDBUF_SIZE (1024*64)
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstruct solr_fts_backend_build_context {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct fts_backend_build_context ctx;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct solr_connection_post *post;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uint32_t prev_uid, uid_validity;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen string_t *cmd;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen bool headers;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen};
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic struct solr_connection *solr_conn = NULL;
499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic void solr_quote_str(string_t *dest, const char *str)
f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen{
f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen solr_connection_quote_str(solr_conn, dest, str);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen}
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainenstatic void
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainenxml_encode_data(string_t *dest, const unsigned char *data, unsigned int len)
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen{
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen unsigned int i;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen for (i = 0; i < len; i++) {
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen switch (data[i]) {
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen case '&':
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen str_append(dest, "&amp;");
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen break;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen case '<':
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen str_append(dest, "&lt;");
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen break;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen case '>':
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen str_append(dest, "&gt;");
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen break;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen default:
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen str_append_c(dest, data[i]);
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen break;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen }
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen }
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen}
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
11352dc3e4b29f3d2763c82f8ea4f99e8daf4fa3Timo Sirainenstatic void xml_encode(string_t *dest, const char *str)
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen{
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen xml_encode_data(dest, (const unsigned char *)str, strlen(str));
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen}
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen
942302b0247403645394d848b3c620ead262a2a5Timo Sirainenstatic struct fts_backend *
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainenfts_backend_solr_init(struct mailbox *box ATTR_UNUSED)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen{
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct fts_backend *backend;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (solr_conn == NULL)
cd8507179823de33d6e8242e10dbc15d136245b5Timo Sirainen solr_conn = solr_connection_init(getenv("FTS_SOLR"));
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen backend = i_new(struct fts_backend, 1);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen *backend = fts_backend_solr;
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen return backend;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen}
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic void fts_backend_solr_deinit(struct fts_backend *backend)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen{
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen i_free(backend);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen}
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainenstatic int fts_backend_solr_get_last_uid(struct fts_backend *backend,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen uint32_t *last_uid_r)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen{
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen struct mailbox_status status;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen ARRAY_TYPE(seq_range) uids;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen const struct seq_range *uidvals;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen unsigned int count;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen string_t *str;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen str = t_str_new(256);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen str_append(str, "fl=uid&rows=1&sort=uid%20desc&q=");
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen str_printfa(str, "uidv:%u%%20box:", status.uidvalidity);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen solr_quote_str(str, backend->box->name);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen str_append(str, "%20user:");
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen solr_quote_str(str, backend->box->storage->user);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen t_array_init(&uids, 1);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (solr_connection_select(solr_conn, str_c(str), &uids) < 0)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return -1;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uidvals = array_get(&uids, &count);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (count == 0) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* nothing indexed yet for this mailbox */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen *last_uid_r = 0;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen *last_uid_r = uidvals[0].seq1;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen } else {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen i_error("fts_solr: Last UID lookup returned multiple rows");
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen return -1;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return 0;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen}
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic int
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenfts_backend_solr_build_init(struct fts_backend *backend, uint32_t *last_uid_r,
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen struct fts_backend_build_context **ctx_r)
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen{
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen struct solr_fts_backend_build_context *ctx;
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen struct mailbox_status status;
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen *last_uid_r = (uint32_t)-1;
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen ctx = i_new(struct solr_fts_backend_build_context, 1);
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen ctx->ctx.backend = backend;
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen ctx->post = solr_connection_post_begin(solr_conn);
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen ctx->uid_validity = status.uidvalidity;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen *ctx_r = &ctx->ctx;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen return 0;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen}
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainenstatic int
11352dc3e4b29f3d2763c82f8ea4f99e8daf4fa3Timo Sirainenfts_backend_solr_build_more(struct fts_backend_build_context *_ctx,
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen uint32_t uid, const unsigned char *data,
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen size_t size, bool headers)
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen{
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen struct solr_fts_backend_build_context *ctx =
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen (struct solr_fts_backend_build_context *)_ctx;
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen struct mailbox *box = _ctx->backend->box;
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen string_t *cmd = ctx->cmd;
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen /* body comes first, then headers */
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen if (ctx->prev_uid != uid) {
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen /* uid changed */
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen if (ctx->prev_uid == 0)
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen str_append(cmd, "<add>");
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen else
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen str_append(cmd, "</field></doc>");
8305127d1074cf9a1e25dec9be2735276462079dTimo Sirainen ctx->prev_uid = uid;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen str_printfa(cmd, "<doc>"
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen "<field name=\"uid\">%u</field>"
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen "<field name=\"uidv\">%u</field>",
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen uid, ctx->uid_validity);
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen str_append(cmd, "<field name=\"box\">");
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen xml_encode(cmd, box->name);
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen str_append(cmd, "</field><field name=\"user\">");
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen xml_encode(cmd, box->storage->user);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen str_printfa(cmd, "</field><field name=\"id\">%u/%u/",
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uid, ctx->uid_validity);
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen xml_encode(cmd, box->storage->user);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen str_append_c(cmd, '/');
596433ccbca59ce2328dc1d029586154cd937155Timo Sirainen xml_encode(cmd, box->name);
596433ccbca59ce2328dc1d029586154cd937155Timo Sirainen str_append(cmd, "</field>");
4145cbac82bfc0c8bfeceeca0ef841700117930cTimo Sirainen
4145cbac82bfc0c8bfeceeca0ef841700117930cTimo Sirainen ctx->headers = headers;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (headers) {
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen str_append(cmd, "<field name=\"hdr\">");
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen } else {
e5fd6dfd0a492e4708d4dbb7971d7fc5d7b8fd85Timo Sirainen str_append(cmd, "<field name=\"body\">");
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen }
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen } else if (headers && !ctx->headers) {
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen str_append(cmd, "</field><field name=\"hdr\">");
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen } else {
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen i_assert(!(!headers && ctx->headers));
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen }
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen xml_encode_data(cmd, data, size);
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen if (str_len(cmd) > SOLR_CMDBUF_SIZE-128) {
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen solr_connection_post_more(ctx->post, str_data(cmd),
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen str_len(cmd));
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen str_truncate(cmd, 0);
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen }
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen return 0;
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen}
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic int
7f3b826a89bcb7a72759912e99f574b28309fe1bTimo Sirainenfts_backend_solr_build_deinit(struct fts_backend_build_context *_ctx)
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen{
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen struct solr_fts_backend_build_context *ctx =
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen (struct solr_fts_backend_build_context *)_ctx;
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen int ret = 0;
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen if (ctx->prev_uid != 0) {
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen str_append(ctx->cmd, "</field></doc></add>");
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen solr_connection_post_more(ctx->post, str_data(ctx->cmd),
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen str_len(ctx->cmd));
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen ret = solr_connection_post_end(ctx->post);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* commit and wait until the documents we just indexed are
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen visible to the following search */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (solr_connection_post(solr_conn,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen "<commit waitFlush=\"false\" "
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen "waitSearcher=\"true\"/>") < 0)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen ret = -1;
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen }
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen str_free(&ctx->cmd);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen i_free(ctx);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return ret;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen}
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic void
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenfts_backend_solr_expunge(struct fts_backend *backend ATTR_UNUSED,
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen struct mail *mail)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen{
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct mailbox_status status;
18ccd19c244f49665fe03cda785efa066d2c38dfTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mailbox_get_status(mail->box, STATUS_UIDVALIDITY, &status);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen T_BEGIN {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen string_t *cmd;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
d22301419109ed4a38351715e6760011421dadecTimo Sirainen cmd = t_str_new(256);
d22301419109ed4a38351715e6760011421dadecTimo Sirainen str_printfa(cmd, "<delete><id>%u/%u/",
d22301419109ed4a38351715e6760011421dadecTimo Sirainen mail->uid, status.uidvalidity);
d22301419109ed4a38351715e6760011421dadecTimo Sirainen xml_encode(cmd, mail->box->storage->user);
d22301419109ed4a38351715e6760011421dadecTimo Sirainen str_append_c(cmd, '/');
499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen xml_encode(cmd, mail->box->name);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen str_append(cmd, "</id></delete>");
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen (void)solr_connection_post(solr_conn, str_c(cmd));
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen } T_END;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen}
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
d22301419109ed4a38351715e6760011421dadecTimo Sirainenstatic void
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenfts_backend_solr_expunge_finish(struct fts_backend *backend ATTR_UNUSED,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct mailbox *box ATTR_UNUSED,
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen bool committed ATTR_UNUSED)
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen{
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen solr_connection_post(solr_conn,
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen "<commit waitFlush=\"false\" waitSearcher=\"false\"/>");
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen}
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainenstatic int fts_backend_solr_lock(struct fts_backend *backend ATTR_UNUSED)
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen{
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen return 1;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen}
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainenstatic void fts_backend_solr_unlock(struct fts_backend *backend ATTR_UNUSED)
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen{
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen}
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainenstatic int
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainenfts_backend_solr_lookup(struct fts_backend *backend, const char *key,
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen enum fts_lookup_flags flags,
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ARRAY_TYPE(seq_range) *definite_uids,
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ARRAY_TYPE(seq_range) *maybe_uids)
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen{
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen struct mailbox_status status;
c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen string_t *str;
c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen
c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen i_assert((flags & FTS_LOOKUP_FLAG_INVERT) == 0);
047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen
c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen str = t_str_new(256);
047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen str_append(str, "fl=uid&q=");
047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen if ((flags & FTS_LOOKUP_FLAG_HEADER) == 0) {
047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen /* body only */
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen i_assert((flags & FTS_LOOKUP_FLAG_BODY) != 0);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen str_append(str, "body:");
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen } else if ((flags & FTS_LOOKUP_FLAG_BODY) == 0) {
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen /* header only */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen str_append(str, "hdr:");
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen } else {
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen /* both */
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen str_append(str, "any:");
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen }
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen solr_quote_str(str, key);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen str_printfa(str, "%%20uidv:%u%%20box:", status.uidvalidity);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen solr_quote_str(str, backend->box->name);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen str_append(str, "%20user:");
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen solr_quote_str(str, backend->box->storage->user);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen array_clear(maybe_uids);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen return solr_connection_select(solr_conn, str_c(str), definite_uids);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen}
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainenstruct fts_backend fts_backend_solr = {
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen MEMBER(name) "solr",
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen MEMBER(flags) 0,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_backend_solr_init,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_backend_solr_deinit,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_backend_solr_get_last_uid,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_backend_solr_build_init,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_backend_solr_build_more,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen fts_backend_solr_build_deinit,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen fts_backend_solr_expunge,
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen fts_backend_solr_expunge_finish,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen fts_backend_solr_lock,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen fts_backend_solr_unlock,
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen fts_backend_solr_lookup,
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen NULL
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen }
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen};
18ccd19c244f49665fe03cda785efa066d2c38dfTimo Sirainen