master-connection.c revision ecf44c74416ffa4e7c331e49a1e283be6b1aa668
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen/* Copyright (c) 2011 Dovecot authors, see the included COPYING file */
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "lib.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "ioloop.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "istream.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "write-full.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "strescape.h"
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen#include "process-title.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "master-service.h"
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen#include "master-service-settings.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "mail-namespace.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "mail-storage.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "mail-storage-service.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "master-connection.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include <unistd.h>
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#define INDEXER_PROTOCOL_MAJOR_VERSION 1
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#define INDEXER_PROTOCOL_MINOR_VERSION 0
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#define INDEXER_WORKER_HANDSHAKE "VERSION\tindexer-worker-master\t1\t0\n%u\n"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#define INDEXER_MASTER_NAME "indexer-master-worker"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstruct master_connection {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct mail_storage_service_ctx *storage_service;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen int fd;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct io *io;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct istream *input;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct ostream *output;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen unsigned int version_received:1;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen};
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainenstatic void
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainenindexer_worker_refresh_proctitle(const char *username, const char *mailbox)
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen{
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen if (!master_service_settings_get(master_service)->verbose_proctitle)
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen return;
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen if (username != NULL)
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen process_title_set(t_strdup_printf("[%s %s]", username, mailbox));
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen else
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen process_title_set("[idling]");
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen}
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainenstatic int index_mailbox(struct mail_user *user, const char *mailbox,
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen unsigned int max_recent_msgs)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct mail_namespace *ns;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct mailbox *box;
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen struct mailbox_status status;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen const char *errstr;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen enum mail_error error;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen int ret = 0;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen ns = mail_namespace_find(user->namespaces, mailbox);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen if (ns == NULL) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen i_error("Namespace not found for mailbox %s: ", mailbox);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen return -1;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen /* FIXME: the current lib-storage API doesn't allow sending
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen "n% competed" notifications */
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen box = mailbox_alloc(ns->list, mailbox, MAILBOX_FLAG_KEEP_RECENT);
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen if (max_recent_msgs != 0) {
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen /* index only if there aren't too many recent messages.
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen don't bother syncing the mailbox, that alone can take a
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen while with large maildirs. */
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen if (mailbox_open(box) < 0) {
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen i_error("Opening mailbox %s failed: %s", mailbox,
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen mail_storage_get_last_error(mailbox_get_storage(box), NULL));
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen ret = -1;
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen } else {
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen mailbox_get_open_status(box, STATUS_RECENT, &status);
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen }
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen if (ret < 0 || status.recent > max_recent_msgs) {
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen mailbox_free(&box);
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen return ret;
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen }
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ |
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen MAILBOX_SYNC_FLAG_PRECACHE) < 0) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen errstr = mail_storage_get_last_error(mailbox_get_storage(box),
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen &error);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen if (error != MAIL_ERROR_NOTFOUND) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen i_error("Syncing mailbox %s failed: %s",
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen mailbox, errstr);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen } else if (user->mail_debug) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen i_debug("Syncing mailbox %s failed: %s",
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen mailbox, errstr);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen ret = -1;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen mailbox_free(&box);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen return ret;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstatic int
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenmaster_connection_input_line(struct master_connection *conn, const char *line)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen const char *const *args = t_strsplit_tabescaped(line);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct mail_storage_service_input input;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct mail_storage_service_user *service_user;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct mail_user *user;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen const char *str, *error;
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen unsigned int max_recent_msgs;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen int ret;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen /* <username> <mailbox> <max_recent_msgs> */
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen if (str_array_length(args) != 3 ||
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen str_to_uint(args[2], &max_recent_msgs) < 0) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen i_error("Invalid input from master: %s", line);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen return -1;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen memset(&input, 0, sizeof(input));
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen input.module = "mail";
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen input.service = "indexer-worker";
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen input.username = args[0];
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen if (mail_storage_service_lookup_next(conn->storage_service, &input,
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen &service_user, &user, &error) <= 0) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen i_error("User %s lookup failed: %s", args[0], error);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen ret = -1;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen } else {
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen indexer_worker_refresh_proctitle(user->username, args[1]);
7c0e7d96e104a59df7b5aecdc0cbe4f6e304b7c7Timo Sirainen ret = index_mailbox(user, args[1], max_recent_msgs);
0be9d7c58e51c21eb521501e212a8e8ff7593769Timo Sirainen indexer_worker_refresh_proctitle(NULL, NULL);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen mail_user_unref(&user);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen mail_storage_service_user_free(&service_user);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen str = ret < 0 ? "-1\n" : "100\n";
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen return write_full(conn->fd, str, strlen(str));
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstatic void master_connection_input(struct master_connection *conn)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen const char *line;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen int ret;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen if (i_stream_read(conn->input) < 0) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen master_service_stop(master_service);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen return;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen if (!conn->version_received) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen if ((line = i_stream_next_line(conn->input)) == NULL)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen return;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen if (!version_string_verify(line, INDEXER_MASTER_NAME,
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen INDEXER_PROTOCOL_MAJOR_VERSION)) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen i_error("Indexer master not compatible with this master "
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen "(mixed old and new binaries?)");
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen master_service_stop(master_service);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen return;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen conn->version_received = TRUE;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen while ((line = i_stream_next_line(conn->input)) != NULL) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen T_BEGIN {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen ret = master_connection_input_line(conn, line);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen } T_END;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen if (ret < 0) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen master_service_stop(master_service);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen break;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstruct master_connection *
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenmaster_connection_create(int fd, struct mail_storage_service_ctx *storage_service)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct master_connection *conn;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen const char *handshake;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen conn = i_new(struct master_connection, 1);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen conn->storage_service = storage_service;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen conn->fd = fd;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen conn->io = io_add(conn->fd, IO_READ, master_connection_input, conn);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen conn->input = i_stream_create_fd(conn->fd, (size_t)-1, FALSE);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen handshake = t_strdup_printf(INDEXER_WORKER_HANDSHAKE,
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen master_service_get_process_limit(master_service));
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen (void)write_full(conn->fd, handshake, strlen(handshake));
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen return conn;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenvoid master_connection_destroy(struct master_connection **_conn)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen struct master_connection *conn = *_conn;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen *_conn = NULL;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen io_remove(&conn->io);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen i_stream_destroy(&conn->input);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen if (close(conn->fd) < 0)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen i_error("close(master conn) failed: %m");
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen i_free(conn);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen master_service_client_connection_destroyed(master_service);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}