src/indexer/indexer.c

bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "lib.h"
e4109fe523db23973c6fbfccf370b5dd3d754372Timo Sirainen#include "ioloop.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "restrict-access.h"
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen#include "process-title.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "master-service.h"
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen#include "master-service-settings.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "indexer-client.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "indexer-queue.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "worker-pool.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen#include "worker-connection.h"
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstruct worker_request {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    struct worker_connection *conn;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    struct indexer_request *request;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen};
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainenstatic const struct master_service_settings *set;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstatic struct indexer_queue *queue;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstatic struct worker_pool *worker_pool;
e4109fe523db23973c6fbfccf370b5dd3d754372Timo Sirainenstatic struct timeout *to_send_more;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainenvoid indexer_refresh_proctitle(void)
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen{
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen    if (!set->verbose_proctitle)
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen        return;
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen    process_title_set(t_strdup_printf("[%u clients, %u requests]",
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen                      indexer_clients_get_count(),
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen                      indexer_queue_count(queue)));
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen}
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstatic bool idle_die(void)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
df26373d0aa3cb208da213fce32e2abc5d97f90bTimo Sirainen    return indexer_queue_is_empty(queue) &&
df26373d0aa3cb208da213fce32e2abc5d97f90bTimo Sirainen        !worker_pool_have_busy_connections(worker_pool);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstatic void client_connected(struct master_service_connection *conn)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    master_service_client_connection_accept(conn);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    (void)indexer_client_create(conn->fd, queue);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstatic void worker_send_request(struct worker_connection *conn,
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen                struct indexer_request *request)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    struct worker_request *wrequest;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    wrequest = i_new(struct worker_request, 1);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    wrequest->conn = conn;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    wrequest->request = request;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
218095792e0af7c1609da4901cfb60e78139595eTimo Sirainen    indexer_queue_request_work(request);
294f579cd3803e2d9997231fdc46523c23774a8fTimo Sirainen    worker_connection_request(conn, request, wrequest);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
be98c0c6da8ee8c00ed731bcb833f491daf76757Timo Sirainenstatic void queue_try_send_more(struct indexer_queue *queue)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    struct worker_connection *conn;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    struct indexer_request *request;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
0d1b8b6bec79746c5d89d57dd8c1688946bd9237Josef 'Jeff' Sipek    timeout_remove(&to_send_more);
e4109fe523db23973c6fbfccf370b5dd3d754372Timo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    while ((request = indexer_queue_request_peek(queue)) != NULL) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen        conn = worker_pool_find_username_connection(worker_pool,
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen                                request->username);
be98c0c6da8ee8c00ed731bcb833f491daf76757Timo Sirainen        if (conn != NULL) {
be98c0c6da8ee8c00ed731bcb833f491daf76757Timo Sirainen            /* there is already a worker handling this user.
be98c0c6da8ee8c00ed731bcb833f491daf76757Timo Sirainen               it must be the one doing the indexing. use the same
be98c0c6da8ee8c00ed731bcb833f491daf76757Timo Sirainen               connection for sending this next request. */
be98c0c6da8ee8c00ed731bcb833f491daf76757Timo Sirainen        } else {
be98c0c6da8ee8c00ed731bcb833f491daf76757Timo Sirainen            /* try to find an empty worker */
be98c0c6da8ee8c00ed731bcb833f491daf76757Timo Sirainen            if (!worker_pool_get_connection(worker_pool, &conn))
be98c0c6da8ee8c00ed731bcb833f491daf76757Timo Sirainen                break;
be98c0c6da8ee8c00ed731bcb833f491daf76757Timo Sirainen        }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen        indexer_queue_request_remove(queue);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen        worker_send_request(conn, request);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstatic void queue_listen_callback(struct indexer_queue *queue)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    queue_try_send_more(queue);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenstatic void worker_status_callback(int percentage, void *context)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    struct worker_request *request = context;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    if (percentage >= 0 && percentage < 100) {
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen        indexer_queue_request_status(queue, request->request,
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen                         percentage);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen        return;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    }
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    indexer_queue_request_finish(queue, &request->request,
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen                     percentage == 100);
947e4e68bbbfc584368b0e4febedbcc338650b88Timo Sirainen    if (worker_pool != NULL) /* not in deinit */
947e4e68bbbfc584368b0e4febedbcc338650b88Timo Sirainen        worker_pool_release_connection(worker_pool, request->conn);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    i_free(request);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    /* if this was the last request for the connection, we can send more
e4109fe523db23973c6fbfccf370b5dd3d754372Timo Sirainen       through it. delay it a bit, since we may be coming here from
e4109fe523db23973c6fbfccf370b5dd3d754372Timo Sirainen       worker_connection_disconnect() and we want to finish it up. */
e4109fe523db23973c6fbfccf370b5dd3d754372Timo Sirainen    if (to_send_more == NULL)
e4109fe523db23973c6fbfccf370b5dd3d754372Timo Sirainen        to_send_more = timeout_add_short(0, queue_try_send_more, queue);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainenint main(int argc, char *argv[])
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen{
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen    const char *error;
cfa8a04466cd46bdc422bd9eba8e6794758d677bTimo Sirainen
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen    master_service = master_service_init("indexer", 0, &argc, &argv, "");
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    if (master_getopt(master_service) > 0)
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen        return FATAL_DEFAULT;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
fe9fb910df7f6932b204184400c6321e5dee5b13Timo Sirainen    if (master_service_settings_read_simple(master_service, NULL,
fe9fb910df7f6932b204184400c6321e5dee5b13Timo Sirainen                        &error) < 0)
fe9fb910df7f6932b204184400c6321e5dee5b13Timo Sirainen        i_fatal("Error reading configuration: %s", error);
fe9fb910df7f6932b204184400c6321e5dee5b13Timo Sirainen    set = master_service_settings_get(master_service);
fe9fb910df7f6932b204184400c6321e5dee5b13Timo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    master_service_init_log(master_service, "indexer: ");
816d20be0cf95fc4eb1a8aa716639e73b8ba525eMartti Rannanjärvi    restrict_access_by_env(RESTRICT_ACCESS_FLAG_ALLOW_ROOT, NULL);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    restrict_access_allow_coredumps(TRUE);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    master_service_set_idle_die_callback(master_service, idle_die);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    queue = indexer_queue_init(indexer_client_status_callback);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    indexer_queue_set_listen_callback(queue, queue_listen_callback);
947e4e68bbbfc584368b0e4febedbcc338650b88Timo Sirainen    worker_pool = worker_pool_init("indexer-worker",
947e4e68bbbfc584368b0e4febedbcc338650b88Timo Sirainen                       worker_status_callback);
d6b3cfd855c0eebed68be50d3111de1b5a6afeb0Timo Sirainen    master_service_init_finish(master_service);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    master_service_run(master_service, client_connected);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    indexer_queue_cancel_all(queue);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    indexer_clients_destroy_all();
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    worker_pool_deinit(&worker_pool);
947e4e68bbbfc584368b0e4febedbcc338650b88Timo Sirainen    indexer_queue_deinit(&queue);
0d1b8b6bec79746c5d89d57dd8c1688946bd9237Josef 'Jeff' Sipek    timeout_remove(&to_send_more);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen    master_service_deinit(&master_service);
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen        return 0;
d9e404180ff26dbbaea68534a5f176765022b76bTimo Sirainen}