service-monitor.c revision 17706107e6efc2f15973a7a63a834cb7c0a6dc68
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2005-2012 Dovecot authors, see the included COPYING file */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "common.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "array.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "ioloop.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "fd-close-on-exec.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "hash.h"
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen#include "str.h"
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen#include "safe-mkstemp.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "service.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "service-process.h"
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen#include "service-process-notify.h"
fb08a91e3f2949ecefb647fa38206ca9aad5307fTimo Sirainen#include "service-anvil.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "service-log.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "service-monitor.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen#include <stdlib.h>
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include <unistd.h>
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include <sys/wait.h>
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include <syslog.h>
d0720f3037064af4b92eccfc20a8814adcacf827Timo Sirainen#include <signal.h>
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen#define SERVICE_DROP_WARN_INTERVAL_SECS 60
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen#define SERVICE_DROP_TIMEOUT_MSECS (10*1000)
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen#define MAX_DIE_WAIT_SECS 5
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen#define SERVICE_MAX_EXIT_FAILURES_IN_SEC 10
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainenstatic void service_monitor_start_extra_avail(struct service *service);
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainenstatic void service_status_more(struct service_process *process,
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen const struct master_status *status);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainenstatic void service_monitor_listen_start_force(struct service *service);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainenstatic void service_process_kill_idle(struct service_process *process)
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen{
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen struct service *service = process->service;
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen struct master_status status;
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen i_assert(process->available_count == service->client_limit);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
c0e5c6a86e1de5d4f5591d39b4aa921a23c807d7Timo Sirainen if (service->process_avail <= service->set->process_min_avail) {
c0e5c6a86e1de5d4f5591d39b4aa921a23c807d7Timo Sirainen /* we don't have any extra idling processes anymore. */
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen timeout_remove(&process->to_idle);
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen } else if (process->last_kill_sent > process->last_status_update+1) {
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen service_error(service, "Process %s is ignoring idle SIGINT",
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen dec2str(process->pid));
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen /* assume this process is busy */
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen memset(&status, 0, sizeof(status));
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen service_status_more(process, &status);
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen process->available_count = 0;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen } else {
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen if (kill(process->pid, SIGINT) < 0 && errno != ESRCH) {
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen service_error(service, "kill(%s, SIGINT) failed: %m",
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen dec2str(process->pid));
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen }
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen process->last_kill_sent = ioloop_time;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen }
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen}
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainenstatic void service_status_more(struct service_process *process,
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen const struct master_status *status)
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen{
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen struct service *service = process->service;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen process->total_count +=
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen process->available_count - status->available_count;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen process->idle_start = 0;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen if (process->to_idle != NULL)
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen timeout_remove(&process->to_idle);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen if (status->available_count != 0)
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen return;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen /* process used up all of its clients */
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen i_assert(service->process_avail > 0);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen service->process_avail--;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (service->type == SERVICE_TYPE_LOGIN &&
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen service->process_avail == 0 &&
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen service->process_count == service->process_limit)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen service_login_notify(service, TRUE);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen /* we may need to start more */
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen service_monitor_start_extra_avail(service);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen service_monitor_listen_start(service);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen}
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainenstatic void service_status_less(struct service_process *process,
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen const struct master_status *status)
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen{
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen struct service *service = process->service;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen if (process->available_count == 0) {
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen /* process can accept more clients again */
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen if (service->process_avail++ == 0)
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen service_monitor_listen_stop(service);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen i_assert(service->process_avail <= service->process_count);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen }
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen if (status->available_count == service->client_limit) {
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen process->idle_start = ioloop_time;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen if (service->process_avail > service->set->process_min_avail &&
c0e5c6a86e1de5d4f5591d39b4aa921a23c807d7Timo Sirainen process->to_idle == NULL &&
93a7d1ee4b518b5c85f9721dc6539e4dab6aae00Timo Sirainen service->idle_kill != -1U) {
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen /* we have more processes than we really need.
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen add a bit of randomness so that we don't send the
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen signal to all of them at once */
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen process->to_idle =
c05d0937e228c2817fa2295fc53e8cb81ae5cb8aTimo Sirainen timeout_add((service->idle_kill * 1000) +
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen (rand() % 100)*10,
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen service_process_kill_idle,
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen process);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen }
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen }
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (service->type == SERVICE_TYPE_LOGIN)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen service_login_notify(service, FALSE);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen}
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainenstatic void
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainenservice_status_input_one(struct service *service,
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen const struct master_status *status)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct service_process *process;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen process = hash_table_lookup(service_pids, &status->pid);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (process == NULL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* we've probably wait()ed it away already. ignore */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen if (process->uid != status->uid || process->service != service) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* a) Process was closed and another process was created with
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen the same PID, but we're still receiving status update from
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen the old process.
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen b) Some process is trying to corrupt our internal state by
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen trying to pretend to be someone else. We could use stronger
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen randomness here, but the worst they can do is DoS and there
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen are already more serious problems if someone is able to do
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen this.. */
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen service_error(service, "Ignoring invalid update from child %s "
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen "(UID=%u)", dec2str(status->pid), status->uid);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen process->last_status_update = ioloop_time;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (process->to_status != NULL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* first status notification */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen timeout_remove(&process->to_status);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen if (process->available_count == status->available_count)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen if (process->available_count > status->available_count) {
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen /* process started servicing some more clients */
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen service_status_more(process, status);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen } else {
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen /* process finished servicing some clients */
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen service_status_less(process, status);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen process->available_count = status->available_count;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen}
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainenstatic void service_status_input(struct service *service)
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen{
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen struct master_status status[1024/sizeof(struct master_status)];
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen unsigned int i, count;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen ssize_t ret;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen ret = read(service->status_fd[0], &status, sizeof(status));
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen if (ret <= 0) {
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen if (ret == 0)
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen service_error(service, "read(status) failed: EOF");
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen else if (errno != EAGAIN)
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen service_error(service, "read(status) failed: %m");
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen else
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen return;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen service_monitor_stop(service);
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen return;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen }
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen if ((ret % sizeof(struct master_status)) != 0) {
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen service_error(service, "service sent partial status update "
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen "(%d bytes)", (int)ret);
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen return;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen }
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen count = ret / sizeof(struct master_status);
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen for (i = 0; i < count; i++)
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen service_status_input_one(service, &status[i]);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic void service_monitor_throttle(struct service *service)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (service->to_throttle != NULL)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
cb211cc64c1c6ac8343d60e7a058be42fdba2f71Timo Sirainen i_assert(service->throttle_secs > 0);
cb211cc64c1c6ac8343d60e7a058be42fdba2f71Timo Sirainen
cb211cc64c1c6ac8343d60e7a058be42fdba2f71Timo Sirainen service_error(service, "command startup failed, throttling for %u secs",
cb211cc64c1c6ac8343d60e7a058be42fdba2f71Timo Sirainen service->throttle_secs);
cb211cc64c1c6ac8343d60e7a058be42fdba2f71Timo Sirainen service_throttle(service, service->throttle_secs);
cb211cc64c1c6ac8343d60e7a058be42fdba2f71Timo Sirainen service->throttle_secs *= 2;
cb211cc64c1c6ac8343d60e7a058be42fdba2f71Timo Sirainen if (service->throttle_secs > SERVICE_STARTUP_FAILURE_THROTTLE_MAX_SECS)
cb211cc64c1c6ac8343d60e7a058be42fdba2f71Timo Sirainen service->throttle_secs = SERVICE_STARTUP_FAILURE_THROTTLE_MAX_SECS;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainenstatic void service_drop_timeout(struct service *service)
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen{
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen struct service_listener *const *lp;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen int fd;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen i_assert(service->process_avail == 0);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen /* drop all pending connections */
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen array_foreach(&service->listeners, lp) {
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen while ((fd = net_accept((*lp)->fd, NULL, NULL)) > 0)
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen net_disconnect(fd);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen }
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen service_monitor_listen_start_force(service);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen service->listen_pending = TRUE;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen}
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainenstatic void service_monitor_listen_pending(struct service *service)
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen{
2f6a8ef44beaee4ef74adfcda455ce426c6e7c45Timo Sirainen i_assert(service->process_avail == 0);
2f6a8ef44beaee4ef74adfcda455ce426c6e7c45Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen service_monitor_listen_stop(service);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen service->listen_pending = TRUE;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen service->to_drop = timeout_add(SERVICE_DROP_TIMEOUT_MSECS,
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen service_drop_timeout, service);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen}
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainenstatic void service_drop_connections(struct service_listener *l)
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen{
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen struct service *service = l->service;
3a79fdaf3253dae045dfa14d2a88b94086327da4Timo Sirainen unsigned int limit;
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen int fd;
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen if (service->last_drop_warning +
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen SERVICE_DROP_WARN_INTERVAL_SECS < ioloop_time) {
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen service->last_drop_warning = ioloop_time;
3a79fdaf3253dae045dfa14d2a88b94086327da4Timo Sirainen limit = service->process_limit > 1 ?
3a79fdaf3253dae045dfa14d2a88b94086327da4Timo Sirainen service->process_limit : service->client_limit;
3a79fdaf3253dae045dfa14d2a88b94086327da4Timo Sirainen i_warning("service(%s): %s (%u) reached, "
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen "client connections are being dropped",
699b27536eeb446e05ed9c9c1ab35cc6174ac0aaTimo Sirainen service->set->name,
dadd250347d90d90bd65f529d45b50f79d6a0674Timo Sirainen service->process_limit > 1 ?
3a79fdaf3253dae045dfa14d2a88b94086327da4Timo Sirainen "process_limit" : "client_limit", limit);
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen }
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (service->type == SERVICE_TYPE_LOGIN) {
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen /* reached process limit, notify processes that they
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen need to start killing existing connections if they
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen reach connection limit */
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen service_login_notify(service, TRUE);
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen service_monitor_listen_pending(service);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen } else if (!service->listen_pending) {
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen /* maybe this is a temporary peak, stop for a while and
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen see if it goes away */
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen service_monitor_listen_pending(service);
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen } else {
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen /* this has been happening for a while now. just accept and
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen close the connection, so it's clear that this is happening
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen because of the limit, rather than because the service
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen processes aren't answering fast enough */
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen fd = net_accept(l->fd, NULL, NULL);
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen if (fd > 0)
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen net_disconnect(fd);
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen }
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen}
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainenstatic void service_accept(struct service_listener *l)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen struct service *service = l->service;
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(service->process_avail == 0);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (service->process_count == service->process_limit) {
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen /* we've reached our limits, new clients will have to
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen wait until there are more processes available */
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen service_drop_connections(l);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* create a child process and let it accept() this connection */
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (service_process_create(service) == NULL)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen service_monitor_throttle(service);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen else
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen service_monitor_listen_stop(service);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainenstatic void service_monitor_start_extra_avail(struct service *service)
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen{
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen unsigned int i, count;
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen
2806f15ceb68023baf65a9daad9dfdf54c622708Timo Sirainen if (service->process_avail >= service->set->process_min_avail ||
2806f15ceb68023baf65a9daad9dfdf54c622708Timo Sirainen service->list->destroying)
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen return;
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen count = service->set->process_min_avail - service->process_avail;
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen if (service->process_count + count > service->process_limit)
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen count = service->process_limit - service->process_count;
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen for (i = 0; i < count; i++) {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (service_process_create(service) == NULL) {
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen service_monitor_throttle(service);
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen break;
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen }
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen }
2f6a8ef44beaee4ef74adfcda455ce426c6e7c45Timo Sirainen if (i > 0) {
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen /* we created some processes, they'll do the listening now */
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen service_monitor_listen_stop(service);
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen }
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen}
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainenstatic void service_monitor_listen_start_force(struct service *service)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct service_listener *const *listeners;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen service->listening = TRUE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen service->listen_pending = FALSE;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen if (service->to_drop != NULL)
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen timeout_remove(&service->to_drop);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen array_foreach(&service->listeners, listeners) {
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen struct service_listener *l = *listeners;
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen if (l->io == NULL && l->fd != -1)
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen l->io = io_add(l->fd, IO_READ, service_accept, l);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainenvoid service_monitor_listen_start(struct service *service)
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen{
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen if (service->process_avail > 0 ||
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen (service->process_count == service->process_limit &&
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen service->listen_pending))
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen return;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen service_monitor_listen_start_force(service);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen}
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainenvoid service_monitor_listen_stop(struct service *service)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct service_listener *const *listeners;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen array_foreach(&service->listeners, listeners) {
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen struct service_listener *l = *listeners;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (l->io != NULL)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen io_remove(&l->io);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen service->listening = FALSE;
605cce7e45c80fa8a07a10155c353ca541a3041fTimo Sirainen service->listen_pending = FALSE;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen if (service->to_drop != NULL)
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen timeout_remove(&service->to_drop);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenstatic int service_login_create_notify_fd(struct service *service)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen{
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen int fd;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (service->login_notify_fd != -1)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen return 0;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen T_BEGIN {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen string_t *prefix = t_str_new(128);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen const char *path;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
3005627bf2ed223194c2d08a8c1630769d048f69Timo Sirainen str_append(prefix, service->set->master_set->base_dir);
3005627bf2ed223194c2d08a8c1630769d048f69Timo Sirainen str_append(prefix, "/login-master-notify");
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen fd = safe_mkstemp(prefix, 0600, (uid_t)-1, (gid_t)-1);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen path = str_c(prefix);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (fd == -1) {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen service_error(service, "safe_mkstemp(%s) failed: %m",
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen path);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen } else if (unlink(path) < 0) {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen service_error(service, "unlink(%s) failed: %m", path);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen } else {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen fd_close_on_exec(fd, TRUE);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen service->login_notify_fd = fd;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen }
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen } T_END;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (fd != service->login_notify_fd)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen (void)close(fd);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen return fd == -1 ? -1 : 0;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen}
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenvoid services_monitor_start(struct service_list *service_list)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct service *const *services;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
17706107e6efc2f15973a7a63a834cb7c0a6dc68Timo Sirainen if (services_log_init(service_list) < 0)
17706107e6efc2f15973a7a63a834cb7c0a6dc68Timo Sirainen return;
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen service_anvil_monitor_start(service_list);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen if (pipe(service_list->master_dead_pipe_fd) < 0)
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen i_error("pipe() failed: %m");
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen fd_close_on_exec(service_list->master_dead_pipe_fd[0], TRUE);
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen fd_close_on_exec(service_list->master_dead_pipe_fd[1], TRUE);
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen array_foreach(&service_list->services, services) {
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen struct service *service = *services;
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen if (service->type == SERVICE_TYPE_LOGIN) {
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen if (service_login_create_notify_fd(service) < 0)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen continue;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen }
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen if (service->status_fd[0] == -1) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* we haven't yet created status pipe */
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen if (pipe(service->status_fd) < 0) {
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen service_error(service, "pipe() failed: %m");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen continue;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen net_set_nonblock(service->status_fd[0], TRUE);
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen fd_close_on_exec(service->status_fd[0], TRUE);
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen net_set_nonblock(service->status_fd[1], TRUE);
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen fd_close_on_exec(service->status_fd[1], TRUE);
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen }
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen if (service->io_status == NULL) {
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen service->io_status =
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen io_add(service->status_fd[0], IO_READ,
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen service_status_input, service);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen service_monitor_start_extra_avail(service);
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen service_monitor_listen_start(service);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
17706107e6efc2f15973a7a63a834cb7c0a6dc68Timo Sirainen if (service_list->log->status_fd[0] != -1) {
17706107e6efc2f15973a7a63a834cb7c0a6dc68Timo Sirainen if (service_process_create(service_list->log) != NULL)
17706107e6efc2f15973a7a63a834cb7c0a6dc68Timo Sirainen service_monitor_listen_stop(service_list->log);
17706107e6efc2f15973a7a63a834cb7c0a6dc68Timo Sirainen }
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen /* start up a process for startup-services */
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen array_foreach(&service_list->services, services) {
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen struct service *service = *services;
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen
17706107e6efc2f15973a7a63a834cb7c0a6dc68Timo Sirainen if (service->type == SERVICE_TYPE_STARTUP &&
17706107e6efc2f15973a7a63a834cb7c0a6dc68Timo Sirainen service->status_fd[0] != -1) {
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen if (service_process_create(service) != NULL)
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen service_monitor_listen_stop(service);
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen }
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainenvoid service_monitor_stop(struct service *service)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen int i;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (service->io_status != NULL)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen io_remove(&service->io_status);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen if (service->status_fd[0] != -1 &&
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen service->type != SERVICE_TYPE_ANVIL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen for (i = 0; i < 2; i++) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (close(service->status_fd[i]) < 0) {
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen service_error(service,
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen "close(status fd) failed: %m");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen service->status_fd[i] = -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (service->login_notify_fd != -1) {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (close(service->login_notify_fd) < 0) {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen service_error(service,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen "close(login notify fd) failed: %m");
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen }
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen service->login_notify_fd = -1;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen }
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (service->to_login_notify != NULL)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen timeout_remove(&service->to_login_notify);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen service_monitor_listen_stop(service);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (service->to_throttle != NULL)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen timeout_remove(&service->to_throttle);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainenstatic void services_monitor_wait(struct service_list *service_list)
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen{
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen struct service *const *servicep;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen time_t max_wait_time = time(NULL) + MAX_DIE_WAIT_SECS;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen bool finished;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen for (;;) {
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen finished = TRUE;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen services_monitor_reap_children();
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen array_foreach(&service_list->services, servicep) {
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen if ((*servicep)->status_fd[0] != -1)
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen service_status_input(*servicep);
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen if ((*servicep)->process_avail > 0)
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen finished = FALSE;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen }
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen if (finished || time(NULL) > max_wait_time)
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen break;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen usleep(100000);
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen }
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen}
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainenvoid services_monitor_stop(struct service_list *service_list, bool wait)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct service *const *services;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen if (service_list->master_dead_pipe_fd[0] != -1) {
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen if (close(service_list->master_dead_pipe_fd[0]) < 0)
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen i_error("close(master dead pipe) failed: %m");
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen if (close(service_list->master_dead_pipe_fd[1]) < 0)
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen i_error("close(master dead pipe) failed: %m");
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen service_list->master_dead_pipe_fd[0] = -1;
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen service_list->master_dead_pipe_fd[1] = -1;
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen }
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen if (wait) {
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen /* we've notified all children that the master is dead.
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen now wait for the children to either die or to tell that
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen they're no longer listening for new connections */
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen services_monitor_wait(service_list);
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen }
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen array_foreach(&service_list->services, services)
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen service_monitor_stop(*services);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen services_log_deinit(service_list);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainenstatic bool
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainenservice_process_failure(struct service_process *process, int status)
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen{
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen struct service *service = process->service;
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen bool throttle;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen service_process_log_status_error(process, status);
904324b95569604138d24f2dc951f9fb3cc246dcTimo Sirainen throttle = process->to_status != NULL;
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen if (!throttle && !service->have_successful_exits) {
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen /* this service has seen no successful exits yet.
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen try to avoid failure storms by throttling the service if it
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen only keeps failing rapidly. this is no longer done after
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen one success to avoid intentional DoSing, in case attacker
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen finds a way to quickly crash his own session. */
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen if (service->exit_failure_last != ioloop_time) {
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen service->exit_failure_last = ioloop_time;
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen service->exit_failures_in_sec = 0;
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen }
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen if (++service->exit_failures_in_sec > SERVICE_MAX_EXIT_FAILURES_IN_SEC)
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen throttle = TRUE;
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen }
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen service_process_notify_add(service_anvil_global->kills, process);
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen return throttle;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen}
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainenvoid services_monitor_reap_children(void)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct service_process *process;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct service *service;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen pid_t pid;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen int status;
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen bool service_stopped, throttle;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen while ((pid = waitpid(-1, &status, WNOHANG)) > 0) {
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen process = hash_table_lookup(service_pids, &pid);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (process == NULL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_error("waitpid() returned unknown PID %s",
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen dec2str(pid));
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen continue;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen service = process->service;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (status == 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* success */
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen if (service->listen_pending &&
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen !service->list->destroying)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen service_monitor_listen_start(service);
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen /* one success resets all failures */
321f17803ad71171ad2408399b6cc8efd2d1479aTimo Sirainen service->have_successful_exits = TRUE;
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen service->exit_failures_in_sec = 0;
cb211cc64c1c6ac8343d60e7a058be42fdba2f71Timo Sirainen service->throttle_secs =
cb211cc64c1c6ac8343d60e7a058be42fdba2f71Timo Sirainen SERVICE_STARTUP_FAILURE_THROTTLE_MIN_SECS;
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen throttle = FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen } else {
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen throttle = service_process_failure(process, status);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen if (service->type == SERVICE_TYPE_ANVIL)
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen service_anvil_process_destroyed(process);
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen /* if we're reloading, we may get here with a service list
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen that's going to be destroyed after this process is
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen destroyed. keep the list referenced until we're done. */
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen service_list_ref(service->list);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen service_process_destroy(process);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen if (throttle)
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen service_monitor_throttle(service);
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen service_stopped = service->status_fd[0] == -1;
601b455f4d5e780044b9e4fac5f687c1b07ae145Timo Sirainen if (!service_stopped) {
2806f15ceb68023baf65a9daad9dfdf54c622708Timo Sirainen service_monitor_start_extra_avail(service);
3b959c98e05e780de2a063a4a9d8d393dc61ed58Timo Sirainen if (service->to_throttle == NULL)
3b959c98e05e780de2a063a4a9d8d393dc61ed58Timo Sirainen service_monitor_listen_start(service);
3b959c98e05e780de2a063a4a9d8d393dc61ed58Timo Sirainen }
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen service_list_unref(service->list);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}