director.c revision 6f8d511666b292196f305d40b9e5e8b017c27a3c
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2010-2016 Dovecot authors, see the included COPYING file */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#define DIRECTOR_RECONNECT_TIMEOUT_MSECS (30*1000)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#define DIRECTOR_USER_MOVE_TIMEOUT_MSECS (30*1000)
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi#define DIRECTOR_QUICK_RECONNECT_TIMEOUT_MSECS 1000
4fe3f07477bae6da3fb8d8fa9bab10ab82ada2bdTimo Sirainen#define DIRECTOR_DELAYED_DIR_REMOVE_MSECS (1000*30)
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainenstatic bool director_is_self_ip_set(struct director *dir)
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomistatic void director_find_self_ip(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen unsigned int i, count;
db623deb12c3566618faba5a35a44ceed83c3dc0Martti Rannanjärvi hosts = array_get(&dir->dir_hosts, &count);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; i < count; i++) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i_fatal("director_servers doesn't list ourself");
a356c4736fe6041142c6096045bc00c15a80af4eTimo Sirainen dir->self_host = director_host_lookup(dir, &dir->self_ip,
a356c4736fe6041142c6096045bc00c15a80af4eTimo Sirainen i_fatal("director_servers doesn't list ourself (%s:%u)",
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic unsigned int director_find_self_idx(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen unsigned int i, count;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; i < count; i++) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainendirector_has_outgoing_connection(struct director *dir,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (director_connection_get_host(*connp) == host &&
caae18c876f81e261350e4957471efa453c0ea9fTimo Sirainenint director_connect_host(struct director *dir, struct director_host *host)
caae18c876f81e261350e4957471efa453c0ea9fTimo Sirainen if (director_has_outgoing_connection(dir, host))
caae18c876f81e261350e4957471efa453c0ea9fTimo Sirainen str_printfa(str, "Connecting to %s:%u (as %s",
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen str_printfa(str, ", last network failure %ds ago",
009217abb57a24a4076092e8e4e165545747839eStephan Bosch (int)(ioloop_time - host->last_network_failure));
caae18c876f81e261350e4957471efa453c0ea9fTimo Sirainen str_printfa(str, ", last protocol failure %ds ago",
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen (int)(ioloop_time - host->last_protocol_failure));
447653aca14335b7096fb2dd2c7103631b336094Timo Sirainen port = dir->test_port != 0 ? dir->test_port : host->port;
447653aca14335b7096fb2dd2c7103631b336094Timo Sirainen fd = net_connect_ip(&host->ip, port, &dir->self_ip);
447653aca14335b7096fb2dd2c7103631b336094Timo Sirainen i_error("connect(%s) failed: %m", host->name);
447653aca14335b7096fb2dd2c7103631b336094Timo Sirainen /* Reset timestamp so that director_connect() won't skip this host
447653aca14335b7096fb2dd2c7103631b336094Timo Sirainen while we're still trying to connect to it */
447653aca14335b7096fb2dd2c7103631b336094Timo Sirainen (void)director_connection_init_out(dir, fd, host);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainendirector_get_preferred_right_host(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; i < count; i++) {
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen /* self, with some removed hosts */
42401be443f96c91a20fc976d66ca626fa6e14ecTimo Sirainenstatic bool director_wait_for_others(struct director *dir)
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* don't assume we're alone until we've attempted to connect
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen to others for a while */
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen ioloop_time - dir->ring_first_alone > DIRECTOR_RING_MIN_WAIT_SECS)
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* reset all failures and try again */
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen dir->to_reconnect = timeout_add(DIRECTOR_QUICK_RECONNECT_TIMEOUT_MSECS,
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen /* try to connect to first working server on our right side.
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen the left side is supposed to connect to us. */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen DIRECTOR_RECONNECT_RETRY_SECS > ioloop_time) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* connection failed recently, don't try retrying here */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen DIRECTOR_PROTOCOL_FAILURE_RETRY_SECS > ioloop_time) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* the director recently sent invalid protocol data,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen don't try retrying yet */
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen if (director_connect_host(dir, hosts[idx]) == 0) {
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen /* success */
8c2b4a45f17a5cb13bb01058ca37798cf48d91baTimo Sirainen if (count > 1 && director_wait_for_others(dir))
8c2b4a45f17a5cb13bb01058ca37798cf48d91baTimo Sirainen /* we're the only one */
8e1ecc6542da1e14c14e2e59d39dbccdbf68e2b5Timo Sirainen i_warning("director: Couldn't connect to right side, "
8e1ecc6542da1e14c14e2e59d39dbccdbf68e2b5Timo Sirainen "we must be the only director left");
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen /* since we couldn't connect to it,
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen it must have failed recently */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen i_warning("director: Assuming %s is dead, disconnecting",
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen "This connection is dead?");
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen dir->ring_min_version = DIRECTOR_VERSION_MINOR;
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainenvoid director_set_ring_handshaked(struct director *dir)
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen "continuing delayed requests");
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenstatic void director_reconnect_timeout(struct director *dir)
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen struct director_host *cur_host, *preferred_host =
4aab01f4eade3d278b61471516c062ce30a84b5fTimo Sirainen /* all directors have been removed, try again later */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen (void)director_connect_host(dir, preferred_host);
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* the connection hasn't finished sync yet.
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen keep this timeout for now. */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenvoid director_set_ring_synced(struct director *dir)
42401be443f96c91a20fc976d66ca626fa6e14ecTimo Sirainen i_assert((dir->left != NULL && dir->right != NULL) ||
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen i_warning("Ring is synced, continuing delayed requests "
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen "(syncing took %d secs, hosts_hash=%u)",
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen (int)(ioloop_time - dir->ring_last_sync_time),
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen if (host != director_get_preferred_right_host(dir)) {
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* try to reconnect to preferred host later */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen director_connection_set_synced(dir->left, TRUE);
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen director_connection_set_synced(dir->right, TRUE);
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenvoid director_sync_send(struct director *dir, struct director_host *host,
f64b5bc9e73bedc63ba3c072c286542c29c69e43Timo Sirainen unsigned int timestamp, unsigned int hosts_hash)
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainen dir->last_sync_sent_ring_change_counter = dir->ring_change_counter;
4aab01f4eade3d278b61471516c062ce30a84b5fTimo Sirainen director_connection_get_minor_version(dir->right) > 0) {
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen /* only minor_version>0 supports extra parameters */
468c28dfb03613ab8d487b5aebc985a969193aceTimo Sirainen str_printfa(str, "\t%u\t%u\t%u", minor_version,
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen director_connection_send(dir->right, str_c(str));
58b8a301b7b36047f10a592751094fbed86d6f0cTimo Sirainen /* ping our connections in case either of them are hanging.
58b8a301b7b36047f10a592751094fbed86d6f0cTimo Sirainen if they are, we want to know it fast. */
468c28dfb03613ab8d487b5aebc985a969193aceTimo Sirainenbool director_resend_sync(struct director *dir)
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen if (!dir->ring_synced && dir->left != NULL && dir->right != NULL) {
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen /* send a new SYNC in case the previous one got dropped */
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen dir->self_host->last_sync_timestamp = ioloop_time;
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen director_sync_send(dir, dir->self_host, dir->sync_seq,
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainenstatic void director_sync_timeout(struct director *dir)
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen i_error("Ring SYNC appears to have got lost, resending");
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainenvoid director_set_ring_unsynced(struct director *dir)
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainen dir->to_sync = timeout_add(DIRECTOR_SYNC_TIMEOUT_MSECS,
152db3f90f298b7fb2dbbd4276f0fc30a9bc30f6Timo Sirainenstatic void director_sync(struct director *dir)
152db3f90f298b7fb2dbbd4276f0fc30a9bc30f6Timo Sirainen /* we're synced again when we receive this SYNC back */
152db3f90f298b7fb2dbbd4276f0fc30a9bc30f6Timo Sirainen if (dir->right == NULL && dir->left == NULL) {
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainen /* we're alone. if we're already synced,
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainen don't become unsynced. */
c88275be7fcb92b10f412cecdd6c2ceaae17917eTimo Sirainen dir_debug("Ring is desynced (seq=%u, sending SYNC to %s)",
c88275be7fcb92b10f412cecdd6c2ceaae17917eTimo Sirainen dir->sync_seq, dir->right == NULL ? "(nowhere)" :
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* send PINGs to our connections more rapidly until we've synced again.
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen if the connection has actually died, we don't need to wait (and
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen delay requests) for as long to detect it */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen director_connection_set_synced(dir->left, FALSE);
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen director_connection_set_synced(dir->right, FALSE);
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen director_sync_send(dir, dir->self_host, dir->sync_seq,
1df39b899804fd1dbc560f75382364822935c857Timo Sirainenvoid director_sync_freeze(struct director *dir)
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenvoid director_notify_ring_added(struct director_host *added_host,
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen const char *cmd;
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen net_ip2addr(&added_host->ip), added_host->port);
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen director_update_send(added_host->dir, src, cmd);
5f5713d6468dca1acf3d350dd8a33057331f78c5Timo Sirainenstatic void director_delayed_dir_remove_timeout(struct director *dir)
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen unsigned int i, count;
58b8a301b7b36047f10a592751094fbed86d6f0cTimo Sirainen for (i = 0; i < count; ) {
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainenvoid director_ring_remove(struct director_host *removed_host,
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen struct director_connection *const *conns, *conn;
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen unsigned int i, count;
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen const char *cmd;
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen /* others will just disconnect us */
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen /* mark the host as removed and fully remove it later. this delay is
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen needed, because the removal may trigger director reconnections,
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen which may send the director back and we don't want to re-add it */
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen timeout_add(DIRECTOR_DELAYED_DIR_REMOVE_MSECS,
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen /* disconnect any connections to the host */
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen for (i = 0; i < count; ) {
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen if (director_connection_get_host(conn) != removed_host)
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen director_connection_deinit(&conn, "Removing from ring");
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen cmd = t_strdup_printf("DIRECTOR-REMOVE\t%s\t%u\n",
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainendirector_send_host(struct director *dir, struct director_host *src,
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen const char *host_tag = mail_host_get_tag(host);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (dir->ring_min_version >= DIRECTOR_VERSION_TAGS_V2) {
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen dir->ring_min_version < DIRECTOR_VERSION_TAGS_V2) {
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen if (dir->ring_min_version < DIRECTOR_VERSION_TAGS) {
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen i_error("Ring has directors that don't support tags - removing host %s with tag '%s'",
09060303d565e15d54e42b4ef722f9d3c26f5336Timo Sirainen i_error("Ring has directors that support mixed versions of tags - removing host %s with tag '%s'",
a5ddfd7a8b473f73135b93d5e081e470a87f0f7eTimo Sirainen if (dir->ring_min_version >= DIRECTOR_VERSION_UPDOWN) {
093b42b11c1236a687d3da564b26a324e2189ae6Timo Sirainen str_printfa(str, "\t%c%ld\t", host->down ? 'D' : 'U',
ae32667c54480d329eed994b3defab89cd76c077Timo Sirainen /* add any further version checks here - these directors ignore
ae32667c54480d329eed994b3defab89cd76c077Timo Sirainen any extra unknown arguments */
09060303d565e15d54e42b4ef722f9d3c26f5336Timo Sirainenvoid director_resend_hosts(struct director *dir)
a5ddfd7a8b473f73135b93d5e081e470a87f0f7eTimo Sirainen array_foreach(mail_hosts_get(dir->mail_hosts), hostp)
9de5eb9e1ac3a07c4197a60fdefd412d6cc78eb2Timo Sirainen director_send_host(dir, dir->self_host, NULL, *hostp);
9de5eb9e1ac3a07c4197a60fdefd412d6cc78eb2Timo Sirainenvoid director_update_host(struct director *dir, struct director_host *src,
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen /* update state in case this is the first mail host being added */
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen "down=%d last_updown_change=%ld (hosts_hash=%u)",
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen net_ip2addr(&host->ip), host->vhost_count, host->down,
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen /* mark the host desynced until ring is synced again. except if we're
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen alone in the ring that never happens. */
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainenvoid director_remove_host(struct director *dir, struct director_host *src,
a93c909cfa1139226891d198dc6f7a7c98509b49Timo Sirainen director_update_send(dir, src, t_strdup_printf(
242abe3ad2423776e9cf05e1304eb8fda4831b23Timo Sirainen "HOST-REMOVE\t%s\t%u\t%u\t%s\n",
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenvoid director_flush_host(struct director *dir, struct director_host *src,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen director_update_send(dir, src, t_strdup_printf(
87842f621233257b7a7945d994ba931508b34877Timo Sirainen "HOST-FLUSH\t%s\t%u\t%u\t%s\n",
242abe3ad2423776e9cf05e1304eb8fda4831b23Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainenvoid director_update_user(struct director *dir, struct director_host *src,
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen director_update_send(dir, src, t_strdup_printf("USER\t%u\t%s\n",
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen user->username_hash, net_ip2addr(&user->host->ip)));
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainenvoid director_update_user_weak(struct director *dir, struct director_host *src,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const char *cmd;
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainen cmd = t_strdup_printf("USER-WEAK\t%s\t%u\t%u\t%u\t%s\n",
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen user->username_hash, net_ip2addr(&user->host->ip));
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainen if (src != dir->self_host && dir->left != NULL && dir->right != NULL &&
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen /* only two directors in this ring and we're forwarding
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen USER-WEAK from one director back to itself via another
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen so it sees we've received it. we can't use
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen director_update_send() for this, because it doesn't send
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen data back to the source. */
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainendirector_user_kill_finish_delayed_to(struct director_user_kill_finish_ctx *ctx)
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainen i_assert(ctx->user->kill_state == USER_KILL_STATE_DELAY);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainendirector_user_kill_finish_delayed(struct director *dir, struct user *user)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen ctx = i_new(struct director_user_kill_finish_ctx, 1);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi /* wait for a while for the kills to finish in the backend server,
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi so there are no longer any processes running for the user before we
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi start letting new in connections to the new server. */
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi user->to_move = timeout_add(dir->set->director_user_kick_delay * 1000,
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomidirector_finish_user_kill(struct director *dir, struct user *user, bool self)
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi i_assert(user->kill_state != USER_KILL_STATE_DELAY);
9faac59f664d4003d1412278ac8db4aad1c2067cTimo Sirainen /* we're alone */
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi } else if (self ||
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi user->kill_state == USER_KILL_STATE_KILLING_NOTIFY_RECEIVED) {
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi director_connection_send(dir->right, t_strdup_printf(
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi user->kill_state = USER_KILL_STATE_KILLED_WAITING_FOR_EVERYONE;
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi i_assert(user->kill_state == USER_KILL_STATE_KILLING);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi user->kill_state = USER_KILL_STATE_KILLED_WAITING_FOR_NOTIFY;
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomistatic void director_kill_user_callback(enum ipc_client_cmd_state state,
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi /* this is an asynchronous notification about user being killed.
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi there are no guarantees about what might have happened to the user
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi in the mean time. */
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi /* shouldn't get here. the command reply isn't finished yet. */
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi /* we can't really do anything but continue anyway */
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi user = user_directory_lookup(ctx->dir->users, ctx->username_hash);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi /* user was already freed - ignore */
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi } else if (user->kill_state == USER_KILL_STATE_KILLING ||
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi user->kill_state == USER_KILL_STATE_KILLING_NOTIFY_RECEIVED) {
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi /* we were still waiting for the kill notification */
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi director_finish_user_kill(ctx->dir, user, ctx->self);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi /* we don't currently want to kill the user */
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomistatic void director_user_move_timeout(struct user *user)
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi i_assert(user->kill_state != USER_KILL_STATE_DELAY);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi "its state may now be inconsistent", user->username_hash);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomivoid director_move_user(struct director *dir, struct director_host *src,
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi unsigned int username_hash, struct mail_host *host)
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi const char *cmd;
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi /* 1. move this user's host, and set its "killing" flag to delay all of
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen its future connections until all directors have killed the
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen connections and notified us about it.
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 2. tell the other directors about the move
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 3. once user kill callback is called, tell the other directors
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi with USER-KILLED that we're done killing the user.
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 4. when some director gets a duplicate USER-KILLED, it's
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen responsible for notifying all directors that user is completely
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 5. after receiving USER-KILLED-EVERYWHERE notification,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen new connections are again allowed for the user.
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi user = user_directory_lookup(dir->users, username_hash);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user = user_directory_add(dir->users, username_hash,
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi /* user is already in this host */
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen if (user->kill_state == USER_KILL_STATE_NONE) {
3e10cd470988298dc2b37f548ad03da4d7c11ffaTimo Sirainen user->to_move = timeout_add(DIRECTOR_USER_MOVE_TIMEOUT_MSECS,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen cmd = t_strdup_printf("proxy\t*\tKICK-DIRECTOR-HASH\t%u",
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_update_send(dir, src, t_strdup_printf(
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen "USER-MOVE\t%s\t%u\t%u\t%u\t%s\n",
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
5f5713d6468dca1acf3d350dd8a33057331f78c5Timo Sirainen user->username_hash, net_ip2addr(&user->host->ip)));
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainendirector_kick_user_callback(enum ipc_client_cmd_state state ATTR_UNUSED,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenvoid director_kick_user(struct director *dir, struct director_host *src,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen struct director_host *orig_src, const char *username)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen const char *cmd;
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen cmd = t_strdup_printf("proxy\t*\tKICK\t%s", username);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen cmd = t_strdup_printf("USER-KICK\t%s\t%u\t%u\t%s\n",
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen director_update_send_version(dir, src, DIRECTOR_VERSION_USER_KICK, cmd);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenvoid director_kick_user_hash(struct director *dir, struct director_host *src,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen const char *cmd;
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen cmd = t_strdup_printf("proxy\t*\tKICK-DIRECTOR-HASH\t%u\t%s",
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen cmd = t_strdup_printf("USER-KICK-HASH\t%s\t%u\t%u\t%u\t%s\n",
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen director_update_send_version(dir, src, DIRECTOR_VERSION_USER_KICK, cmd);
488d92156e431ef201faf75910955588d0b768e3Timo Sirainenvoid director_user_killed(struct director *dir, unsigned int username_hash)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user = user_directory_lookup(dir->users, username_hash);
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen user->kill_state = USER_KILL_STATE_KILLING_NOTIFY_RECEIVED;
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen case USER_KILL_STATE_KILLED_WAITING_FOR_NOTIFY:
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen case USER_KILL_STATE_KILLED_WAITING_FOR_EVERYONE:
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen director_user_killed_everywhere(dir, dir->self_host,
ffedd89747e11373c0dd3a172d3e94e0cf724fd7Timo Sirainenvoid director_user_killed_everywhere(struct director *dir,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user = user_directory_lookup(dir->users, username_hash);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user->kill_state != USER_KILL_STATE_KILLED_WAITING_FOR_EVERYONE)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_update_send(dir, src, t_strdup_printf(
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen "USER-KILLED-EVERYWHERE\t%s\t%u\t%u\t%u\n",
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenstatic void director_state_callback_timeout(struct director *dir)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenvoid director_set_state_changed(struct director *dir)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen /* we may get called to here from various places. use a timeout to
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen make sure the state callback is called with a clean state. */
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen timeout_add(0, director_state_callback_timeout, dir);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenvoid director_update_send(struct director *dir, struct director_host *src,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen const char *cmd)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_update_send_version(dir, src, 0, cmd);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenvoid director_update_send_version(struct director *dir,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen if (director_connection_get_host(*connp) != src &&
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_connection_get_minor_version(*connp) >= min_version)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainendirector_init(const struct director_settings *set,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen const struct ip_addr *listen_ip, in_port_t listen_port,
550d2fe097e95f12e8fa60ef52753ea7fe53d4eaTimo Sirainen dir->users = user_directory_init(set->director_user_expire,
550d2fe097e95f12e8fa60ef52753ea7fe53d4eaTimo Sirainen dir->mail_hosts = mail_hosts_init(set->director_consistent_hashing);
4a4fdb14788aec22dab8f1b91e9523233e5548dcTimo Sirainen dir->ipc_proxy = ipc_client_init(DIRECTOR_IPC_PROXY_PATH);
4a4fdb14788aec22dab8f1b91e9523233e5548dcTimo Sirainen dir->ring_min_version = DIRECTOR_VERSION_MINOR;
4a4fdb14788aec22dab8f1b91e9523233e5548dcTimo Sirainen struct director_connection *conn, *const *connp;
bfef6891565ff9018ac92add6eae401e9352c657Timo Sirainen director_connection_deinit(&conn, "Shutting down");