bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen#define DIRECTOR_RECONNECT_TIMEOUT_MSECS (30*1000)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen#define DIRECTOR_USER_MOVE_TIMEOUT_MSECS (30*1000)
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen#define DIRECTOR_QUICK_RECONNECT_TIMEOUT_MSECS 1000
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen#define DIRECTOR_DELAYED_DIR_REMOVE_MSECS (1000*30)
c13fce16374a6fa8d127742c527498d38e777789Timo Sirainenconst char *user_kill_state_names[USER_KILL_STATE_DELAY+1] = {
c13fce16374a6fa8d127742c527498d38e777789Timo Sirainen "notify-received",
c13fce16374a6fa8d127742c527498d38e777789Timo Sirainen "waiting-for-notify",
c13fce16374a6fa8d127742c527498d38e777789Timo Sirainen "waiting-for-everyone",
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainenstatic struct log_throttle *user_move_throttle;
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainenstatic struct log_throttle *user_kill_fail_throttle;
eb5809d0666810ff5ba9da188e1e4e2092ee0eeaTimo Sirainenstatic void director_hosts_purge_removed(struct director *dir);
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainenstatic const struct log_throttle_settings director_log_throttle_settings = {
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomidirector_user_kill_finish_delayed(struct director *dir, struct user *user,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic bool director_is_self_ip_set(struct director *dir)
db623deb12c3566618faba5a35a44ceed83c3dc0Martti Rannanjärvi if (net_ip_compare(&dir->self_ip, &net_ip4_any))
db623deb12c3566618faba5a35a44ceed83c3dc0Martti Rannanjärvi if (net_ip_compare(&dir->self_ip, &net_ip6_any))
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic void director_find_self_ip(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; i < count; i++) {
a356c4736fe6041142c6096045bc00c15a80af4eTimo Sirainen i_fatal("director_servers doesn't list ourself");
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen dir->self_host = director_host_lookup(dir, &dir->self_ip,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i_fatal("director_servers doesn't list ourself (%s:%u)",
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic unsigned int director_find_self_idx(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; i < count; i++) {
caae18c876f81e261350e4957471efa453c0ea9fTimo Sirainendirector_has_outgoing_connection(struct director *dir,
caae18c876f81e261350e4957471efa453c0ea9fTimo Sirainen if (director_connection_get_host(*connp) == host &&
e2fdcdb4ee53ab769123e27997713aaea34910e1Timo Sirainendirector_log_connect(struct director *dir, struct director_host *host,
e9d302dceba09ceefa0aaddf7eafafd760cd1736Timo Sirainen str_printfa(str, ", last network failure %ds ago",
e9d302dceba09ceefa0aaddf7eafafd760cd1736Timo Sirainen (int)(ioloop_time - host->last_network_failure));
e9d302dceba09ceefa0aaddf7eafafd760cd1736Timo Sirainen str_printfa(str, ", last protocol failure %ds ago",
e9d302dceba09ceefa0aaddf7eafafd760cd1736Timo Sirainen (int)(ioloop_time - host->last_protocol_failure));
e2fdcdb4ee53ab769123e27997713aaea34910e1Timo Sirainen net_ip2addr(&dir->self_ip), str_c(str), reason);
e2fdcdb4ee53ab769123e27997713aaea34910e1Timo Sirainenint director_connect_host(struct director *dir, struct director_host *host,
caae18c876f81e261350e4957471efa453c0ea9fTimo Sirainen if (director_has_outgoing_connection(dir, host))
00e7c3010f7da4a49881a7feb05e413af353af0aTimo Sirainen port = dir->test_port != 0 ? dir->test_port : host->port;
00e7c3010f7da4a49881a7feb05e413af353af0aTimo Sirainen fd = net_connect_ip(&host->ip, port, &dir->self_ip);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i_error("connect(%s) failed: %m", host->name);
4c20178a7f70bfe43d252e50796013aac1d8c74aTimo Sirainen /* Reset timestamp so that director_connect() won't skip this host
4c20178a7f70bfe43d252e50796013aac1d8c74aTimo Sirainen while we're still trying to connect to it */
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)director_connection_init_out(dir, fd, host);
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainendirector_get_preferred_right_host(struct director *dir)
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen for (i = 0; i < count; i++) {
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen /* self, with some removed hosts */
e2fdcdb4ee53ab769123e27997713aaea34910e1Timo Sirainenstatic void director_quick_reconnect_retry(struct director *dir)
e2fdcdb4ee53ab769123e27997713aaea34910e1Timo Sirainen director_connect(dir, "Alone in director ring - trying to connect to others");
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainenstatic bool director_wait_for_others(struct director *dir)
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen /* don't assume we're alone until we've attempted to connect
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen to others for a while */
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen ioloop_time - dir->ring_first_alone > DIRECTOR_RING_MIN_WAIT_SECS)
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen /* reset all failures and try again */
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen dir->to_reconnect = timeout_add(DIRECTOR_QUICK_RECONNECT_TIMEOUT_MSECS,
e2fdcdb4ee53ab769123e27997713aaea34910e1Timo Sirainenvoid director_connect(struct director *dir, const char *reason)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* try to connect to first working server on our right side.
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen the left side is supposed to connect to us. */
8e1ecc6542da1e14c14e2e59d39dbccdbf68e2b5Timo Sirainen DIRECTOR_RECONNECT_RETRY_SECS > ioloop_time) {
8c2b4a45f17a5cb13bb01058ca37798cf48d91baTimo Sirainen /* connection failed recently, don't try retrying here */
8c2b4a45f17a5cb13bb01058ca37798cf48d91baTimo Sirainen DIRECTOR_PROTOCOL_FAILURE_RETRY_SECS > ioloop_time) {
8c2b4a45f17a5cb13bb01058ca37798cf48d91baTimo Sirainen /* the director recently sent invalid protocol data,
8c2b4a45f17a5cb13bb01058ca37798cf48d91baTimo Sirainen don't try retrying yet */
e2fdcdb4ee53ab769123e27997713aaea34910e1Timo Sirainen if (director_connect_host(dir, hosts[idx], reason) == 0) {
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen /* success */
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen if (count > 1 && director_wait_for_others(dir))
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen /* we're the only one */
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen i_warning("director: Couldn't connect to right side, "
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen "we must be the only director left");
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen /* since we couldn't connect to it,
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen it must have failed recently */
dff32d11a411a24f3b76003c1ae22c5a960f180eTimo Sirainen i_warning("director: Assuming %s is dead, disconnecting",
dff32d11a411a24f3b76003c1ae22c5a960f180eTimo Sirainen "This connection is dead?");
89d6cd658eabf46e07e40037b0e641ed9be1a2a3Timo Sirainen dir->ring_min_version = DIRECTOR_VERSION_MINOR;
4aab01f4eade3d278b61471516c062ce30a84b5fTimo Sirainenvoid director_set_ring_handshaked(struct director *dir)
0d1b8b6bec79746c5d89d57dd8c1688946bd9237Josef 'Jeff' Sipek timeout_remove(&dir->to_handshake_warning);
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen "continuing delayed requests");
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenstatic void director_reconnect_timeout(struct director *dir)
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen struct director_host *cur_host, *preferred_host =
42401be443f96c91a20fc976d66ca626fa6e14ecTimo Sirainen /* all directors have been removed, try again later */
e2fdcdb4ee53ab769123e27997713aaea34910e1Timo Sirainen (void)director_connect_host(dir, preferred_host,
e2fdcdb4ee53ab769123e27997713aaea34910e1Timo Sirainen "Reconnect attempt to preferred director");
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* the connection hasn't finished sync yet.
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen keep this timeout for now. */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenvoid director_set_ring_synced(struct director *dir)
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen i_assert((dir->left != NULL && dir->right != NULL) ||
0d1b8b6bec79746c5d89d57dd8c1688946bd9237Josef 'Jeff' Sipek timeout_remove(&dir->to_handshake_warning);
56c69b4b17af0b5a2c71705d5edae746f00780b9Timo Sirainen i_warning("Ring is synced, continuing delayed requests "
1574df6b0bc965212f1152e480e7a762cdaa8226Timo Sirainen "(syncing took %d secs, hosts_hash=%u)",
1574df6b0bc965212f1152e480e7a762cdaa8226Timo Sirainen (int)(ioloop_time - dir->ring_last_sync_time),
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen if (host != director_get_preferred_right_host(dir)) {
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* try to reconnect to preferred host later */
f64b5bc9e73bedc63ba3c072c286542c29c69e43Timo Sirainen director_connection_set_synced(dir->left, TRUE);
f64b5bc9e73bedc63ba3c072c286542c29c69e43Timo Sirainen director_connection_set_synced(dir->right, TRUE);
eb5809d0666810ff5ba9da188e1e4e2092ee0eeaTimo Sirainen /* If there are any director hosts still marked as "removed", we can
eb5809d0666810ff5ba9da188e1e4e2092ee0eeaTimo Sirainen safely remove those now. The entire director cluster knows about the
eb5809d0666810ff5ba9da188e1e4e2092ee0eeaTimo Sirainen removal now. */
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainenvoid director_sync_send(struct director *dir, struct director_host *host,
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen unsigned int timestamp, unsigned int hosts_hash)
58b8a301b7b36047f10a592751094fbed86d6f0cTimo Sirainen dir->last_sync_sent_ring_change_counter = dir->ring_change_counter;
ee3362f3b78827a2c9a7e9ddee83f5a429c06213Timo Sirainen director_connection_get_minor_version(dir->right) > 0) {
468c28dfb03613ab8d487b5aebc985a969193aceTimo Sirainen /* only minor_version>0 supports extra parameters */
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen str_printfa(str, "\t%u\t%u\t%u", minor_version,
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen director_connection_send(dir->right, str_c(str));
5f5713d6468dca1acf3d350dd8a33057331f78c5Timo Sirainen /* ping our connections in case either of them are hanging.
5f5713d6468dca1acf3d350dd8a33057331f78c5Timo Sirainen if they are, we want to know it fast. */
d5967394396acaa4aaa249ea973423aa5b9c38edTimo Sirainendirector_has_any_outgoing_connections(struct director *dir)
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainenbool director_resend_sync(struct director *dir)
d5967394396acaa4aaa249ea973423aa5b9c38edTimo Sirainen /* everything ok, no need to do anything */
d5967394396acaa4aaa249ea973423aa5b9c38edTimo Sirainen /* right side connection is missing. make sure we're not
d5967394396acaa4aaa249ea973423aa5b9c38edTimo Sirainen hanging due to some bug. */
d5967394396acaa4aaa249ea973423aa5b9c38edTimo Sirainen !director_has_any_outgoing_connections(dir)) {
d5967394396acaa4aaa249ea973423aa5b9c38edTimo Sirainen i_warning("Right side connection is unexpectedly lost, reconnecting");
d5967394396acaa4aaa249ea973423aa5b9c38edTimo Sirainen director_connect(dir, "Right side connection lost");
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainen /* send a new SYNC in case the previous one got dropped */
468c28dfb03613ab8d487b5aebc985a969193aceTimo Sirainen dir->self_host->last_sync_timestamp = ioloop_time;
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen director_sync_send(dir, dir->self_host, dir->sync_seq,
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainenstatic void director_sync_timeout(struct director *dir)
3a7a0c35c9f827f86fb437e640287f3a467ef692Timo Sirainen i_error("Ring SYNC seq=%u appears to have got lost, resending", dir->sync_seq);
152db3f90f298b7fb2dbbd4276f0fc30a9bc30f6Timo Sirainenvoid director_set_ring_unsynced(struct director *dir)
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainen dir->to_sync = timeout_add(DIRECTOR_SYNC_TIMEOUT_MSECS,
242abe3ad2423776e9cf05e1304eb8fda4831b23Timo Sirainenstatic void director_sync(struct director *dir)
002f2544421891472dc9aeb79d3abdde6a8ed4c8Timo Sirainen /* we're synced again when we receive this SYNC back */
c88275be7fcb92b10f412cecdd6c2ceaae17917eTimo Sirainen if (dir->right == NULL && dir->left == NULL) {
c88275be7fcb92b10f412cecdd6c2ceaae17917eTimo Sirainen /* we're alone. if we're already synced,
c88275be7fcb92b10f412cecdd6c2ceaae17917eTimo Sirainen don't become unsynced. */
465fb89a877b778f68734f6cc36db9ce2a4a0a71Timo Sirainen dir_debug("Ring is desynced (seq=%u, no right connection)",
1df39b899804fd1dbc560f75382364822935c857Timo Sirainen dir_debug("Ring is desynced (seq=%u, sending SYNC to %s)",
1df39b899804fd1dbc560f75382364822935c857Timo Sirainen dir->sync_seq, dir->right == NULL ? "(nowhere)" :
f64b5bc9e73bedc63ba3c072c286542c29c69e43Timo Sirainen /* send PINGs to our connections more rapidly until we've synced again.
f64b5bc9e73bedc63ba3c072c286542c29c69e43Timo Sirainen if the connection has actually died, we don't need to wait (and
f64b5bc9e73bedc63ba3c072c286542c29c69e43Timo Sirainen delay requests) for as long to detect it */
f64b5bc9e73bedc63ba3c072c286542c29c69e43Timo Sirainen director_connection_set_synced(dir->left, FALSE);
f64b5bc9e73bedc63ba3c072c286542c29c69e43Timo Sirainen director_connection_set_synced(dir->right, FALSE);
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen director_sync_send(dir, dir->self_host, dir->sync_seq,
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenvoid director_sync_freeze(struct director *dir)
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainenvoid director_notify_ring_added(struct director_host *added_host,
027f58ea63a1822bbf13d99ee5572e5f8b9e8d8bTimo Sirainen i_info("Adding director %s to ring (requested by %s)",
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen director_update_send(added_host->dir, src, cmd);
de69258ceb035b1e08301dc70e257bef341dea85Timo Sirainenstatic void director_hosts_purge_removed(struct director *dir)
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen for (i = 0; i < count; ) {
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainenvoid director_ring_remove(struct director_host *removed_host,
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen struct director_connection *const *conns, *conn;
027f58ea63a1822bbf13d99ee5572e5f8b9e8d8bTimo Sirainen i_info("Removing director %s from ring (requested by %s)",
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen /* others will just disconnect us */
b0421c7397be2146988ee3afb5dcc491c01206ccTimo Sirainen /* mark the host as removed and fully remove it later. this
b0421c7397be2146988ee3afb5dcc491c01206ccTimo Sirainen delay is needed, because the removal may trigger director
b0421c7397be2146988ee3afb5dcc491c01206ccTimo Sirainen reconnections, which may send the director back and we don't
b0421c7397be2146988ee3afb5dcc491c01206ccTimo Sirainen want to re-add it */
b0421c7397be2146988ee3afb5dcc491c01206ccTimo Sirainen timeout_add(DIRECTOR_DELAYED_DIR_REMOVE_MSECS,
83b9d8afc2776ebf61ddb77b85241f2591ef7f7bTimo Sirainen /* if our left or ride side gets removed, notify them first
83b9d8afc2776ebf61ddb77b85241f2591ef7f7bTimo Sirainen before disconnecting. */
83b9d8afc2776ebf61ddb77b85241f2591ef7f7bTimo Sirainen cmd = t_strdup_printf("DIRECTOR-REMOVE\t%s\t%u\n",
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen /* disconnect any connections to the host */
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen for (i = 0; i < count; ) {
b0421c7397be2146988ee3afb5dcc491c01206ccTimo Sirainen if (director_connection_get_host(conn) != removed_host ||
dff32d11a411a24f3b76003c1ae22c5a960f180eTimo Sirainen director_connection_deinit(&conn, "Removing from ring");
e2fdcdb4ee53ab769123e27997713aaea34910e1Timo Sirainen director_connect(dir, "Reconnecting after director was removed");
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainendirector_send_host(struct director *dir, struct director_host *src,
093b42b11c1236a687d3da564b26a324e2189ae6Timo Sirainen const char *host_tag = mail_host_get_tag(host);
bd4bbe6478a97e3fab77b05257dd1397c7c090eaTimo Sirainen orig_src->ip_str, orig_src->port, orig_src->last_seq,
ae32667c54480d329eed994b3defab89cd76c077Timo Sirainen if (dir->ring_min_version >= DIRECTOR_VERSION_TAGS_V2) {
ae32667c54480d329eed994b3defab89cd76c077Timo Sirainen dir->ring_min_version < DIRECTOR_VERSION_TAGS_V2) {
ae32667c54480d329eed994b3defab89cd76c077Timo Sirainen if (dir->ring_min_version < DIRECTOR_VERSION_TAGS) {
ae32667c54480d329eed994b3defab89cd76c077Timo Sirainen i_error("Ring has directors that don't support tags - removing host %s with tag '%s'",
ae32667c54480d329eed994b3defab89cd76c077Timo Sirainen i_error("Ring has directors that support mixed versions of tags - removing host %s with tag '%s'",
a5ddfd7a8b473f73135b93d5e081e470a87f0f7eTimo Sirainen if (dir->ring_min_version >= DIRECTOR_VERSION_UPDOWN) {
9de5eb9e1ac3a07c4197a60fdefd412d6cc78eb2Timo Sirainen str_printfa(str, "\t%c%ld\t", host->down ? 'D' : 'U',
9de5eb9e1ac3a07c4197a60fdefd412d6cc78eb2Timo Sirainen /* add any further version checks here - these directors ignore
9de5eb9e1ac3a07c4197a60fdefd412d6cc78eb2Timo Sirainen any extra unknown arguments */
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainenvoid director_resend_hosts(struct director *dir)
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen array_foreach(mail_hosts_get(dir->mail_hosts), hostp)
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen director_send_host(dir, dir->self_host, NULL, *hostp);
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainenvoid director_update_host(struct director *dir, struct director_host *src,
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen /* update state in case this is the first mail host being added */
abe29107f5dce932d28a00912d2d75a01021bef1Timo Sirainen "down=%d last_updown_change=%ld (hosts_hash=%u)",
bd4bbe6478a97e3fab77b05257dd1397c7c090eaTimo Sirainen host->ip_str, host->vhost_count, host->down ? 1 : 0,
a93c909cfa1139226891d198dc6f7a7c98509b49Timo Sirainen /* mark the host desynced until ring is synced again. except if we're
a93c909cfa1139226891d198dc6f7a7c98509b49Timo Sirainen alone in the ring that never happens. */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenvoid director_remove_host(struct director *dir, struct director_host *src,
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen director_update_send(dir, src, t_strdup_printf(
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen "HOST-REMOVE\t%s\t%u\t%u\t%s\n",
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainenvoid director_flush_host(struct director *dir, struct director_host *src,
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen director_update_send(dir, src, t_strdup_printf(
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen "HOST-FLUSH\t%s\t%u\t%u\t%s\n",
bd4bbe6478a97e3fab77b05257dd1397c7c090eaTimo Sirainen orig_src->ip_str, orig_src->port, orig_src->last_seq,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenvoid director_update_user(struct director *dir, struct director_host *src,
b3e36790ca9e16d022118012b46ed50f73a45046Timo Sirainen if (director_connection_get_host(*connp) == src)
b3e36790ca9e16d022118012b46ed50f73a45046Timo Sirainen if (director_connection_get_minor_version(*connp) >= DIRECTOR_VERSION_USER_TIMESTAMP) {
b3e36790ca9e16d022118012b46ed50f73a45046Timo Sirainen director_connection_send(*connp, t_strdup_printf(
b3e36790ca9e16d022118012b46ed50f73a45046Timo Sirainen "USER\t%u\t%s\t%u\n", user->username_hash, user->host->ip_str,
b3e36790ca9e16d022118012b46ed50f73a45046Timo Sirainen director_connection_send(*connp, t_strdup_printf(
b3e36790ca9e16d022118012b46ed50f73a45046Timo Sirainen "USER\t%u\t%s\n", user->username_hash, user->host->ip_str));
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainenvoid director_update_user_weak(struct director *dir, struct director_host *src,
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainen cmd = t_strdup_printf("USER-WEAK\t%s\t%u\t%u\t%u\t%s\n",
bd4bbe6478a97e3fab77b05257dd1397c7c090eaTimo Sirainen orig_src->ip_str, orig_src->port, orig_src->last_seq,
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainen if (src != dir->self_host && dir->left != NULL && dir->right != NULL &&
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainen /* only two directors in this ring and we're forwarding
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainen USER-WEAK from one director back to itself via another
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainen so it sees we've received it. we can't use
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainen director_update_send() for this, because it doesn't send
ab779efe68458cf6fdcaa4f99527685d5563df0aTimo Sirainen data back to the source. */
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainendirector_flush_user_continue(int result, struct director_kill_context *ctx)
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi struct user *user = user_directory_lookup(ctx->tag->users,
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi struct istream *is = iostream_temp_finish(&ctx->reply, (size_t)-1);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi i_error("%s: Failed to flush user hash %u in host %s: %s",
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi while((data = i_stream_read_next_line(is)) != NULL) {
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi i_error("%s: Failed to flush user hash %u in host %s: %s",
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen if (!DIRECTOR_KILL_CONTEXT_IS_VALID(user, ctx)) {
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen /* user was already freed - ignore */
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen dir_debug("User %u freed while flushing, result=%d",
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen /* ctx is freed later via user->kill_ctx */
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen dir_debug("Flushing user %u finished, result=%d",
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen director_user_kill_finish_delayed(dir, user, result == 1);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomidirector_flush_user(struct director *dir, struct user *user)
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen struct director_kill_context *ctx = user->kill_ctx;
6af8cc0573832465c621bcd91634e15461e8bf58Timo Sirainen /* Execute flush script, if set. Only the director that started the
6af8cc0573832465c621bcd91634e15461e8bf58Timo Sirainen user moving will call the flush script. Having each director do it
6af8cc0573832465c621bcd91634e15461e8bf58Timo Sirainen would be redundant since they're all supposed to be performing the
b93ec4e5b569e1216d0c7a6f28cbdccfe4fb32caTimo Sirainen same flush task to the same backend.
b93ec4e5b569e1216d0c7a6f28cbdccfe4fb32caTimo Sirainen Flushing is also not triggered if we're moving a user that we just
b93ec4e5b569e1216d0c7a6f28cbdccfe4fb32caTimo Sirainen created due to the user move. This means that the user doesn't have
b93ec4e5b569e1216d0c7a6f28cbdccfe4fb32caTimo Sirainen an old host, so we couldn't really even perform any flushing on the
6af8cc0573832465c621bcd91634e15461e8bf58Timo Sirainen if (*dir->set->director_flush_socket == '\0' ||
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi director_user_kill_finish_delayed(dir, user, FALSE);
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen if (var_expand(s_sock, dir->set->director_flush_socket, tab, &error) <= 0) {
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen i_error("Failed to expand director_flush_socket=%s: %s",
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen director_user_kill_finish_delayed(dir, user, FALSE);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi ctx->socket_path = str_free_without_data(&s_sock);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi .dns_client_socket_path = DIRECTOR_DNS_SOCKET_PATH,
0874d7a4fa15322318c71291f0134ff7cc49bbb8Timo Sirainen dir_debug("Flushing user %u via %s", user->username_hash,
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi if ((program_client_create(ctx->socket_path, args, &set, FALSE,
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi i_error("%s: Failed to flush user hash %u in host %s: %s",
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi program_client_set_output(ctx->pclient, ctx->reply);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomi program_client_run_async(ctx->pclient, director_flush_user_continue, ctx);
34642ccff3ad4906cd365d0cb3deaa1e4f16610bTimo Sirainenstatic void director_user_move_finished(struct director *dir)
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainenstatic void director_user_move_free(struct user *user)
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen struct director_kill_context *kill_ctx = user->kill_ctx;
0874d7a4fa15322318c71291f0134ff7cc49bbb8Timo Sirainen dir_debug("User %u move finished at state=%s", user->username_hash,
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainendirector_user_kill_finish_delayed_to(struct user *user)
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen i_assert(user->kill_ctx->kill_state == USER_KILL_STATE_DELAY);
cf7857bce68a33cdabe88bb15568e21921eb5ac7Aki Tuomidirector_user_kill_finish_delayed(struct director *dir, struct user *user,
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen user->kill_ctx->kill_state = USER_KILL_STATE_NONE;
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen user->kill_ctx->kill_state = USER_KILL_STATE_DELAY;
3e10cd470988298dc2b37f548ad03da4d7c11ffaTimo Sirainen /* wait for a while for the kills to finish in the backend server,
3e10cd470988298dc2b37f548ad03da4d7c11ffaTimo Sirainen so there are no longer any processes running for the user before we
3e10cd470988298dc2b37f548ad03da4d7c11ffaTimo Sirainen start letting new in connections to the new server. */
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen timeout_add(dir->set->director_user_kick_delay * 1000,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainendirector_finish_user_kill(struct director *dir, struct user *user, bool self)
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen struct director_kill_context *kill_ctx = user->kill_ctx;
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen i_assert(kill_ctx->kill_state != USER_KILL_STATE_FLUSHING);
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen i_assert(kill_ctx->kill_state != USER_KILL_STATE_DELAY);
edaf7b320f860428b5cd3a0847dc0f7689816129Timo Sirainen dir_debug("User %u kill finished - %sstate=%s", user->username_hash,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen /* we're alone */
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen } else if (self ||
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen kill_ctx->kill_state == USER_KILL_STATE_KILLING_NOTIFY_RECEIVED) {
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_connection_send(dir->right, t_strdup_printf(
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen kill_ctx->kill_state = USER_KILL_STATE_KILLED_WAITING_FOR_EVERYONE;
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen i_assert(kill_ctx->kill_state == USER_KILL_STATE_KILLING);
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen kill_ctx->kill_state = USER_KILL_STATE_KILLED_WAITING_FOR_NOTIFY;
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainenstatic void director_user_kill_fail_throttled(unsigned int new_events_count,
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen i_error("Failed to kill %u users' connections", new_events_count);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenstatic void director_kill_user_callback(enum ipc_client_cmd_state state,
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen /* this is an asynchronous notification about user being killed.
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen there are no guarantees about what might have happened to the user
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen in the mean time. */
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen /* shouldn't get here. the command reply isn't finished yet. */
6f20cb5225ddc5ec76f4c9bf6505d959270624deTimo Sirainen i_error("login process sent unexpected reply to kick: %s", data);
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen if (log_throttle_accept(user_kill_fail_throttle)) {
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen i_error("Failed to kill user %u connections: %s",
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen /* we can't really do anything but continue anyway */
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi user = user_directory_lookup(ctx->tag->users, ctx->username_hash);
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen if (!DIRECTOR_KILL_CONTEXT_IS_VALID(user, ctx)) {
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen /* user was already freed - ignore */
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen i_assert(ctx->kill_state == USER_KILL_STATE_KILLING ||
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen ctx->kill_state == USER_KILL_STATE_KILLING_NOTIFY_RECEIVED);
488d92156e431ef201faf75910955588d0b768e3Timo Sirainen /* we were still waiting for the kill notification */
778a9861ee39eb23ec443a03cde832e3e31d7d37Timo Sirainen director_finish_user_kill(ctx->dir, user, ctx->kill_is_self_initiated);
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainenstatic void director_user_move_throttled(unsigned int new_events_count,
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen i_error("%u users' move timed out, their state may now be inconsistent",
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenstatic void director_user_move_timeout(struct user *user)
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen i_assert(user->kill_ctx->kill_state != USER_KILL_STATE_DELAY);
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen if (log_throttle_accept(user_move_throttle)) {
3c89b4794ac52cc229b2b1dcec2324b490539c30Timo Sirainen "its state may now be inconsistent (state=%s)",
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen user_kill_state_names[user->kill_ctx->kill_state]);
6ce2e6f7aba7b36649b9d9d599bf59f8519c48d8Timo Sirainen if (user->kill_ctx->kill_state == USER_KILL_STATE_FLUSHING) {
6ce2e6f7aba7b36649b9d9d599bf59f8519c48d8Timo Sirainen program_client_destroy(&user->kill_ctx->pclient);
f5d82a4b87a9b17894e1869cfe8b1a90afbced59Timo Sirainenvoid director_kill_user(struct director *dir, struct director_host *src,
3404b66f3fda78955be9b6891ce5f5d6e13ef97aTimo Sirainen /* User is being moved again before the previous move
3404b66f3fda78955be9b6891ce5f5d6e13ef97aTimo Sirainen finished. We'll just continue wherever we left off
3404b66f3fda78955be9b6891ce5f5d6e13ef97aTimo Sirainen dir_debug("User %u move restarted - previous kill_state=%s",
3404b66f3fda78955be9b6891ce5f5d6e13ef97aTimo Sirainen user_kill_state_names[user->kill_ctx->kill_state]);
3404b66f3fda78955be9b6891ce5f5d6e13ef97aTimo Sirainen user->kill_ctx = ctx = i_new(struct director_kill_context, 1);
cdc7ca129c3433d3e4b9d5e90f4c209e4546dfe9Timo Sirainen ctx->old_host_vhost_count = old_host->vhost_count;
3404b66f3fda78955be9b6891ce5f5d6e13ef97aTimo Sirainen ctx->to_move = timeout_add(DIRECTOR_USER_MOVE_TIMEOUT_MSECS,
ce8d63810932f48176304ed08cd8b7652c4a8addTimo Sirainen if ((old_host != NULL && old_host != user->host) || forced_kick) {
aad23ef1531748997776e50a5272a2fc8e022f04Timo Sirainen cmd = t_strdup_printf("proxy\t*\tKICK-DIRECTOR-HASH\t%u",
47f09f848021d147ab0cc7d75c4f138b40dbff8cTimo Sirainen /* a) we didn't even know about the user before now.
aad23ef1531748997776e50a5272a2fc8e022f04Timo Sirainen don't bother performing a local kick, since it wouldn't
47f09f848021d147ab0cc7d75c4f138b40dbff8cTimo Sirainen kick anything.
47f09f848021d147ab0cc7d75c4f138b40dbff8cTimo Sirainen b) our host was already correct. notify others that we have
47f09f848021d147ab0cc7d75c4f138b40dbff8cTimo Sirainen killed the user, but don't really do it. */
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenvoid director_move_user(struct director *dir, struct director_host *src,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen unsigned int username_hash, struct mail_host *host)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen /* 1. move this user's host, and set its "killing" flag to delay all of
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen its future connections until all directors have killed the
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen connections and notified us about it.
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 2. tell the other directors about the move
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 3. once user kill callback is called, tell the other directors
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen with USER-KILLED that we're done killing the user.
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 4. when some director gets a duplicate USER-KILLED, it's
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen responsible for notifying all directors that user is completely
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 5. after receiving USER-KILLED-EVERYWHERE notification,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen new connections are again allowed for the user.
68d87d8fb8f23ffed031ddfd9c410f3c929777faTimo Sirainen user = user_directory_lookup(users, username_hash);
edaf7b320f860428b5cd3a0847dc0f7689816129Timo Sirainen dir_debug("User %u move started: User was nonexistent",
68d87d8fb8f23ffed031ddfd9c410f3c929777faTimo Sirainen user = user_directory_add(users, username_hash,
47f09f848021d147ab0cc7d75c4f138b40dbff8cTimo Sirainen /* User is already in the wanted host, but another director
47f09f848021d147ab0cc7d75c4f138b40dbff8cTimo Sirainen didn't think so. We'll need to finish the move without
47f09f848021d147ab0cc7d75c4f138b40dbff8cTimo Sirainen killing any of our connections. */
edaf7b320f860428b5cd3a0847dc0f7689816129Timo Sirainen dir_debug("User %u move forwarded: host is already %s",
7a380e77afc69a81725d410cd67082a37cf140d2Timo Sirainen /* user is looked up via the new host's tag, so if it's found
7a380e77afc69a81725d410cd67082a37cf140d2Timo Sirainen the old tag has to be the same. */
edaf7b320f860428b5cd3a0847dc0f7689816129Timo Sirainen dir_debug("User %u move started: host %s -> %s",
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_update_send(dir, src, t_strdup_printf(
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen "USER-MOVE\t%s\t%u\t%u\t%u\t%s\n",
bd4bbe6478a97e3fab77b05257dd1397c7c090eaTimo Sirainen orig_src->ip_str, orig_src->port, orig_src->last_seq,
3638fd7bb3d9b900894eed0bb6296a9c2294c8f6Timo Sirainen /* kill the user only after sending the USER-MOVE, because the kill
3638fd7bb3d9b900894eed0bb6296a9c2294c8f6Timo Sirainen may finish instantly. */
ce8d63810932f48176304ed08cd8b7652c4a8addTimo Sirainen director_kill_user(dir, src, user, host->tag, old_host, FALSE);
6f20cb5225ddc5ec76f4c9bf6505d959270624deTimo Sirainendirector_kick_user_callback(enum ipc_client_cmd_state state,
6f20cb5225ddc5ec76f4c9bf6505d959270624deTimo Sirainen /* shouldn't get here. the command reply isn't finished yet. */
6f20cb5225ddc5ec76f4c9bf6505d959270624deTimo Sirainen i_error("login process sent unexpected reply to kick: %s", data);
550d2fe097e95f12e8fa60ef52753ea7fe53d4eaTimo Sirainenvoid director_kick_user(struct director *dir, struct director_host *src,
550d2fe097e95f12e8fa60ef52753ea7fe53d4eaTimo Sirainen struct director_host *orig_src, const char *username)
bd4bbe6478a97e3fab77b05257dd1397c7c090eaTimo Sirainen orig_src->ip_str, orig_src->port, orig_src->last_seq);
4a4fdb14788aec22dab8f1b91e9523233e5548dcTimo Sirainen director_update_send_version(dir, src, DIRECTOR_VERSION_USER_KICK, str_c(cmd));
bfef6891565ff9018ac92add6eae401e9352c657Timo Sirainenvoid director_kick_user_alt(struct director *dir, struct director_host *src,
bfef6891565ff9018ac92add6eae401e9352c657Timo Sirainen str_printfa(cmd, "USER-KICK-ALT\t%s\t%u\t%u\t",
bd4bbe6478a97e3fab77b05257dd1397c7c090eaTimo Sirainen orig_src->ip_str, orig_src->port, orig_src->last_seq);
bfef6891565ff9018ac92add6eae401e9352c657Timo Sirainen director_update_send_version(dir, src, DIRECTOR_VERSION_USER_KICK_ALT, str_c(cmd));
36e091dc733c6cd690c5aae6e411e41adb1eca73Timo Sirainenvoid director_kick_user_hash(struct director *dir, struct director_host *src,
36e091dc733c6cd690c5aae6e411e41adb1eca73Timo Sirainen cmd = t_strdup_printf("proxy\t*\tKICK-DIRECTOR-HASH\t%u\t%s",
36e091dc733c6cd690c5aae6e411e41adb1eca73Timo Sirainen cmd = t_strdup_printf("USER-KICK-HASH\t%s\t%u\t%u\t%u\t%s\n",
bd4bbe6478a97e3fab77b05257dd1397c7c090eaTimo Sirainen orig_src->ip_str, orig_src->port, orig_src->last_seq,
36e091dc733c6cd690c5aae6e411e41adb1eca73Timo Sirainen director_update_send_version(dir, src, DIRECTOR_VERSION_USER_KICK, cmd);
fd882f791279d0bb0cbf15ccec75cff9f6b399ccTimo Sirainendirector_send_user_killed_everywhere(struct director *dir,
fd882f791279d0bb0cbf15ccec75cff9f6b399ccTimo Sirainen director_update_send(dir, src, t_strdup_printf(
fd882f791279d0bb0cbf15ccec75cff9f6b399ccTimo Sirainen "USER-KILLED-EVERYWHERE\t%s\t%u\t%u\t%u\n",
bd4bbe6478a97e3fab77b05257dd1397c7c090eaTimo Sirainen orig_src->ip_str, orig_src->port, orig_src->last_seq,
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomidirector_user_tag_killed(struct director *dir, struct mail_tag *tag,
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi user = user_directory_lookup(tag->users, username_hash);
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen if (user == NULL || !USER_IS_BEING_KILLED(user))
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen user->kill_ctx->kill_state = USER_KILL_STATE_KILLING_NOTIFY_RECEIVED;
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen case USER_KILL_STATE_KILLED_WAITING_FOR_NOTIFY:
fd882f791279d0bb0cbf15ccec75cff9f6b399ccTimo Sirainen dir_debug("User %u kill_state=%s - ignoring USER-KILLED",
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen username_hash, user_kill_state_names[user->kill_ctx->kill_state]);
fd882f791279d0bb0cbf15ccec75cff9f6b399ccTimo Sirainen /* move restarted. state=none can also happen if USER-MOVE was
fd882f791279d0bb0cbf15ccec75cff9f6b399ccTimo Sirainen sent while we were still moving. send back
fd882f791279d0bb0cbf15ccec75cff9f6b399ccTimo Sirainen USER-KILLED-EVERYWHERE to avoid hangs. */
fd882f791279d0bb0cbf15ccec75cff9f6b399ccTimo Sirainen director_send_user_killed_everywhere(dir, dir->self_host, NULL,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen case USER_KILL_STATE_KILLED_WAITING_FOR_EVERYONE:
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_user_killed_everywhere(dir, dir->self_host,
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomivoid director_user_killed(struct director *dir, unsigned int username_hash)
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi array_foreach(mail_hosts_get_tags(dir->mail_hosts), tagp)
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi director_user_tag_killed(dir, *tagp, username_hash);
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomidirector_user_tag_killed_everywhere(struct director *dir,
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi user = user_directory_lookup(tag->users, username_hash);
0874d7a4fa15322318c71291f0134ff7cc49bbb8Timo Sirainen dir_debug("User %u no longer exists - ignoring USER-KILLED-EVERYWHERE",
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen dir_debug("User %u is no longer being killed - ignoring USER-KILLED-EVERYWHERE",
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen if (user->kill_ctx->kill_state != USER_KILL_STATE_KILLED_WAITING_FOR_EVERYONE) {
0874d7a4fa15322318c71291f0134ff7cc49bbb8Timo Sirainen dir_debug("User %u kill_state=%s - ignoring USER-KILLED-EVERYWHERE",
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen username_hash, user_kill_state_names[user->kill_ctx->kill_state]);
fd882f791279d0bb0cbf15ccec75cff9f6b399ccTimo Sirainen director_send_user_killed_everywhere(dir, src, orig_src, username_hash);
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomivoid director_user_killed_everywhere(struct director *dir,
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi array_foreach(mail_hosts_get_tags(dir->mail_hosts), tagp) {
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi director_user_tag_killed_everywhere(dir, *tagp, src, orig_src,
3ee8a7ee6912c7caa4e83d3ce5a5db1590a7ffcdTimo Sirainenstatic void director_state_callback_timeout(struct director *dir)
3ee8a7ee6912c7caa4e83d3ce5a5db1590a7ffcdTimo Sirainenvoid director_set_state_changed(struct director *dir)
3ee8a7ee6912c7caa4e83d3ce5a5db1590a7ffcdTimo Sirainen /* we may get called to here from various places. use a timeout to
3ee8a7ee6912c7caa4e83d3ce5a5db1590a7ffcdTimo Sirainen make sure the state callback is called with a clean state. */
3ee8a7ee6912c7caa4e83d3ce5a5db1590a7ffcdTimo Sirainen timeout_add(0, director_state_callback_timeout, dir);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenvoid director_update_send(struct director *dir, struct director_host *src,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const char *cmd)
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen director_update_send_version(dir, src, 0, cmd);
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainenvoid director_update_send_version(struct director *dir,
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen if (director_connection_get_host(*connp) != src &&
e4194f4703eeec32b432371ae30fc8f25ab720d8Timo Sirainen director_connection_get_minor_version(*connp) >= min_version)
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainenstatic void director_user_freed(struct user *user)
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen /* director_user_expire is very short. user expired before
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen moving the user finished or timed out. */
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen /* kill_ctx is used as a callback parameter.
1b7cd57585d8c2f133dd612d2d5d9c775595659fTimo Sirainen only remove the timeout and finish the free later. */
0d1b8b6bec79746c5d89d57dd8c1688946bd9237Josef 'Jeff' Sipek timeout_remove(&user->kill_ctx->to_move);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainendirector_init(const struct director_settings *set,
009217abb57a24a4076092e8e4e165545747839eStephan Bosch const struct ip_addr *listen_ip, in_port_t listen_port,
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi dir->mail_hosts = mail_hosts_init(set->director_user_expire,
fb35b9f2c80954da842c20d5128b5e506835d93eTimo Sirainen dir->ipc_proxy = ipc_client_init(DIRECTOR_IPC_PROXY_PATH);
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen dir->ring_min_version = DIRECTOR_VERSION_MINOR;
caae18c876f81e261350e4957471efa453c0ea9fTimo Sirainen struct director_connection *conn, *const *connp;
dff32d11a411a24f3b76003c1ae22c5a960f180eTimo Sirainen director_connection_deinit(&conn, "Shutting down");
0d1b8b6bec79746c5d89d57dd8c1688946bd9237Josef 'Jeff' Sipek timeout_remove(&dir->to_handshake_warning);
14660f677e16a5c36f3c43e9e64f5e021fda627bTimo Sirainendirector_iterate_users_init(struct director *dir, bool iter_until_current_tail)
1f7f4294207557edf83171642ef62ce4922ffc9dTimo Sirainen struct director_user_iter *iter = i_new(struct director_user_iter, 1);
14660f677e16a5c36f3c43e9e64f5e021fda627bTimo Sirainen iter->iter_until_current_tail = iter_until_current_tail;
1f7f4294207557edf83171642ef62ce4922ffc9dTimo Sirainenstruct user *director_iterate_users_next(struct director_user_iter *iter)
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi tags = mail_hosts_get_tags(iter->dir->mail_hosts);
b44033e45e9f48f8a6e1ac5905234fec5de6d6ccAki Tuomi struct mail_tag *const *tagp = array_idx(tags, iter->tag_idx);
14660f677e16a5c36f3c43e9e64f5e021fda627bTimo Sirainen iter->user_iter = user_directory_iter_init((*tagp)->users,
1f7f4294207557edf83171642ef62ce4922ffc9dTimo Sirainenvoid director_iterate_users_deinit(struct director_user_iter **_iter)
de92873c366becfaea1554642f89b9169d7955e2Timo Sirainendirector_get_username_hash(struct director *dir, const char *username,
de92873c366becfaea1554642f89b9169d7955e2Timo Sirainen unsigned int *hash_r)
de92873c366becfaea1554642f89b9169d7955e2Timo Sirainen if (mail_user_hash(username, dir->set->director_username_hash, hash_r,
de92873c366becfaea1554642f89b9169d7955e2Timo Sirainen i_error("Failed to expand director_user_expire=%s: %s",
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen log_throttle_init(&director_log_throttle_settings,
c3a2a487e23a282e59254b82deb9344ed0306bb2Timo Sirainen log_throttle_init(&director_log_throttle_settings,