director.c revision b6b9c99fefbbc662bd9a0006566133c4480bf0e8
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen#define DIRECTOR_RECONNECT_TIMEOUT_MSECS (30*1000)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen#define DIRECTOR_USER_MOVE_TIMEOUT_MSECS (30*1000)
5411e6f71afb0f753cdf831f4da4ee73e928796eTimo Sirainen#define DIRECTOR_USER_MOVE_FINISH_DELAY_MSECS (2*1000)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic bool director_is_self_ip_set(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic void director_find_self_ip(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen unsigned int i, count;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; i < count; i++) {
a356c4736fe6041142c6096045bc00c15a80af4eTimo Sirainen i_fatal("director_servers doesn't list ourself");
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen dir->self_host = director_host_lookup(dir, &dir->self_ip,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i_fatal("director_servers doesn't list ourself (%s:%u)",
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic unsigned int director_find_self_idx(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen unsigned int i, count;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; i < count; i++) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenint director_connect_host(struct director *dir, struct director_host *host)
00e7c3010f7da4a49881a7feb05e413af353af0aTimo Sirainen unsigned int port;
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen if (director_connection_find_outgoing(dir, host) != NULL)
00e7c3010f7da4a49881a7feb05e413af353af0aTimo Sirainen port = dir->test_port != 0 ? dir->test_port : host->port;
00e7c3010f7da4a49881a7feb05e413af353af0aTimo Sirainen fd = net_connect_ip(&host->ip, port, &dir->self_ip);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i_error("connect(%s) failed: %m", host->name);
4c20178a7f70bfe43d252e50796013aac1d8c74aTimo Sirainen /* Reset timestamp so that director_connect() won't skip this host
4c20178a7f70bfe43d252e50796013aac1d8c74aTimo Sirainen while we're still trying to connect to it */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainendirector_get_preferred_right_host(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* try to connect to first working server on our right side.
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen the left side is supposed to connect to us. */
8e1ecc6542da1e14c14e2e59d39dbccdbf68e2b5Timo Sirainen DIRECTOR_RECONNECT_RETRY_SECS > ioloop_time) {
8e1ecc6542da1e14c14e2e59d39dbccdbf68e2b5Timo Sirainen /* failed recently, don't try retrying here */
4fe3f07477bae6da3fb8d8fa9bab10ab82ada2bdTimo Sirainen if (director_connect_host(dir, hosts[idx]) == 0)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* we're the only one */
4e43828ef88183a0750a8a374b6ba4ecf227c58fTimo Sirainen i_debug("director: Couldn't connect to right side, "
4e43828ef88183a0750a8a374b6ba4ecf227c58fTimo Sirainen "we must be the only director left");
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* since we couldn't connect to it,
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen it must have failed recently */
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen dir->ring_min_version = DIRECTOR_VERSION_MINOR;
4aab01f4eade3d278b61471516c062ce30a84b5fTimo Sirainenvoid director_set_ring_handshaked(struct director *dir)
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen "continuing delayed requests");
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenstatic void director_reconnect_timeout(struct director *dir)
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen struct director_host *cur_host, *preferred_host =
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen (void)director_connect_host(dir, preferred_host);
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* the connection hasn't finished sync yet.
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen keep this timeout for now. */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenvoid director_set_ring_synced(struct director *dir)
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen i_assert((dir->left != NULL && dir->right != NULL) ||
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen i_warning("Ring is synced, continuing delayed requests");
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen if (host != director_get_preferred_right_host(dir)) {
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* try to reconnect to preferred host later */
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainenvoid director_sync_send(struct director *dir, struct director_host *host,
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen /* only minor_version>0 supports this parameter */
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen director_connection_send(dir->right, str_c(str));
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainenbool director_resend_sync(struct director *dir)
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainen if (!dir->ring_synced && dir->left != NULL && dir->right != NULL) {
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainen /* send a new SYNC in case the previous one got dropped */
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen director_sync_send(dir, dir->self_host, dir->sync_seq,
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainenstatic void director_sync_timeout(struct director *dir)
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainen i_error("Ring SYNC appears to have got lost, resending");
152db3f90f298b7fb2dbbd4276f0fc30a9bc30f6Timo Sirainenvoid director_set_ring_unsynced(struct director *dir)
433f5c9cc560a8cbff47257513d0bacb1cf250f4Timo Sirainen dir->to_sync = timeout_add(DIRECTOR_SYNC_TIMEOUT_MSECS,
242abe3ad2423776e9cf05e1304eb8fda4831b23Timo Sirainenstatic void director_sync(struct director *dir)
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* we're synced again when we receive this SYNC back */
242abe3ad2423776e9cf05e1304eb8fda4831b23Timo Sirainen i_debug("Ring is desynced (seq=%u, sending SYNC to %s)",
883c882ede03123e771e5f1cd1d4fbdbddafd6c2Timo Sirainen dir->sync_seq, dir->right == NULL ? "(nowhere)" :
aa797403d51ff047727b77d64532001d6b6cc21aTimo Sirainen director_sync_send(dir, dir->self_host, dir->sync_seq,
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainenvoid director_sync_freeze(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenvoid director_update_host(struct director *dir, struct director_host *src,
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* update state in case this is the first mail host being added */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen director_update_send(dir, src, t_strdup_printf(
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen "HOST\t%s\t%u\t%u\t%s\t%u\n",
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenvoid director_remove_host(struct director *dir, struct director_host *src,
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen director_update_send(dir, src, t_strdup_printf(
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen "HOST-REMOVE\t%s\t%u\t%u\t%s\n",
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainenvoid director_flush_host(struct director *dir, struct director_host *src,
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen director_update_send(dir, src, t_strdup_printf(
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen "HOST-FLUSH\t%s\t%u\t%u\t%s\n",
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenvoid director_update_user(struct director *dir, struct director_host *src,
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen director_update_send(dir, src, t_strdup_printf("USER\t%u\t%s\n",
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen user->username_hash, net_ip2addr(&user->host->ip)));
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainenvoid director_update_user_weak(struct director *dir, struct director_host *src,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen director_update_send(dir, src, t_strdup_printf(
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen "USER-WEAK\t%s\t%u\t%u\t%u\t%s\n",
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen user->username_hash, net_ip2addr(&user->host->ip)));
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainendirector_user_kill_finish_delayed_to(struct director_user_kill_finish_ctx *ctx)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen i_assert(ctx->user->kill_state == USER_KILL_STATE_DELAY);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainendirector_user_kill_finish_delayed(struct director *dir, struct user *user)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen ctx = i_new(struct director_user_kill_finish_ctx, 1);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user->to_move = timeout_add(DIRECTOR_USER_MOVE_FINISH_DELAY_MSECS,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainendirector_finish_user_kill(struct director *dir, struct user *user, bool self)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen if (dir->right == NULL || dir->right == dir->left) {
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen /* we're alone */
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen } else if (self ||
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user->kill_state == USER_KILL_STATE_KILLING_NOTIFY_RECEIVED) {
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_connection_send(dir->right, t_strdup_printf(
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user->kill_state = USER_KILL_STATE_KILLED_WAITING_FOR_EVERYONE;
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen i_assert(user->kill_state == USER_KILL_STATE_KILLING);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user->kill_state = USER_KILL_STATE_KILLED_WAITING_FOR_NOTIFY;
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenstatic void director_kill_user_callback(enum ipc_client_cmd_state state,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen i_error("Failed to kill user %u connections: %s",
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen /* we can't really do anything but continue anyway */
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user = user_directory_lookup(ctx->dir->users, ctx->username_hash);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen if (user == NULL || user->kill_state == USER_KILL_STATE_NONE)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_finish_user_kill(ctx->dir, user, ctx->self);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenstatic void director_user_move_timeout(struct user *user)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen "its state may now be inconsistent", user->username_hash);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenvoid director_move_user(struct director *dir, struct director_host *src,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen unsigned int username_hash, struct mail_host *host)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen const char *cmd;
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen /* 1. move this user's host, and set its "killing" flag to delay all of
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen its future connections until all directors have killed the
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen connections and notified us about it.
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 2. tell the other directors about the move
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 3. once user kill callback is called, tell the other directors
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen with USER-KILLED that we're done killing the user.
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 4. when some director gets a duplicate USER-KILLED, it's
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen responsible for notifying all directors that user is completely
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen 5. after receiving USER-KILLED-EVERYWHERE notification,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen new connections are again allowed for the user.
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user = user_directory_lookup(dir->users, username_hash);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user = user_directory_add(dir->users, username_hash,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen /* user is already in this host */
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen if (user->kill_state == USER_KILL_STATE_NONE) {
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user->to_move = timeout_add(DIRECTOR_USER_MOVE_TIMEOUT_MSECS,
ac1118842c3d80285e32d2cd53bda3e95e5be217Timo Sirainen cmd = t_strdup_printf("proxy\t*\tKICK-DIRECTOR-HASH\t%u",
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_update_send(dir, src, t_strdup_printf(
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen "USER-MOVE\t%s\t%u\t%u\t%u\t%s\n",
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user->username_hash, net_ip2addr(&user->host->ip)));
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenvoid director_user_killed(struct director *dir, unsigned int username_hash)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user = user_directory_lookup(dir->users, username_hash);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user->kill_state = USER_KILL_STATE_KILLING_NOTIFY_RECEIVED;
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen case USER_KILL_STATE_KILLED_WAITING_FOR_NOTIFY:
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen case USER_KILL_STATE_KILLED_WAITING_FOR_EVERYONE:
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_user_killed_everywhere(dir, dir->self_host,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainenvoid director_user_killed_everywhere(struct director *dir,
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user = user_directory_lookup(dir->users, username_hash);
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen user->kill_state != USER_KILL_STATE_KILLED_WAITING_FOR_EVERYONE)
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen director_update_send(dir, src, t_strdup_printf(
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen "USER-KILLED-EVERYWHERE\t%s\t%u\t%u\t%u\n",
15ab2452b0220a115f4351ad9d7fd5ec70ae7966Timo Sirainen net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenvoid director_set_state_changed(struct director *dir)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenvoid director_update_send(struct director *dir, struct director_host *src,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const char *cmd)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen director_connection_send_except(dir->left, src, cmd);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (dir->right != NULL && dir->right != dir->left)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen director_connection_send_except(dir->right, src, cmd);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainendirector_init(const struct director_settings *set,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const struct ip_addr *listen_ip, unsigned int listen_port,
5733207dc3ec10e6e5a6e0a8b30fbd1b061062b9Timo Sirainen dir->users = user_directory_init(set->director_user_expire,
fb35b9f2c80954da842c20d5128b5e506835d93eTimo Sirainen dir->ipc_proxy = ipc_client_init(DIRECTOR_IPC_PROXY_PATH);