http-client.c revision 6ee9ce5ed955a1283dc22ad28980bf9cc23d4c4e
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield Acts much like a browser; it is not dedicated to a single host. Client can
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield accept requests to different hosts, which can be served at different IPs.
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield Redirects are handled in the background by making a new connection.
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield Connections to new hosts are created once needed for servicing a request.
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield http-client-request:
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield The request semantics are similar to imapc commands. Create a request,
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield optionally modify some aspects of it and finally submit it. Once finished,
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield a callback is called with the returned response.
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield http-client-host:
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield We maintain a 'cache' of hosts for which we have looked up IPs. One host
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield can have multiple IPs.
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield http-client-queue:
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield Requests are queued in a queue object. These queues are maintained for each
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield host:port target and listed in the host object. The queue object is
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield responsible for starting connection attempts to TCP port at the various IPs
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield known for the host.
826cde7c2100e1f4419a54b5c930c0854e01e87eMichael H. Warfield http-client-peer:
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield The peer object groups multiple connections to the same ip/port
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield (== peer_addr).
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield http-client-connection:
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield This is an actual connection to a server. Once a connection is ready to
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield handle requests, it claims a request from a queue object. One connection can
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield service multiple hosts and one host can have multiple associated connections,
b4f7af7a520b23c873e404562ec518a576e63d4cMichael H. Warfield possibly to different ips and ports.
826cde7c2100e1f4419a54b5c930c0854e01e87eMichael H. Warfieldstatic inline void
826cde7c2100e1f4419a54b5c930c0854e01e87eMichael H. Warfieldhttp_client_debug(struct http_client *client,
826cde7c2100e1f4419a54b5c930c0854e01e87eMichael H. Warfield const char *format, ...) ATTR_FORMAT(2, 3);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfieldstatic inline void
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfieldhttp_client_debug(struct http_client *client,
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield const char *format, ...)
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield i_debug("http-client: %s", t_strdup_vprintf(format, args));
8ec981fc8b0105da5f071e40811e0c2472a6c3c9Stéphane Graberstruct http_client *http_client_init(const struct http_client_settings *set)
207bf0e475f1dc6e9a2dac2cee3a209b56427855Stéphane Graber pool = pool_alloconly_create("http client", 1024);
207bf0e475f1dc6e9a2dac2cee3a209b56427855Stéphane Graber client = p_new(pool, struct http_client, 1);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.dns_client = set->dns_client;
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield p_strdup_empty(pool, set->dns_client_socket_path);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.user_agent = p_strdup_empty(pool, set->user_agent);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.rawlog_dir = p_strdup_empty(pool, set->rawlog_dir);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.ssl_ca_dir = p_strdup(pool, set->ssl_ca_dir);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.ssl_ca_file = p_strdup(pool, set->ssl_ca_file);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.ssl_ca = p_strdup(pool, set->ssl_ca);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.ssl_crypto_device = p_strdup(pool, set->ssl_crypto_device);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.ssl_allow_invalid_cert = set->ssl_allow_invalid_cert;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.ssl_cert = p_strdup(pool, set->ssl_cert);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.ssl_key = p_strdup(pool, set->ssl_key);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.ssl_key_password = p_strdup(pool, set->ssl_key_password);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield if (set->proxy_socket_path != NULL && *set->proxy_socket_path != '\0') {
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.proxy_socket_path = p_strdup(pool, set->proxy_socket_path);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->set.proxy_url = http_url_clone(pool, set->proxy_url);
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.proxy_username = p_strdup_empty(pool, set->proxy_username);
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.proxy_password = p_strdup_empty(pool, set->proxy_password);
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.max_idle_time_msecs = set->max_idle_time_msecs;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield (set->max_parallel_connections > 0 ? set->max_parallel_connections : 1);
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1);
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.max_attempts = set->max_attempts;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.no_auto_redirect = set->no_auto_redirect;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.no_ssl_tunnel = set->no_ssl_tunnel;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.max_redirects = set->max_redirects;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.response_hdr_limits = set->response_hdr_limits;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.request_timeout_msecs = set->request_timeout_msecs;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.connect_timeout_msecs = set->connect_timeout_msecs;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield client->set.max_auto_retry_delay = set->max_auto_retry_delay;
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield client->conn_list = http_client_connection_list_init();
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield hash_table_create(&client->hosts, default_pool, 0, str_hash, strcmp);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield hash_table_create(&client->peers, default_pool, 0,
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield http_client_peer_addr_hash, http_client_peer_addr_cmp);
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfieldvoid http_client_deinit(struct http_client **_client)
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield /* free peers */
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield /* free hosts */
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield connection_list_deinit(&client->conn_list);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield ssl_iostream_context_deinit(&client->ssl_ctx);
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfieldvoid http_client_switch_ioloop(struct http_client *client)
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield struct connection *_conn = client->conn_list->connections;
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield /* move connections */
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield /* FIXME: we wouldn't necessarily need to switch all of them
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield immediately, only those that have requests now. but also connections
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield that get new requests before ioloop is switched again.. */
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield for (; _conn != NULL; _conn = _conn->next) {
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield http_client_connection_switch_ioloop(conn);
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield /* move peers */
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield for (peer = client->peers_list; peer != NULL; peer = peer->next)
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield /* move dns lookups and delayed requests */
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield for (host = client->hosts_list; host != NULL; host = host->next)
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfieldvoid http_client_wait(struct http_client *client)
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield struct ioloop *prev_ioloop = current_ioloop;
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield dns_client_switch_ioloop(client->set.dns_client);
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield /* either we're waiting for network I/O or we're getting out of a
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield callback using timeout_add_short(0) */
c6df5ca4603c630a7189cdb1653c96bd2808c7e5Michael H. Warfield i_assert(io_loop_have_ios(client->ioloop) ||
f5067ecbcc1e97052c33269b4afa6375073a91a1Michael H. Warfield io_loop_have_immediate_timeouts(client->ioloop));
f5067ecbcc1e97052c33269b4afa6375073a91a1Michael H. Warfield "Waiting for %d requests to finish", client->pending_requests);
99c2fb07d74c20d0eec38c05c4ac64e5782d8e7dMichael H. Warfield http_client_debug(client, "All requests finished");
99c2fb07d74c20d0eec38c05c4ac64e5782d8e7dMichael H. Warfield dns_client_switch_ioloop(client->set.dns_client);
99c2fb07d74c20d0eec38c05c4ac64e5782d8e7dMichael H. Warfieldunsigned int http_client_get_pending_request_count(struct http_client *client)
99c2fb07d74c20d0eec38c05c4ac64e5782d8e7dMichael H. Warfieldint http_client_init_ssl_ctx(struct http_client *client, const char **error_r)
99c2fb07d74c20d0eec38c05c4ac64e5782d8e7dMichael H. Warfield ssl_set.ca_dir = client->set.ssl_ca_dir;
99c2fb07d74c20d0eec38c05c4ac64e5782d8e7dMichael H. Warfield ssl_set.ca_file = client->set.ssl_ca_file;
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield ssl_set.crypto_device = client->set.ssl_crypto_device;
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield ssl_set.key_password = client->set.ssl_key_password;
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield ssl_set.verbose_invalid_cert = client->set.debug;
164105f6563d98b832f603e28e506dbabed22cf3Michael H. Warfield if (ssl_iostream_context_init_client(&ssl_set, &client->ssl_ctx, &error) < 0) {
91c5c53f59262704538a7b8f9571d626e4e08011Kalman Olah *error_r = t_strdup_printf("Couldn't initialize SSL context: %s",