http-client.c revision 2454dfa32c93c20a8522c6ed42fe057baaac9f9a
2454dfa32c93c20a8522c6ed42fe057baaac9f9aStephan Bosch/* Copyright (c) 2013-2017 Dovecot authors, see the included COPYING file */
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch Acts much like a browser; it is not dedicated to a single host. Client can
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch accept requests to different hosts, which can be served at different IPs.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch Redirects are handled in the background by making a new connection.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch Connections to new hosts are created once needed for servicing a request.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch http-client-request:
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch The request semantics are similar to imapc commands. Create a request,
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch optionally modify some aspects of it and finally submit it. Once finished,
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch a callback is called with the returned response.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch http-client-host:
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch We maintain a 'cache' of hosts for which we have looked up IPs. One host
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch can have multiple IPs.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch http-client-queue:
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch Requests are queued in a queue object. These queues are maintained for each
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch host:port target and listed in the host object. The queue object is
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch responsible for starting connection attempts to TCP port at the various IPs
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch known for the host.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch http-client-peer:
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch The peer object groups multiple connections to the same ip/port
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch (== peer_addr).
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch http-client-connection:
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch This is an actual connection to a server. Once a connection is ready to
6ee9ce5ed955a1283dc22ad28980bf9cc23d4c4eStephan Bosch handle requests, it claims a request from a queue object. One connection can
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch service multiple hosts and one host can have multiple associated connections,
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch possibly to different ips and ports.
7384b4e78eaab44693c985192276e31322155e32Stephan Boschstatic inline void
7384b4e78eaab44693c985192276e31322155e32Stephan Boschstatic inline void
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch const char *format, ...)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_debug("http-client: %s", t_strdup_vprintf(format, args));
7384b4e78eaab44693c985192276e31322155e32Stephan Boschstruct http_client *http_client_init(const struct http_client_settings *set)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch pool = pool_alloconly_create("http client", 1024);
b99130e4cf4af4e6b103b949456222f3a2dff424Timo Sirainen p_strdup_empty(pool, set->dns_client_socket_path);
1e63e30812158e6446d81cdbb2f45954794d4f8aStephan Bosch client->set.dns_ttl_msecs = (set->dns_ttl_msecs == 0 ?
1e63e30812158e6446d81cdbb2f45954794d4f8aStephan Bosch HTTP_CLIENT_DEFAULT_DNS_TTL_MSECS : set->dns_ttl_msecs);
b99130e4cf4af4e6b103b949456222f3a2dff424Timo Sirainen client->set.user_agent = p_strdup_empty(pool, set->user_agent);
b99130e4cf4af4e6b103b949456222f3a2dff424Timo Sirainen client->set.rawlog_dir = p_strdup_empty(pool, set->rawlog_dir);
5d31e4b38ef03b002e2ab245a7f8a4c0da3dd03dTimo Sirainen client->set.ssl = ssl_iostream_settings_dup(client->pool, set->ssl);
4219de12b28f1936219e27501b9c4b27a4f8d53cStephan Bosch if (set->proxy_socket_path != NULL && *set->proxy_socket_path != '\0') {
4219de12b28f1936219e27501b9c4b27a4f8d53cStephan Bosch client->set.proxy_socket_path = p_strdup(pool, set->proxy_socket_path);
4219de12b28f1936219e27501b9c4b27a4f8d53cStephan Bosch client->set.proxy_url = http_url_clone(pool, set->proxy_url);
4c4c4a740bbb1b674d4b0dae009d1919f8ad96b7Stephan Bosch if (set->proxy_username != NULL && *set->proxy_username != '\0') {
4c4c4a740bbb1b674d4b0dae009d1919f8ad96b7Stephan Bosch client->set.proxy_username = p_strdup_empty(pool, set->proxy_username);
4c4c4a740bbb1b674d4b0dae009d1919f8ad96b7Stephan Bosch client->set.proxy_password = p_strdup(pool, set->proxy_password);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch client->set.max_idle_time_msecs = set->max_idle_time_msecs;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch (set->max_parallel_connections > 0 ? set->max_parallel_connections : 1);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1);
9d746c6785d17e421c3f3c74cf29d059ae2ab233Stephan Bosch client->set.max_connect_attempts = set->max_connect_attempts;
ad03049781fc14807248007d524be4daf06c3ee2Stephan Bosch client->set.no_auto_redirect = set->no_auto_redirect;
486c7c8d9e725e0227c7723aa43b7fce724eb9eeStephan Bosch client->set.no_auto_retry = set->no_auto_retry;
e47c2f17d8136c4d972d1074a3f84ba2ecef4fdcStephan Bosch client->set.no_ssl_tunnel = set->no_ssl_tunnel;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch client->set.max_redirects = set->max_redirects;
feba5e502b2131c9a1c766b7ef9ff041dbf71d1dStephan Bosch client->set.response_hdr_limits = set->response_hdr_limits;
a8c4e79ff50fac21b05a7368b052583d410ca15cTimo Sirainen client->set.connect_timeout_msecs = set->connect_timeout_msecs;
70505f4839520ac67895992621c97d2480c22e7fTimo Sirainen client->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs;
93cc87bb22386e020cee1093b6bd59295e0b33f0Stephan Bosch client->set.max_auto_retry_delay = set->max_auto_retry_delay;
c972eaa3565e849df71b44cf0cd45d38c5567d07Stephan Bosch client->set.socket_send_buffer_size = set->socket_send_buffer_size;
c972eaa3565e849df71b44cf0cd45d38c5567d07Stephan Bosch client->set.socket_recv_buffer_size = set->socket_recv_buffer_size;
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch i_array_init(&client->delayed_failing_requests, 1);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch client->conn_list = http_client_connection_list_init();
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch hash_table_create(&client->hosts, default_pool, 0, str_hash, strcmp);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch hash_table_create(&client->peers, default_pool, 0,
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch http_client_peer_addr_hash, http_client_peer_addr_cmp);
7384b4e78eaab44693c985192276e31322155e32Stephan Boschvoid http_client_deinit(struct http_client **_client)
6ab81c81be13f33486746deeffe02a1ef2bcc821Stephan Bosch /* destroy requests without calling callbacks */
6ab81c81be13f33486746deeffe02a1ef2bcc821Stephan Bosch struct http_client_request *next_req = req->next;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch /* free peers */
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch /* free hosts */
6ab81c81be13f33486746deeffe02a1ef2bcc821Stephan Bosch array_free(&client->delayed_failing_requests);
415e16c3dc185578695b7d88e561a52de6c8b1b1Timo Sirainen ssl_iostream_context_deinit(&client->ssl_ctx);
7384b4e78eaab44693c985192276e31322155e32Stephan Boschvoid http_client_switch_ioloop(struct http_client *client)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct connection *_conn = client->conn_list->connections;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch /* move connections */
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch /* FIXME: we wouldn't necessarily need to switch all of them
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch immediately, only those that have requests now. but also connections
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch that get new requests before ioloop is switched again.. */
fc94140acba51adafedafbc8491a3223a51db7a8Stephan Bosch /* move peers */
fc94140acba51adafedafbc8491a3223a51db7a8Stephan Bosch for (peer = client->peers_list; peer != NULL; peer = peer->next)
636d0f43138468f8efe685a681326b123f660e49Timo Sirainen /* move dns lookups and delayed requests */
636d0f43138468f8efe685a681326b123f660e49Timo Sirainen for (host = client->hosts_list; host != NULL; host = host->next)
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch /* move timeouts */
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch io_loop_move_timeout(&client->to_failing_requests);
7384b4e78eaab44693c985192276e31322155e32Stephan Boschvoid http_client_wait(struct http_client *client)
856ae2ad98cee79b2719911a3cc131d7f4ec8a90Timo Sirainen dns_client_switch_ioloop(client->set.dns_client);
1a9a35a6b307f8d5b25345af55e40a99162b4072Timo Sirainen /* either we're waiting for network I/O or we're getting out of a
1a9a35a6b307f8d5b25345af55e40a99162b4072Timo Sirainen callback using timeout_add_short(0) */
1a9a35a6b307f8d5b25345af55e40a99162b4072Timo Sirainen io_loop_have_immediate_timeouts(client->ioloop));
a991cfe2157e58ee43bc580f517ce9ef0dfb7acfStephan Bosch "Waiting for %d requests to finish", client->requests_count);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch http_client_debug(client, "All requests finished");
856ae2ad98cee79b2719911a3cc131d7f4ec8a90Timo Sirainen dns_client_switch_ioloop(client->set.dns_client);
17cd0e0963f2fb0e66d49703e8cd0bda1b842468Timo Sirainenunsigned int http_client_get_pending_request_count(struct http_client *client)
415e16c3dc185578695b7d88e561a52de6c8b1b1Timo Sirainenint http_client_init_ssl_ctx(struct http_client *client, const char **error_r)
5d31e4b38ef03b002e2ab245a7f8a4c0da3dd03dTimo Sirainen *error_r = "Requested https connection, but no SSL settings given";
c275cef636c79e1d08b3a82462c4abdca6f8cef3Martti Rannanjärvi if (ssl_iostream_context_init_client(client->set.ssl, &client->ssl_ctx, &error) < 0) {
ba1c847d0af4afe4787ed470d0c818e948e184e2Timo Sirainen *error_r = t_strdup_printf("Couldn't initialize SSL context: %s",
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch * Delayed request errors
fca68889b287d8eed4babe72a231bd6079da012dStephan Boschhttp_client_handle_request_errors(struct http_client *client)
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch while (array_count(&client->delayed_failing_requests) > 0) {
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch array_idx(&client->delayed_failing_requests, 0);
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch array_clear(&client->delayed_failing_requests);
fca68889b287d8eed4babe72a231bd6079da012dStephan Boschvoid http_client_delay_request_error(struct http_client *client,
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch client->to_failing_requests = timeout_add_short(0,
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch array_append(&client->delayed_failing_requests, &req, 1);
fca68889b287d8eed4babe72a231bd6079da012dStephan Boschvoid http_client_remove_request_error(struct http_client *client,
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch unsigned int i, count;
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch reqs = array_get(&client->delayed_failing_requests, &count);
fca68889b287d8eed4babe72a231bd6079da012dStephan Bosch for (i = 0; i < count; i++) {