http-client.c revision b66def5dadd3e7c250313a938d26ad113663f86b
a8c5a86d183db25a57bf193c06b41e092ec2e151Timo Sirainen/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch Acts much like a browser; it is not dedicated to a single host. Client can
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch accept requests to different hosts, which can be served at different IPs.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch Redirects are handled in the background by making a new connection.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch Connections to new hosts are created once needed for servicing a request.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch http-client-request:
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch The request semantics are similar to imapc commands. Create a request,
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch optionally modify some aspects of it and finally submit it. Once finished,
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch a callback is called with the returned response.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch http-client-host:
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch We maintain a 'cache' of hosts for which we have looked up IPs. One host
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch can have multiple IPs.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch http-client-queue:
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch Requests are queued in a queue object. These queues are maintained for each
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch host:port target and listed in the host object. The queue object is
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch responsible for starting connection attempts to TCP port at the various IPs
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch known for the host.
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch http-client-peer:
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch The peer object groups multiple connections to the same ip/port
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch (== peer_addr).
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch http-client-connection:
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch This is an actual connection to a server. Once a connection is ready to
6ee9ce5ed955a1283dc22ad28980bf9cc23d4c4eStephan Bosch handle requests, it claims a request from a queue object. One connection can
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch service multiple hosts and one host can have multiple associated connections,
de96afeeaa5242cffe89f1966457e935806b5746Stephan Bosch possibly to different ips and ports.
7384b4e78eaab44693c985192276e31322155e32Stephan Boschstatic inline void
7384b4e78eaab44693c985192276e31322155e32Stephan Boschstatic inline void
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch const char *format, ...)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_debug("http-client: %s", t_strdup_vprintf(format, args));
7384b4e78eaab44693c985192276e31322155e32Stephan Boschstruct http_client *http_client_init(const struct http_client_settings *set)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch pool = pool_alloconly_create("http client", 1024);
b99130e4cf4af4e6b103b949456222f3a2dff424Timo Sirainen p_strdup_empty(pool, set->dns_client_socket_path);
b99130e4cf4af4e6b103b949456222f3a2dff424Timo Sirainen client->set.user_agent = p_strdup_empty(pool, set->user_agent);
b99130e4cf4af4e6b103b949456222f3a2dff424Timo Sirainen client->set.rawlog_dir = p_strdup_empty(pool, set->rawlog_dir);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch client->set.ssl_ca_dir = p_strdup(pool, set->ssl_ca_dir);
56d1345c43bbd28c36b7faa85e4163bd9e874290Timo Sirainen client->set.ssl_ca_file = p_strdup(pool, set->ssl_ca_file);
30d917bcd48d70af0371baf27571cc198d621a62Timo Sirainen client->set.ssl_ca = p_strdup(pool, set->ssl_ca);
9d0aee99a8c80d71137aa9b8c216cc203bec7a9aTimo Sirainen client->set.ssl_crypto_device = p_strdup(pool, set->ssl_crypto_device);
9d0aee99a8c80d71137aa9b8c216cc203bec7a9aTimo Sirainen client->set.ssl_allow_invalid_cert = set->ssl_allow_invalid_cert;
35e962a9186b4e9b2001628c1d7b55c24b33ce84Timo Sirainen client->set.ssl_cert = p_strdup(pool, set->ssl_cert);
35e962a9186b4e9b2001628c1d7b55c24b33ce84Timo Sirainen client->set.ssl_key = p_strdup(pool, set->ssl_key);
35e962a9186b4e9b2001628c1d7b55c24b33ce84Timo Sirainen client->set.ssl_key_password = p_strdup(pool, set->ssl_key_password);
4219de12b28f1936219e27501b9c4b27a4f8d53cStephan Bosch if (set->proxy_socket_path != NULL && *set->proxy_socket_path != '\0') {
4219de12b28f1936219e27501b9c4b27a4f8d53cStephan Bosch client->set.proxy_socket_path = p_strdup(pool, set->proxy_socket_path);
4219de12b28f1936219e27501b9c4b27a4f8d53cStephan Bosch client->set.proxy_url = http_url_clone(pool, set->proxy_url);
4219de12b28f1936219e27501b9c4b27a4f8d53cStephan Bosch client->set.proxy_username = p_strdup_empty(pool, set->proxy_username);
4219de12b28f1936219e27501b9c4b27a4f8d53cStephan Bosch client->set.proxy_password = p_strdup_empty(pool, set->proxy_password);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch client->set.max_idle_time_msecs = set->max_idle_time_msecs;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch (set->max_parallel_connections > 0 ? set->max_parallel_connections : 1);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1);
9d746c6785d17e421c3f3c74cf29d059ae2ab233Stephan Bosch client->set.max_connect_attempts = set->max_connect_attempts;
ad03049781fc14807248007d524be4daf06c3ee2Stephan Bosch client->set.no_auto_redirect = set->no_auto_redirect;
e47c2f17d8136c4d972d1074a3f84ba2ecef4fdcStephan Bosch client->set.no_ssl_tunnel = set->no_ssl_tunnel;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch client->set.max_redirects = set->max_redirects;
feba5e502b2131c9a1c766b7ef9ff041dbf71d1dStephan Bosch client->set.response_hdr_limits = set->response_hdr_limits;
a8c4e79ff50fac21b05a7368b052583d410ca15cTimo Sirainen client->set.request_timeout_msecs = set->request_timeout_msecs;
a8c4e79ff50fac21b05a7368b052583d410ca15cTimo Sirainen client->set.connect_timeout_msecs = set->connect_timeout_msecs;
70505f4839520ac67895992621c97d2480c22e7fTimo Sirainen client->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs;
93cc87bb22386e020cee1093b6bd59295e0b33f0Stephan Bosch client->set.max_auto_retry_delay = set->max_auto_retry_delay;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch client->conn_list = http_client_connection_list_init();
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch hash_table_create(&client->hosts, default_pool, 0, str_hash, strcmp);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch hash_table_create(&client->peers, default_pool, 0,
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch http_client_peer_addr_hash, http_client_peer_addr_cmp);
7384b4e78eaab44693c985192276e31322155e32Stephan Boschvoid http_client_deinit(struct http_client **_client)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch /* free peers */
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch /* free hosts */
415e16c3dc185578695b7d88e561a52de6c8b1b1Timo Sirainen ssl_iostream_context_deinit(&client->ssl_ctx);
7384b4e78eaab44693c985192276e31322155e32Stephan Boschvoid http_client_switch_ioloop(struct http_client *client)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct connection *_conn = client->conn_list->connections;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch /* move connections */
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch /* FIXME: we wouldn't necessarily need to switch all of them
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch immediately, only those that have requests now. but also connections
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch that get new requests before ioloop is switched again.. */
fc94140acba51adafedafbc8491a3223a51db7a8Stephan Bosch /* move peers */
fc94140acba51adafedafbc8491a3223a51db7a8Stephan Bosch for (peer = client->peers_list; peer != NULL; peer = peer->next)
636d0f43138468f8efe685a681326b123f660e49Timo Sirainen /* move dns lookups and delayed requests */
636d0f43138468f8efe685a681326b123f660e49Timo Sirainen for (host = client->hosts_list; host != NULL; host = host->next)
7384b4e78eaab44693c985192276e31322155e32Stephan Boschvoid http_client_wait(struct http_client *client)
856ae2ad98cee79b2719911a3cc131d7f4ec8a90Timo Sirainen dns_client_switch_ioloop(client->set.dns_client);
1a9a35a6b307f8d5b25345af55e40a99162b4072Timo Sirainen /* either we're waiting for network I/O or we're getting out of a
1a9a35a6b307f8d5b25345af55e40a99162b4072Timo Sirainen callback using timeout_add_short(0) */
1a9a35a6b307f8d5b25345af55e40a99162b4072Timo Sirainen io_loop_have_immediate_timeouts(client->ioloop));
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch "Waiting for %d requests to finish", client->pending_requests);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch http_client_debug(client, "All requests finished");
856ae2ad98cee79b2719911a3cc131d7f4ec8a90Timo Sirainen dns_client_switch_ioloop(client->set.dns_client);
17cd0e0963f2fb0e66d49703e8cd0bda1b842468Timo Sirainenunsigned int http_client_get_pending_request_count(struct http_client *client)
415e16c3dc185578695b7d88e561a52de6c8b1b1Timo Sirainenint http_client_init_ssl_ctx(struct http_client *client, const char **error_r)
415e16c3dc185578695b7d88e561a52de6c8b1b1Timo Sirainen ssl_set.crypto_device = client->set.ssl_crypto_device;
35e962a9186b4e9b2001628c1d7b55c24b33ce84Timo Sirainen ssl_set.key_password = client->set.ssl_key_password;
d47b9f1bd7274c7b2d9049c2e1718d1cf89cc572Timo Sirainen ssl_set.verbose_invalid_cert = client->set.debug;
ba1c847d0af4afe4787ed470d0c818e948e184e2Timo Sirainen if (ssl_iostream_context_init_client(&ssl_set, &client->ssl_ctx, &error) < 0) {
ba1c847d0af4afe4787ed470d0c818e948e184e2Timo Sirainen *error_r = t_strdup_printf("Couldn't initialize SSL context: %s",