src/lib-http/http-client.c

	http-client.c revision 9fe6a55877bee691b32c12c7be56242054841670
/* Copyright (c) 2013-2017 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "net.h"
#include "str.h"
#include "hash.h"
#include "array.h"
#include "ioloop.h"
#include "istream.h"
#include "ostream.h"
#include "connection.h"
#include "dns-lookup.h"
#include "iostream-rawlog.h"
#include "iostream-ssl.h"
#include "http-url.h"

#include "http-client-private.h"

#define HTTP_DEFAULT_PORT 80
#define HTTPS_DEFAULT_PORT 443

/* Structure:

   http-client:

   Acts much like a browser; it is not dedicated to a single host. Client can
   accept requests to different hosts, which can be served at different IPs.
   Redirects are handled in the background by making a new connection.
   Connections to new hosts are created once needed for servicing a request.

   http-client-request:

   The request semantics are similar to imapc commands. Create a request,
   optionally modify some aspects of it and finally submit it. Once finished,
   a callback is called with the returned response.

   http-client-host:

   We maintain a 'cache' of hosts for which we have looked up IPs. One host
   can have multiple IPs.

   http-client-queue:

   Requests are queued in a queue object. These queues are maintained for each
   host:port target and listed in the host object. The queue object is
   responsible for starting connection attempts to TCP port at the various IPs
   known for the host.

   http-client-peer:

   The peer object groups multiple connections to the same ip/port
   (== peer_addr).

   http-client-connection:

   This is an actual connection to a server. Once a connection is ready to
   handle requests, it claims a request from a queue object. One connection can
   service multiple hosts and one host can have multiple associated connections,
   possibly to different ips and ports.

 */

/*
 * Logging
 */

static inline void
http_client_debug(struct http_client *client,
    const char *format, ...) ATTR_FORMAT(2, 3);

static inline void
http_client_debug(struct http_client *client,
    const char *format, ...)
{
    va_list args;

    va_start(args, format);
    if (client->set.debug)
        i_debug("http-client: %s", t_strdup_vprintf(format, args));
    va_end(args);
}

/*
 * Client
 */

struct http_client *
http_client_init_shared(struct http_client_context *cctx,
    const struct http_client_settings *set)
{
    struct http_client *client;
    pool_t pool;

    pool = pool_alloconly_create("http client", 1024);
    client = p_new(pool, struct http_client, 1);
    client->pool = pool;

    /* create private context if none is provided */
    if (cctx != NULL) {
        client->cctx = cctx;
        http_client_context_ref(cctx);
    } else {
        client->cctx = cctx = http_client_context_create(set);
    }

    /* merge provided settings with context defaults */
    i_assert(cctx != NULL || set != NULL);
    client->set = cctx->set;
    if (cctx != NULL && set != NULL) {
        if (set->dns_client != NULL)
            client->set.dns_client = set->dns_client;
        if (set->dns_client_socket_path != NULL &&
            *set->dns_client_socket_path != '\0') {
            client->set.dns_client_socket_path =
                p_strdup_empty(pool, set->dns_client_socket_path);
        }
        if (set->dns_ttl_msecs > 0)
            client->set.dns_ttl_msecs = set->dns_ttl_msecs;
        if (set->user_agent != NULL && *set->user_agent != '\0')
            client->set.user_agent = p_strdup_empty(pool, set->user_agent);
        if (set->rawlog_dir != NULL && *set->rawlog_dir != '\0')
            client->set.rawlog_dir = p_strdup_empty(pool, set->rawlog_dir);

        if (set->ssl != NULL)
            client->set.ssl = ssl_iostream_settings_dup(pool, set->ssl);

        if (set->proxy_socket_path != NULL && *set->proxy_socket_path != '\0') {
            client->set.proxy_socket_path = p_strdup(pool, set->proxy_socket_path);
            client->set.proxy_url = NULL;
        } else if (set->proxy_url != NULL) {
            client->set.proxy_url = http_url_clone(pool, set->proxy_url);
            client->set.proxy_socket_path = NULL;
        }
        if (set->proxy_username != NULL && *set->proxy_username != '\0') {
            client->set.proxy_username = p_strdup_empty(pool, set->proxy_username);
            client->set.proxy_password = p_strdup(pool, set->proxy_password);
        } else if (set->proxy_url != NULL && set->proxy_url->user != NULL &&
            *set->proxy_url->user != '\0') {
            client->set.proxy_username =
                p_strdup_empty(pool, set->proxy_url->user);
            client->set.proxy_password =
                p_strdup(pool, set->proxy_url->password);
        }

        if (set->max_idle_time_msecs > 0)
            client->set.max_idle_time_msecs = set->max_idle_time_msecs;
        if (set->max_parallel_connections > 0)
            client->set.max_parallel_connections = set->max_parallel_connections;
        if (set->max_pipelined_requests > 0)
            client->set.max_pipelined_requests = set->max_pipelined_requests;
        if (set->max_attempts > 0)
            client->set.max_attempts = set->max_attempts;
        if (set->max_connect_attempts > 0)
            client->set.max_connect_attempts = set->max_connect_attempts;
        if (set->connect_backoff_time_msecs > 0) {
            client->set.connect_backoff_time_msecs =
                set->connect_backoff_time_msecs;
        }
        if (set->connect_backoff_max_time_msecs > 0) {
            client->set.connect_backoff_max_time_msecs =
                set->connect_backoff_max_time_msecs;
        }
        client->set.no_auto_redirect =
            client->set.no_auto_redirect || set->no_auto_redirect;
        client->set.no_auto_retry =
            client->set.no_auto_retry || set->no_auto_retry;
        client->set.no_ssl_tunnel =
            client->set.no_ssl_tunnel || set->no_ssl_tunnel;
        if (set->max_redirects > 0)
            client->set.max_redirects = set->max_redirects;
        client->set.response_hdr_limits = set->response_hdr_limits;
        if (set->request_absolute_timeout_msecs > 0) {
            client->set.request_absolute_timeout_msecs =
                set->request_absolute_timeout_msecs;
        }
        if (set->request_timeout_msecs > 0)
            client->set.request_timeout_msecs = set->request_timeout_msecs;
        if (set->connect_timeout_msecs > 0)
            client->set.connect_timeout_msecs = set->connect_timeout_msecs;
        if (set->soft_connect_timeout_msecs > 0)
            client->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs;
        if (set->max_auto_retry_delay > 0)
            client->set.max_auto_retry_delay = set->max_auto_retry_delay;
        client->set.socket_send_buffer_size = set->socket_send_buffer_size;
        client->set.socket_recv_buffer_size = set->socket_recv_buffer_size;
        client->set.debug = client->set.debug || set->debug;
    }

    i_array_init(&client->delayed_failing_requests, 1);

    hash_table_create(&client->hosts, default_pool, 0, str_hash, strcmp);
    hash_table_create(&client->peers, default_pool, 0,
        http_client_peer_addr_hash, http_client_peer_addr_cmp);

    return client;
}

struct http_client *
http_client_init(const struct http_client_settings *set)
{
    return http_client_init_shared(NULL, set);
}

void http_client_deinit(struct http_client **_client)
{
    struct http_client *client = *_client;
    struct http_client_request *req;
    struct http_client_host *host;
    struct http_client_peer *peer;

    *_client = NULL;

    /* destroy requests without calling callbacks */
    req = client->requests_list;
    while (req != NULL) {
        struct http_client_request *next_req = req->next;
        http_client_request_destroy(&req);
        req = next_req;
    }
    i_assert(client->requests_count == 0);

    /* free peers */
    while (client->peers_list != NULL) {
        peer = client->peers_list;
        http_client_peer_close(&peer);
    }
    hash_table_destroy(&client->peers);

    /* free hosts */
    while (client->hosts_list != NULL) {
        host = client->hosts_list;
        http_client_host_free(&host);
    }
    hash_table_destroy(&client->hosts);

    array_free(&client->delayed_failing_requests);
    timeout_remove(&client->to_failing_requests);

    if (client->ssl_ctx != NULL)
        ssl_iostream_context_unref(&client->ssl_ctx);
    http_client_context_unref(&client->cctx);
    pool_unref(&client->pool);
}

void http_client_switch_ioloop(struct http_client *client)
{
    struct http_client_host *host;
    struct http_client_peer *peer;

    /* move peers */
    for (peer = client->peers_list; peer != NULL; peer = peer->next)
        http_client_peer_switch_ioloop(peer);

    /* move dns lookups and delayed requests */
    for (host = client->hosts_list; host != NULL; host = host->next)
        http_client_host_switch_ioloop(host);

    /* move timeouts */
    if (client->to_failing_requests != NULL) {
        client->to_failing_requests =
            io_loop_move_timeout(&client->to_failing_requests);
    }

    http_client_context_switch_ioloop(client->cctx);
}

void http_client_wait(struct http_client *client)
{
    struct ioloop *prev_ioloop = current_ioloop;

    i_assert(client->ioloop == NULL);

    if (client->requests_count == 0)
        return;

    client->ioloop = io_loop_create();
    http_client_switch_ioloop(client);
    if (client->set.dns_client != NULL)
        dns_client_switch_ioloop(client->set.dns_client);
    /* either we're waiting for network I/O or we're getting out of a
       callback using timeout_add_short(0) */
    i_assert(io_loop_have_ios(client->ioloop) ||
         io_loop_have_immediate_timeouts(client->ioloop));

    do {
        http_client_debug(client,
            "Waiting for %d requests to finish", client->requests_count);
        io_loop_run(client->ioloop);
    } while (client->requests_count > 0);

    http_client_debug(client, "All requests finished");

    io_loop_set_current(prev_ioloop);
    http_client_switch_ioloop(client);
    if (client->set.dns_client != NULL)
        dns_client_switch_ioloop(client->set.dns_client);
    io_loop_set_current(client->ioloop);
    io_loop_destroy(&client->ioloop);
}

unsigned int http_client_get_pending_request_count(struct http_client *client)
{
    return client->requests_count;
}

int http_client_init_ssl_ctx(struct http_client *client, const char **error_r)
{
    const char *error;

    if (client->ssl_ctx != NULL)
        return 0;

    if (client->set.ssl == NULL) {
        *error_r = "Requested https connection, but no SSL settings given";
        return -1;
    }
    if (ssl_iostream_client_context_cache_get(client->set.ssl, &client->ssl_ctx, &error) < 0) {
        *error_r = t_strdup_printf("Couldn't initialize SSL context: %s",
                       error);
        return -1;
    }
    return 0;
}

/*
 * Delayed request errors
 */

static void
http_client_handle_request_errors(struct http_client *client)
{
    struct http_client_request *const *req_idx;

    timeout_remove(&client->to_failing_requests);

    array_foreach(&client->delayed_failing_requests, req_idx) {
        struct http_client_request *req = *req_idx;

        i_assert(req->refcount == 1);
        http_client_request_error_delayed(&req);
    }
    array_clear(&client->delayed_failing_requests);
}

void http_client_delay_request_error(struct http_client *client,
    struct http_client_request *req)
{
    if (client->to_failing_requests == NULL) {
        client->to_failing_requests = timeout_add_short(0,
            http_client_handle_request_errors, client);
    }
    array_append(&client->delayed_failing_requests, &req, 1);
}

void http_client_remove_request_error(struct http_client *client,
    struct http_client_request *req)
{
    struct http_client_request *const *reqs;
    unsigned int i, count;

    reqs = array_get(&client->delayed_failing_requests, &count);
    for (i = 0; i < count; i++) {
        if (reqs[i] == req) {
            array_delete(&client->delayed_failing_requests, i, 1);
            return;
        }
    }
}

/*
 * Client shared context
 */

struct http_client_context *
http_client_context_create(const struct http_client_settings *set)
{
    struct http_client_context *cctx;
    pool_t pool;

    pool = pool_alloconly_create("http client context", 1024);
    cctx = p_new(pool, struct http_client_context, 1);
    cctx->pool = pool;
    cctx->refcount = 1;

    cctx->set.dns_client = set->dns_client;
    cctx->set.dns_client_socket_path =
        p_strdup_empty(pool, set->dns_client_socket_path);
    cctx->set.dns_ttl_msecs = (set->dns_ttl_msecs == 0 ?
            HTTP_CLIENT_DEFAULT_DNS_TTL_MSECS : set->dns_ttl_msecs);
    cctx->set.user_agent = p_strdup_empty(pool, set->user_agent);
    cctx->set.rawlog_dir = p_strdup_empty(pool, set->rawlog_dir);

    if (set->ssl != NULL)
        cctx->set.ssl = ssl_iostream_settings_dup(pool, set->ssl);

    if (set->proxy_socket_path != NULL && *set->proxy_socket_path != '\0') {
        cctx->set.proxy_socket_path = p_strdup(pool, set->proxy_socket_path);
    } else if (set->proxy_url != NULL) {
        cctx->set.proxy_url = http_url_clone(pool, set->proxy_url);
    }
    if (set->proxy_username != NULL && *set->proxy_username != '\0') {
        cctx->set.proxy_username = p_strdup_empty(pool, set->proxy_username);
        cctx->set.proxy_password = p_strdup(pool, set->proxy_password);
    } else if (set->proxy_url != NULL) {
        cctx->set.proxy_username =
            p_strdup_empty(pool, set->proxy_url->user);
        cctx->set.proxy_password =
            p_strdup(pool, set->proxy_url->password);
    }

    cctx->set.max_idle_time_msecs = set->max_idle_time_msecs;
    cctx->set.max_pipelined_requests =
        (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1);
    cctx->set.max_parallel_connections =
        (set->max_parallel_connections > 0 ? set->max_parallel_connections : 1);
    cctx->set.max_attempts = set->max_attempts;
    cctx->set.max_connect_attempts = set->max_connect_attempts;
    cctx->set.connect_backoff_time_msecs =
        set->connect_backoff_time_msecs == 0 ?
            HTTP_CLIENT_DEFAULT_BACKOFF_TIME_MSECS :
            set->connect_backoff_time_msecs;
    cctx->set.connect_backoff_max_time_msecs =
        set->connect_backoff_max_time_msecs == 0 ?
            HTTP_CLIENT_DEFAULT_BACKOFF_MAX_TIME_MSECS :
            set->connect_backoff_max_time_msecs;
    cctx->set.no_auto_redirect = set->no_auto_redirect;
    cctx->set.no_auto_retry = set->no_auto_retry;
    cctx->set.no_ssl_tunnel = set->no_ssl_tunnel;
    cctx->set.max_redirects = set->max_redirects;
    cctx->set.response_hdr_limits = set->response_hdr_limits;
    cctx->set.request_absolute_timeout_msecs =
        set->request_absolute_timeout_msecs;
    cctx->set.request_timeout_msecs =
        set->request_timeout_msecs == 0 ?
            HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS :
            set->request_timeout_msecs;
    cctx->set.connect_timeout_msecs = set->connect_timeout_msecs;
    cctx->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs;
    cctx->set.max_auto_retry_delay = set->max_auto_retry_delay;
    cctx->set.socket_send_buffer_size = set->socket_send_buffer_size;
    cctx->set.socket_recv_buffer_size = set->socket_recv_buffer_size;
    cctx->set.debug = set->debug;

    cctx->conn_list = http_client_connection_list_init();

    return cctx;
}

void http_client_context_ref(struct http_client_context *cctx)
{
    cctx->refcount++;
}

void http_client_context_unref(struct http_client_context **_cctx)
{
    struct http_client_context *cctx = *_cctx;

    *_cctx = NULL;

    i_assert(cctx->refcount > 0);
    if (--cctx->refcount > 0)
        return;

    connection_list_deinit(&cctx->conn_list);

    pool_unref(&cctx->pool);
}

void http_client_context_switch_ioloop(struct http_client_context *cctx)
{
    struct connection *_conn = cctx->conn_list->connections;

    /* move connections */
    /* FIXME: we wouldn't necessarily need to switch all of them
       immediately, only those that have requests now. but also connections
       that get new requests before ioloop is switched again.. */
    for (; _conn != NULL; _conn = _conn->next) {
        struct http_client_connection *conn =
            (struct http_client_connection *)_conn;

        http_client_connection_switch_ioloop(conn);
    }
}