http-client-connection.c revision e9228a3918aa0243eff4aae1ff5462bd3198417f
0N/A/* Copyright (c) 2013-2015 Dovecot authors, see the included COPYING file */
0N/A
0N/A#include "lib.h"
0N/A#include "net.h"
0N/A#include "str.h"
180N/A#include "hash.h"
180N/A#include "llist.h"
51N/A#include "array.h"
51N/A#include "ioloop.h"
51N/A#include "istream.h"
180N/A#include "istream-timeout.h"
180N/A#include "ostream.h"
180N/A#include "time-util.h"
180N/A#include "iostream-rawlog.h"
180N/A#include "iostream-ssl.h"
0N/A#include "http-response-parser.h"
617N/A
617N/A#include "http-client-private.h"
617N/A
617N/A/*
618N/A * Logging
619N/A */
621N/A
617N/Astatic inline void
622N/Ahttp_client_connection_debug(struct http_client_connection *conn,
623N/A const char *format, ...) ATTR_FORMAT(2, 3);
625N/A
629N/Astatic inline void
630N/Ahttp_client_connection_debug(struct http_client_connection *conn,
631N/A const char *format, ...)
632N/A{
637N/A va_list args;
637N/A
637N/A if (conn->client->set.debug) {
638N/A
638N/A va_start(args, format);
639N/A i_debug("http-client: conn %s: %s",
640N/A http_client_connection_label(conn), t_strdup_vprintf(format, args));
641N/A va_end(args);
645N/A }
647N/A}
648N/A
648N/A/*
649N/A * Connection
617N/A */
617N/A
565N/Astatic void http_client_connection_ready(struct http_client_connection *conn);
565N/Astatic void http_client_connection_input(struct connection *_conn);
565N/Astatic void
565N/Ahttp_client_connection_disconnect(struct http_client_connection *conn);
565N/A
565N/Aunsigned int
579N/Ahttp_client_connection_count_pending(struct http_client_connection *conn)
568N/A{
576N/A unsigned int pending_count = array_count(&conn->request_wait_list);
577N/A
580N/A if (conn->in_req_callback || conn->pending_request != NULL)
585N/A pending_count++;
587N/A return pending_count;
593N/A}
595N/A
595N/Abool http_client_connection_is_idle(struct http_client_connection *conn)
596N/A{
600N/A return (conn->to_idle != NULL);
602N/A}
603N/A
604N/Astatic void
565N/Ahttp_client_connection_retry_requests(struct http_client_connection *conn,
565N/A unsigned int status, const char *error)
525N/A{
525N/A struct http_client_request **req;
525N/A
488N/A if (!array_is_created(&conn->request_wait_list))
524N/A return;
524N/A
524N/A array_foreach_modifiable(&conn->request_wait_list, req) {
524N/A if ((*req)->state < HTTP_REQUEST_STATE_FINISHED)
524N/A http_client_request_retry(*req, status, error);
524N/A http_client_request_unref(req);
524N/A }
524N/A array_clear(&conn->request_wait_list);
524N/A}
524N/A
524N/Astatic void
524N/Ahttp_client_connection_server_close(struct http_client_connection **_conn)
524N/A{
524N/A struct http_client_connection *conn = *_conn;
524N/A struct http_client_request **req;
524N/A
524N/A http_client_connection_debug(conn,
540N/A "Server explicitly closed connection");
544N/A
488N/A array_foreach_modifiable(&conn->request_wait_list, req) {
488N/A if ((*req)->state < HTTP_REQUEST_STATE_FINISHED)
534N/A http_client_request_resubmit(*req);
534N/A http_client_request_unref(req);
534N/A }
534N/A array_clear(&conn->request_wait_list);
534N/A
534N/A if (conn->client->ioloop != NULL)
534N/A io_loop_stop(conn->client->ioloop);
534N/A
440N/A http_client_connection_close(_conn);
440N/A}
440N/A
440N/Astatic void
440N/Ahttp_client_connection_abort_error(struct http_client_connection **_conn,
442N/A unsigned int status, const char *error)
443N/A{
445N/A struct http_client_connection *conn = *_conn;
482N/A struct http_client_request **req;
482N/A
444N/A http_client_connection_debug(conn, "Aborting connection: %s", error);
460N/A
468N/A array_foreach_modifiable(&conn->request_wait_list, req) {
470N/A i_assert((*req)->submitted);
472N/A http_client_request_error(*req, status, error);
440N/A http_client_request_unref(req);
440N/A }
336N/A array_clear(&conn->request_wait_list);
336N/A http_client_connection_close(_conn);
336N/A}
336N/A
365N/Astatic void
408N/Ahttp_client_connection_abort_temp_error(struct http_client_connection **_conn,
336N/A unsigned int status, const char *error)
361N/A{
389N/A struct http_client_connection *conn = *_conn;
390N/A const char *sslerr;
413N/A
430N/A if (status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST &&
370N/A conn->ssl_iostream != NULL) {
378N/A sslerr = ssl_iostream_get_last_error(conn->ssl_iostream);
380N/A if (sslerr != NULL) {
392N/A error = t_strdup_printf("%s (last SSL error: %s)",
393N/A error, sslerr);
412N/A }
421N/A if (ssl_iostream_has_handshake_failed(conn->ssl_iostream)) {
423N/A /* this isn't really a "connection lost", but that we
336N/A don't trust the remote's SSL certificate. don't
336N/A retry. */
204N/A http_client_connection_abort_error(_conn,
204N/A HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, error);
204N/A return;
204N/A }
254N/A }
254N/A
204N/A http_client_connection_debug(conn,
217N/A "Aborting connection with temporary error: %s", error);
265N/A
316N/A http_client_connection_retry_requests(conn, status, error);
206N/A http_client_connection_close(_conn);
307N/A}
316N/A
322N/Abool http_client_connection_is_ready(struct http_client_connection *conn)
208N/A{
209N/A int ret;
210N/A
216N/A if (conn->in_req_callback) {
224N/A /* this can happen when a nested ioloop is created inside request
225N/A callback. we currently don't reuse connections that are occupied
240N/A this way, but theoretically we could, although that would add
241N/A quite a bit of complexity.
247N/A */
249N/A return FALSE;
252N/A }
253N/A
262N/A if (!conn->connected || conn->output_locked || conn->output_broken ||
264N/A conn->close_indicated || conn->tunneling ||
267N/A http_client_connection_count_pending(conn) >=
267N/A conn->client->set.max_pipelined_requests)
270N/A return FALSE;
270N/A
272N/A if (conn->last_ioloop != NULL && conn->last_ioloop != current_ioloop) {
275N/A conn->last_ioloop = current_ioloop;
282N/A /* Active ioloop is different from what we saw earlier;
283N/A we may have missed a disconnection event on this connection.
328N/A Verify status by reading from connection. */
204N/A if ((ret=i_stream_read(conn->conn.input)) == -1) {
204N/A int stream_errno = conn->conn.input->stream_errno;
135N/A
135N/A i_assert(conn->conn.input->stream_errno != 0 || conn->conn.input->eof);
135N/A http_client_connection_abort_temp_error(&conn,
135N/A HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
179N/A t_strdup_printf("Connection lost: read(%s) failed: %s",
143N/A i_stream_get_name(conn->conn.input),
158N/A stream_errno != 0 ?
152N/A i_stream_get_error(conn->conn.input) :
159N/A "EOF"));
179N/A return FALSE;
180N/A }
181N/A }
180N/A return TRUE;
135N/A}
137N/A
139N/Astatic void
144N/Ahttp_client_connection_idle_timeout(struct http_client_connection *conn)
153N/A{
155N/A http_client_connection_debug(conn, "Idle connection timed out");
169N/A
174N/A /* cannot get here unless connection was established at some point */
188N/A i_assert(conn->connect_succeeded);
189N/A
135N/A http_client_connection_close(&conn);
135N/A}
3N/A
3N/Avoid http_client_connection_check_idle(struct http_client_connection *conn)
3N/A{
3N/A unsigned int timeout, count;
22N/A
22N/A if (conn->connected &&
3N/A array_is_created(&conn->request_wait_list) &&
38N/A array_count(&conn->request_wait_list) == 0 &&
35N/A !conn->in_req_callback &&
35N/A conn->incoming_payload == NULL &&
36N/A conn->client->set.max_idle_time_msecs > 0) {
36N/A
90N/A if (conn->to_idle != NULL) {
40N/A /* timeout already set */
66N/A return;
66N/A }
79N/A
75N/A if (conn->client->ioloop != NULL)
76N/A io_loop_stop(conn->client->ioloop);
101N/A
104N/A count = array_count(&conn->peer->conns);
180N/A i_assert(count > 0);
180N/A
180N/A /* set timeout for this connection */
180N/A if (count > conn->client->set.max_parallel_connections) {
22N/A /* instant death for (urgent) connections above limit */
22N/A timeout = 0;
33N/A } else {
33N/A unsigned int idle_count = http_client_peer_idle_connections(conn->peer);
45N/A
45N/A /* kill duplicate connections quicker;
55N/A linearly based on the number of connections */
57N/A i_assert(count >= idle_count + 1);
58N/A timeout = (conn->client->set.max_parallel_connections - idle_count) *
60N/A (conn->client->set.max_idle_time_msecs /
62N/A conn->client->set.max_parallel_connections);
73N/A }
73N/A
93N/A http_client_connection_debug(conn,
93N/A "No more requests queued; going idle (timeout = %u msecs)",
81N/A timeout);
85N/A
86N/A conn->to_idle =
88N/A timeout_add(timeout, http_client_connection_idle_timeout, conn);
91N/A
96N/A } else {
3N/A /* there should be no idle timeout */
3N/A i_assert(conn->to_idle == NULL);
0N/A }
0N/A}
0N/A
0N/Astatic void
0N/Ahttp_client_connection_request_timeout(struct http_client_connection *conn)
0N/A{
0N/A unsigned int msecs = conn->client->set.request_timeout_msecs;
0N/A
0N/A conn->conn.input->stream_errno = ETIMEDOUT;
0N/A http_client_connection_abort_temp_error(&conn,
0N/A HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT, t_strdup_printf(
0N/A "No response for request in %u.%03u secs",
0N/A msecs/1000, msecs%1000));
0N/A}
0N/A
0N/Avoid http_client_connection_start_request_timeout(
0N/A struct http_client_connection *conn)
0N/A{
0N/A unsigned int timeout_msecs = conn->client->set.request_timeout_msecs;
0N/A
0N/A if (timeout_msecs == 0)
0N/A ;
0N/A else if (conn->to_requests != NULL)
0N/A timeout_reset(conn->to_requests);
0N/A else {
0N/A conn->to_requests = timeout_add(timeout_msecs,
0N/A http_client_connection_request_timeout, conn);
0N/A }
0N/A}
0N/A
0N/Avoid http_client_connection_reset_request_timeout(
0N/A struct http_client_connection *conn)
0N/A{
0N/A if (conn->to_requests != NULL)
0N/A timeout_reset(conn->to_requests);
0N/A}
0N/A
0N/Avoid http_client_connection_stop_request_timeout(
0N/A struct http_client_connection *conn)
0N/A{
0N/A if (conn->to_requests != NULL)
0N/A timeout_remove(&conn->to_requests);
0N/A}
0N/A
0N/Astatic void
0N/Ahttp_client_connection_continue_timeout(struct http_client_connection *conn)
0N/A{
0N/A struct http_client_request *const *req_idx;
0N/A struct http_client_request *req;
0N/A const char *error;
0N/A
0N/A if (conn->to_response != NULL)
0N/A timeout_remove(&conn->to_response);
0N/A conn->peer->no_payload_sync = TRUE;
0N/A
0N/A http_client_connection_debug(conn,
0N/A "Expected 100-continue response timed out; sending payload anyway");
0N/A
0N/A i_assert(array_count(&conn->request_wait_list) > 0);
0N/A req_idx = array_idx(&conn->request_wait_list,
0N/A array_count(&conn->request_wait_list)-1);
0N/A req = req_idx[0];
0N/A
0N/A conn->payload_continue = TRUE;
0N/A if (http_client_request_send_more(req, &error) < 0) {
0N/A http_client_connection_abort_temp_error(&conn,
0N/A HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
0N/A t_strdup_printf("Failed to send request: %s", error));
0N/A }
0N/A}
0N/A
0N/Aint http_client_connection_next_request(struct http_client_connection *conn)
0N/A{
0N/A struct http_client_request *req = NULL;
0N/A const char *error;
0N/A bool have_pending_requests;
0N/A
0N/A if (!http_client_connection_is_ready(conn)) {
0N/A http_client_connection_debug(conn, "Not ready for next request");
0N/A return 0;
0N/A }
0N/A
0N/A /* claim request, but no urgent request can be second in line */
0N/A have_pending_requests = array_count(&conn->request_wait_list) > 0 ||
0N/A conn->pending_request != NULL;
0N/A req = http_client_peer_claim_request(conn->peer, have_pending_requests);
0N/A if (req == NULL)
0N/A return 0;
0N/A
0N/A i_assert(req->state == HTTP_REQUEST_STATE_QUEUED);
0N/A
0N/A if (conn->to_idle != NULL)
0N/A timeout_remove(&conn->to_idle);
0N/A
0N/A req->conn = conn;
0N/A conn->payload_continue = FALSE;
0N/A if (conn->peer->no_payload_sync)
0N/A req->payload_sync = FALSE;
0N/A
0N/A array_append(&conn->request_wait_list, &req, 1);
0N/A http_client_request_ref(req);
0N/A
0N/A http_client_connection_debug(conn, "Claimed request %s",
0N/A http_client_request_label(req));
0N/A
0N/A if (http_client_request_send(req, &error) < 0) {
0N/A http_client_connection_abort_temp_error(&conn,
0N/A HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
0N/A t_strdup_printf("Failed to send request: %s", error));
0N/A return -1;
0N/A }
0N/A
0N/A if (req->connect_tunnel)
0N/A conn->tunneling = TRUE;
0N/A
0N/A /* RFC 7231, Section 5.1.1: Expect
0N/A
0N/A o A client that sends a 100-continue expectation is not required to
0N/A wait for any specific length of time; such a client MAY proceed to
0N/A send the message body even if it has not yet received a response.
0N/A Furthermore, since 100 (Continue) responses cannot be sent through
0N/A an HTTP/1.0 intermediary, such a client SHOULD NOT wait for an
0N/A indefinite period before sending the message body.
0N/A */
0N/A if (req->payload_sync && !conn->peer->seen_100_response) {
0N/A i_assert(req->payload_chunked || req->payload_size > 0);
0N/A i_assert(conn->to_response == NULL);
0N/A conn->to_response = timeout_add(HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS,
0N/A http_client_connection_continue_timeout, conn);
0N/A }
0N/A
0N/A return 1;
0N/A}
0N/A
0N/Astatic void http_client_connection_destroy(struct connection *_conn)
0N/A{
0N/A struct http_client_connection *conn =
0N/A (struct http_client_connection *)_conn;
0N/A const char *error;
0N/A unsigned int msecs;
0N/A
0N/A switch (_conn->disconnect_reason) {
0N/A case CONNECTION_DISCONNECT_CONNECT_TIMEOUT:
0N/A if (conn->connected_timestamp.tv_sec == 0) {
0N/A msecs = timeval_diff_msecs(&ioloop_timeval,
0N/A &conn->connect_start_timestamp);
0N/A error = t_strdup_printf(
0N/A "connect(%s) failed: Connection timed out in %u.%03u secs",
0N/A _conn->name, msecs/1000, msecs%1000);
0N/A } else {
0N/A msecs = timeval_diff_msecs(&ioloop_timeval,
0N/A &conn->connected_timestamp);
0N/A error = t_strdup_printf(
0N/A "SSL handshaking with %s failed: Connection timed out in %u.%03u secs",
0N/A _conn->name, msecs/1000, msecs%1000);
0N/A }
0N/A http_client_connection_debug(conn, "%s", error);
0N/A http_client_peer_connection_failure(conn->peer, error);
0N/A break;
0N/A case CONNECTION_DISCONNECT_CONN_CLOSED:
0N/A /* retry pending requests if possible */
0N/A error = _conn->input == NULL ? "Connection lost" :
0N/A t_strdup_printf("Connection lost: %s",
0N/A i_stream_get_error(_conn->input));
0N/A http_client_connection_debug(conn, "%s", error);
0N/A http_client_connection_retry_requests(conn,
0N/A HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, error);
0N/A default:
0N/A break;
0N/A }
0N/A
0N/A http_client_connection_close(&conn);
0N/A}
0N/A
0N/Astatic void http_client_payload_finished(struct http_client_connection *conn)
0N/A{
0N/A timeout_remove(&conn->to_input);
0N/A conn->conn.io = io_add_istream(conn->conn.input,
0N/A http_client_connection_input, &conn->conn);
0N/A}
0N/A
0N/Astatic void
0N/Ahttp_client_payload_destroyed_timeout(struct http_client_connection *conn)
0N/A{
0N/A if (conn->close_indicated) {
0N/A http_client_connection_server_close(&conn);
0N/A return;
0N/A }
0N/A http_client_connection_input(&conn->conn);
0N/A}
0N/A
0N/Astatic void http_client_payload_destroyed(struct http_client_request *req)
0N/A{
0N/A struct http_client_connection *conn = req->conn;
0N/A
0N/A i_assert(conn != NULL);
0N/A i_assert(conn->pending_request == req);
0N/A i_assert(conn->incoming_payload != NULL);
0N/A i_assert(conn->conn.io == NULL);
0N/A
0N/A http_client_connection_debug(conn,
0N/A "Response payload stream destroyed (%u ms after initial response)",
0N/A timeval_diff_msecs(&ioloop_timeval, &req->response_time));
0N/A
0N/A /* caller is allowed to change the socket fd to blocking while reading
0N/A the payload. make sure here that it's switched back. */
0N/A net_set_nonblock(conn->conn.fd_in, TRUE);
0N/A
0N/A req->conn = NULL;
0N/A conn->incoming_payload = NULL;
0N/A conn->pending_request = NULL;
0N/A http_client_connection_ref(conn);
0N/A http_client_request_finish(&req);
0N/A
0N/A /* room for new requests */
0N/A if (http_client_connection_is_ready(conn))
0N/A http_client_peer_trigger_request_handler(conn->peer);
0N/A
0N/A /* input stream may have pending input. make sure input handler
0N/A gets called (but don't do it directly, since we get get here
0N/A somewhere from the API user's code, which we can't really know what
0N/A state it is in). this call also triggers sending a new request if
0N/A necessary. */
0N/A conn->to_input =
0N/A timeout_add_short(0, http_client_payload_destroyed_timeout, conn);
0N/A
0N/A i_assert(req != NULL);
0N/A http_client_request_unref(&req);
0N/A http_client_connection_unref(&conn);
0N/A}
0N/A
0N/Astatic bool
0N/Ahttp_client_connection_return_response(struct http_client_connection *conn,
0N/A struct http_client_request *req, struct http_response *response)
0N/A{
0N/A struct istream *payload;
0N/A bool retrying, ret;
0N/A
0N/A i_assert(!conn->in_req_callback);
0N/A i_assert(conn->incoming_payload == NULL);
0N/A i_assert(conn->pending_request == NULL);
0N/A
0N/A http_client_request_ref(req);
0N/A req->state = HTTP_REQUEST_STATE_GOT_RESPONSE;
0N/A
0N/A if (response->payload != NULL) {
0N/A /* wrap the stream to capture the destroy event without destroying the
0N/A actual payload stream. */
0N/A conn->incoming_payload = response->payload =
0N/A i_stream_create_timeout(response->payload,
0N/A conn->client->set.request_timeout_msecs);
0N/A i_stream_add_destroy_callback(response->payload,
0N/A http_client_payload_destroyed,
0N/A req);
0N/A /* the callback may add its own I/O, so we need to remove
0N/A our one before calling it */
0N/A io_remove(&conn->conn.io);
0N/A /* we've received the request itself, and we can't reset the
0N/A timeout during the payload reading. */
0N/A http_client_connection_stop_request_timeout(conn);
0N/A }
0N/A
0N/A conn->in_req_callback = TRUE;
0N/A http_client_connection_ref(conn);
0N/A retrying = !http_client_request_callback(req, response);
0N/A http_client_connection_unref(&conn);
0N/A if (conn == NULL) {
0N/A req->conn = NULL;
0N/A /* the callback managed to get this connection destroyed */
0N/A if (!retrying)
0N/A http_client_request_finish(&req);
0N/A http_client_request_unref(&req);
0N/A return FALSE;
0N/A }
0N/A conn->in_req_callback = FALSE;
0N/A
0N/A if (retrying) {
0N/A /* retrying, don't destroy the request */
0N/A if (response->payload != NULL) {
0N/A i_stream_remove_destroy_callback(conn->incoming_payload,
0N/A http_client_payload_destroyed);
0N/A i_stream_unref(&conn->incoming_payload);
0N/A conn->conn.io = io_add_istream(conn->conn.input,
0N/A http_client_connection_input,
0N/A &conn->conn);
0N/A }
0N/A http_client_request_unref(&req);
0N/A return TRUE;
0N/A }
0N/A
0N/A http_client_connection_ref(conn);
0N/A if (response->payload != NULL) {
0N/A req->state = HTTP_REQUEST_STATE_PAYLOAD_IN;
0N/A payload = response->payload;
0N/A response->payload = NULL;
0N/A conn->pending_request = req;
0N/A
0N/A /* request is dereferenced in payload destroy callback */
0N/A i_stream_unref(&payload);
0N/A
0N/A if (conn->to_input != NULL && conn->conn.input != NULL) {
0N/A /* already finished reading the payload */
0N/A http_client_payload_finished(conn);
0N/A }
0N/A } else {
0N/A req->conn = NULL;
0N/A http_client_request_finish(&req);
0N/A http_client_request_unref(&req);
0N/A }
0N/A
0N/A if (conn->incoming_payload == NULL && conn->conn.input != NULL) {
0N/A i_assert(conn->conn.io != NULL ||
0N/A conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_RAW);
0N/A ret = TRUE;
0N/A } else {
0N/A ret = FALSE;
0N/A }
0N/A http_client_connection_unref(&conn);
0N/A return ret;
0N/A}
0N/A
0N/Astatic void http_client_connection_input(struct connection *_conn)
0N/A{
0N/A struct http_client_connection *conn =
0N/A (struct http_client_connection *)_conn;
0N/A struct http_response response;
0N/A struct http_client_request *const *reqs;
0N/A struct http_client_request *req = NULL;
0N/A enum http_response_payload_type payload_type;
0N/A unsigned int count;
0N/A int finished = 0, ret;
0N/A const char *error;
0N/A
0N/A i_assert(conn->incoming_payload == NULL);
0N/A
0N/A if (conn->ssl_iostream != NULL &&
0N/A !ssl_iostream_is_handshaked(conn->ssl_iostream)) {
0N/A /* finish SSL negotiation by reading from input stream */
0N/A while ((ret=i_stream_read(conn->conn.input)) > 0) {
0N/A if (ssl_iostream_is_handshaked(conn->ssl_iostream))
0N/A break;
0N/A }
0N/A if (ret < 0) {
0N/A int stream_errno = conn->conn.input->stream_errno;
0N/A
0N/A /* failed somehow */
0N/A i_assert(ret != -2);
0N/A error = t_strdup_printf(
0N/A "SSL handshaking with %s failed: "
0N/A "read(%s) failed: %s",
0N/A _conn->name,
0N/A i_stream_get_name(conn->conn.input),
0N/A stream_errno != 0 ?
0N/A i_stream_get_error(conn->conn.input) : "EOF");
0N/A http_client_peer_connection_failure(conn->peer, error);
0N/A http_client_connection_debug(conn, "%s", error);
http_client_connection_close(&conn);
return;
}
if (!ssl_iostream_is_handshaked(conn->ssl_iostream)) {
/* not finished */
i_assert(ret == 0);
return;
}
/* ready for first request */
http_client_connection_ready(conn);
}
if (conn->to_input != NULL) {
/* We came here from a timeout added by
http_client_payload_destroyed(). The IO couldn't be added
back immediately in there, because the HTTP API user may
still have had its own IO pointed to the same fd. It should
be removed by now, so we can add it back. */
http_client_payload_finished(conn);
finished++;
}
/* we've seen activity from the server; reset request timeout */
http_client_connection_reset_request_timeout(conn);
/* get first waiting request */
reqs = array_get(&conn->request_wait_list, &count);
if (count > 0) {
req = reqs[0];
/* determine whether to expect a response payload */
payload_type = http_client_request_get_payload_type(req);
} else {
req = NULL;
payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED;
}
/* drop connection with broken output if last possible input was
received */
if (conn->output_broken && (count == 0 ||
(count == 1 && req->state == HTTP_REQUEST_STATE_ABORTED))) {
http_client_connection_server_close(&conn);
return;
}
while ((ret=http_response_parse_next
(conn->http_parser, payload_type, &response, &error)) > 0) {
bool aborted, early = FALSE;
if (req == NULL) {
/* server sent response without any requests in the wait list */
if (response.status == 408) {
http_client_connection_debug(conn,
"Server explicitly closed connection: 408 %s",
response.reason);
} else {
http_client_connection_debug(conn,
"Got unexpected input from server: %u %s",
response.status, response.reason);
}
http_client_connection_close(&conn);
return;
}
req->response_time = ioloop_timeval;
/* Got some response; cancel response timeout */
if (conn->to_response != NULL)
timeout_remove(&conn->to_response);
/* RFC 7231, Section 6.2:
A client MUST be able to parse one or more 1xx responses received
prior to a final response, even if the client does not expect one. A
user agent MAY ignore unexpected 1xx responses.
*/
if (req->payload_sync && response.status == 100) {
if (conn->payload_continue) {
http_client_connection_debug(conn,
"Got 100-continue response after timeout");
continue;
}
conn->peer->no_payload_sync = FALSE;
conn->peer->seen_100_response = TRUE;
conn->payload_continue = TRUE;
http_client_connection_debug(conn,
"Got expected 100-continue response");
if (req->state == HTTP_REQUEST_STATE_ABORTED) {
http_client_connection_debug(conn,
"Request aborted before sending payload was complete.");
http_client_connection_close(&conn);
return;
}
if (http_client_request_send_more(req, &error) < 0) {
http_client_connection_abort_temp_error(&conn,
HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
t_strdup_printf("Failed to send request: %s", error));
}
return;
} else if (response.status / 100 == 1) {
/* ignore other 1xx for now */
http_client_connection_debug(conn,
"Got unexpected %u response; ignoring", response.status);
continue;
} else if (!req->payload_sync &&
req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) {
/* got early response from server while we're still sending request
payload. we cannot recover from this reliably, so we stop sending
payload and close the connection once the response is processed */
http_client_connection_debug(conn,
"Got early input from server; "
"request payload not completely sent (will close connection)");
o_stream_unset_flush_callback(conn->conn.output);
conn->output_broken = early = TRUE;
}
http_client_connection_debug(conn,
"Got %u response for request %s (took %u ms + %u ms in queue)",
response.status, http_client_request_label(req),
timeval_diff_msecs(&req->response_time, &req->sent_time),
timeval_diff_msecs(&req->sent_time, &req->submit_time));
/* make sure connection output is unlocked if 100-continue failed */
if (req->payload_sync && !conn->payload_continue)
conn->output_locked = FALSE;
/* remove request from queue */
array_delete(&conn->request_wait_list, 0, 1);
aborted = (req->state == HTTP_REQUEST_STATE_ABORTED);
i_assert(req->refcount > 1 || aborted);
http_client_request_unref(&req);
conn->close_indicated = response.connection_close;
if (!aborted) {
bool handled = FALSE;
/* response cannot be 2xx if request payload was not completely sent
*/
if (early && response.status / 100 == 2) {
http_client_request_error(req,
HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE,
"Server responded with success response "
"before all payload was sent");
http_client_request_unref(&req);
http_client_connection_close(&conn);
return;
}
/* don't redirect/retry if we're sending data in small
blocks via http_client_request_send_payload()
and we're not waiting for 100 continue */
if (!req->payload_wait ||
(req->payload_sync && !conn->payload_continue)) {
/* failed Expect: */
if (response.status == 417 && req->payload_sync) {
/* drop Expect: continue */
req->payload_sync = FALSE;
conn->output_locked = FALSE;
conn->peer->no_payload_sync = TRUE;
if (http_client_request_try_retry(req))
handled = TRUE;
/* redirection */
} else if (!req->client->set.no_auto_redirect &&
response.status / 100 == 3 && response.status != 304 &&
response.location != NULL) {
/* redirect (possibly after delay) */
if (http_client_request_delay_from_response(req, &response) >= 0) {
http_client_request_redirect
(req, response.status, response.location);
handled = TRUE;
}
/* service unavailable */
} else if (response.status == 503) {
/* automatically retry after delay if indicated */
if ( response.retry_after != (time_t)-1 &&
http_client_request_delay_from_response(req, &response) > 0 &&
http_client_request_try_retry(req))
handled = TRUE;
/* request timeout (by server) */
} else if (response.status == 408) {
/* automatically retry */
if (http_client_request_try_retry(req))
handled = TRUE;
/* connection close is implicit, although server should indicate
that explicitly */
conn->close_indicated = TRUE;
}
}
if (!handled) {
/* response for application */
if (!http_client_connection_return_response(conn, req, &response))
return;
}
}
finished++;
/* server closing connection? */
if (conn->close_indicated) {
http_client_connection_server_close(&conn);
return;
}
/* get next waiting request */
reqs = array_get(&conn->request_wait_list, &count);
if (count > 0) {
req = reqs[0];
/* determine whether to expect a response payload */
payload_type = http_client_request_get_payload_type(req);
} else {
/* no more requests waiting for the connection */
http_client_connection_stop_request_timeout(conn);
req = NULL;
payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED;
}
/* drop connection with broken output if last possible input was
received */
if (conn->output_broken && (count == 0 ||
(count == 1 && req->state == HTTP_REQUEST_STATE_ABORTED))) {
http_client_connection_server_close(&conn);
return;
}
}
if (ret <= 0 &&
(conn->conn.input->eof || conn->conn.input->stream_errno != 0)) {
int stream_errno = conn->conn.input->stream_errno;
http_client_connection_abort_temp_error(&conn,
HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
t_strdup_printf("Connection lost: read(%s) failed: %s",
i_stream_get_name(conn->conn.input),
stream_errno != 0 ?
i_stream_get_error(conn->conn.input) :
"EOF"));
return;
}
if (ret < 0) {
http_client_connection_abort_error(&conn,
HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, error);
return;
}
if (finished > 0) {
/* connection still alive after (at least one) request;
we can pipeline -> mark for subsequent connections */
conn->peer->allows_pipelining = TRUE;
/* room for new requests */
if (http_client_connection_is_ready(conn))
http_client_peer_trigger_request_handler(conn->peer);
}
}
int http_client_connection_output(struct http_client_connection *conn)
{
struct http_client_request *const *reqs;
struct ostream *output = conn->conn.output;
unsigned int count;
const char *error;
int ret;
/* we've seen activity from the server; reset request timeout */
http_client_connection_reset_request_timeout(conn);
if ((ret = o_stream_flush(output)) <= 0) {
if (ret < 0) {
http_client_connection_abort_temp_error(&conn,
HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
t_strdup_printf("Connection lost: write(%s) failed: %s",
o_stream_get_name(output),
o_stream_get_error(output)));
}
return ret;
}
i_assert(!conn->output_broken);
if (conn->ssl_iostream != NULL &&
!ssl_iostream_is_handshaked(conn->ssl_iostream))
return 1;
reqs = array_get(&conn->request_wait_list, &count);
if (count > 0 && conn->output_locked) {
struct http_client_request *req = reqs[count-1];
if (req->state == HTTP_REQUEST_STATE_ABORTED) {
http_client_connection_debug(conn,
"Request aborted before sending payload was complete.");
if (count == 1) {
http_client_connection_close(&conn);
} else {
o_stream_unset_flush_callback(output);
conn->output_broken = TRUE;
}
return 1;
}
if (!req->payload_sync || conn->payload_continue) {
if (http_client_request_send_more(req, &error) < 0) {
http_client_connection_abort_temp_error(&conn,
HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
t_strdup_printf("Connection lost: %s", error));
return -1;
}
if (!conn->output_locked) {
/* room for new requests */
if (http_client_connection_is_ready(conn))
http_client_peer_trigger_request_handler(conn->peer);
}
}
}
return 1;
}
void
http_client_connection_start_tunnel(struct http_client_connection **_conn,
struct http_client_tunnel *tunnel)
{
struct http_client_connection *conn = *_conn;
i_assert(conn->tunneling);
/* claim connection streams */
memset(tunnel, 0, sizeof(*tunnel));
tunnel->input = conn->conn.input;
tunnel->output = conn->conn.output;
tunnel->fd_in = conn->conn.fd_in;
tunnel->fd_out = conn->conn.fd_out;
/* detach from connection */
conn->conn.input = NULL;
conn->conn.output = NULL;
conn->conn.fd_in = -1;
conn->conn.fd_out = -1;
conn->closing = TRUE;
conn->connected = FALSE;
connection_disconnect(&conn->conn);
http_client_connection_unref(_conn);
}
static void
http_client_connection_ready(struct http_client_connection *conn)
{
http_client_connection_debug(conn, "Ready for requests");
/* connected */
conn->connected = TRUE;
conn->last_ioloop = current_ioloop;
if (conn->to_connect != NULL)
timeout_remove(&conn->to_connect);
/* indicate connection success */
conn->connect_succeeded = TRUE;
http_client_peer_connection_success(conn->peer);
/* start raw log */
if (conn->client->set.rawlog_dir != NULL) {
iostream_rawlog_create(conn->client->set.rawlog_dir,
&conn->conn.input, &conn->conn.output);
}
/* direct tunneling connections handle connect requests just by providing a
raw connection */
if (conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_RAW) {
struct http_client_request *req;
req = http_client_peer_claim_request(conn->peer, FALSE);
if (req != NULL) {
struct http_response response;
http_client_request_ref(req);
req->conn = conn;
conn->tunneling = TRUE;
memset(&response, 0, sizeof(response));
response.status = 200;
response.reason = "OK";
(void)http_client_connection_return_response(conn, req, &response);
http_client_request_unref(&req);
return;
}
http_client_connection_debug(conn,
"No raw connect requests pending; closing useless connection");
http_client_connection_close(&conn);
return;
}
/* start protocol I/O */
conn->http_parser = http_response_parser_init
(conn->conn.input, &conn->client->set.response_hdr_limits);
o_stream_set_flush_callback(conn->conn.output,
http_client_connection_output, conn);
}
static int
http_client_connection_ssl_handshaked(const char **error_r, void *context)
{
struct http_client_connection *conn = context;
const char *error, *host = conn->peer->addr.a.tcp.https_name;
if (ssl_iostream_check_cert_validity(conn->ssl_iostream, host, &error) == 0)
http_client_connection_debug(conn, "SSL handshake successful");
else if (conn->client->set.ssl_allow_invalid_cert) {
http_client_connection_debug(conn, "SSL handshake successful, "
"ignoring invalid certificate: %s", error);
} else {
*error_r = error;
return -1;
}
return 0;
}
static int
http_client_connection_ssl_init(struct http_client_connection *conn,
const char **error_r)
{
struct ssl_iostream_settings ssl_set;
const char *error;
i_assert(conn->client->ssl_ctx != NULL);
memset(&ssl_set, 0, sizeof(ssl_set));
if (!conn->client->set.ssl_allow_invalid_cert) {
ssl_set.verbose_invalid_cert = TRUE;
ssl_set.verify_remote_cert = TRUE;
ssl_set.require_valid_cert = TRUE;
}
if (conn->client->set.debug)
http_client_connection_debug(conn, "Starting SSL handshake");
if (io_stream_create_ssl_client(conn->client->ssl_ctx,
conn->peer->addr.a.tcp.https_name, &ssl_set,
&conn->conn.input, &conn->conn.output,
&conn->ssl_iostream, &error) < 0) {
*error_r = t_strdup_printf(
"Couldn't initialize SSL client for %s: %s",
conn->conn.name, error);
return -1;
}
ssl_iostream_set_handshake_callback(conn->ssl_iostream,
http_client_connection_ssl_handshaked, conn);
if (ssl_iostream_handshake(conn->ssl_iostream) < 0) {
*error_r = t_strdup_printf("SSL handshake to %s failed: %s",
conn->conn.name, ssl_iostream_get_last_error(conn->ssl_iostream));
return -1;
}
if (ssl_iostream_is_handshaked(conn->ssl_iostream)) {
http_client_connection_ready(conn);
} else {
/* wait for handshake to complete; connection input handler does the rest
by reading from the input stream */
o_stream_set_flush_callback(conn->conn.output,
http_client_connection_output, conn);
}
return 0;
}
static void
http_client_connection_connected(struct connection *_conn, bool success)
{
struct http_client_connection *conn =
(struct http_client_connection *)_conn;
const char *error;
if (!success) {
http_client_peer_connection_failure(conn->peer, t_strdup_printf(
"connect(%s) failed: %m", _conn->name));
} else {
conn->connected_timestamp = ioloop_timeval;
http_client_connection_debug(conn, "Connected");
if (http_client_peer_addr_is_https(&conn->peer->addr)) {
if (http_client_connection_ssl_init(conn, &error) < 0) {
http_client_peer_connection_failure(conn->peer, error);
http_client_connection_debug(conn, "%s", error);
http_client_connection_close(&conn);
}
return;
}
http_client_connection_ready(conn);
}
}
static const struct connection_settings http_client_connection_set = {
.input_max_size = (size_t)-1,
.output_max_size = (size_t)-1,
.client = TRUE,
.delayed_unix_client_connected_callback = TRUE
};
static const struct connection_vfuncs http_client_connection_vfuncs = {
.destroy = http_client_connection_destroy,
.input = http_client_connection_input,
.client_connected = http_client_connection_connected
};
struct connection_list *
http_client_connection_list_init(void)
{
return connection_list_init
(&http_client_connection_set, &http_client_connection_vfuncs);
}
static void
http_client_connection_delayed_connect_error(struct http_client_connection *conn)
{
timeout_remove(&conn->to_input);
errno = conn->connect_errno;
http_client_connection_connected(&conn->conn, FALSE);
http_client_connection_close(&conn);
}
static void http_client_connect_timeout(struct http_client_connection *conn)
{
conn->conn.disconnect_reason = CONNECTION_DISCONNECT_CONNECT_TIMEOUT;
http_client_connection_destroy(&conn->conn);
}
static void
http_client_connection_connect(struct http_client_connection *conn)
{
unsigned int msecs;
conn->connect_start_timestamp = ioloop_timeval;
if (connection_client_connect(&conn->conn) < 0) {
conn->connect_errno = errno;
http_client_connection_debug(conn, "Connect failed: %m");
conn->to_input = timeout_add_short(0,
http_client_connection_delayed_connect_error, conn);
return;
}
/* don't use connection.h timeout because we want this timeout
to include also the SSL handshake */
msecs = conn->client->set.connect_timeout_msecs;
if (msecs == 0)
msecs = conn->client->set.request_timeout_msecs;
if (msecs > 0) {
conn->to_connect =
timeout_add(msecs, http_client_connect_timeout, conn);
}
}
static void
http_client_connect_tunnel_timeout(struct http_client_connection *conn)
{
const char *error, *name = http_client_peer_addr2str(&conn->peer->addr);
unsigned int msecs;
msecs = timeval_diff_msecs(&ioloop_timeval,
&conn->connect_start_timestamp);
error = t_strdup_printf(
"Tunnel connect(%s) failed: "
"Connection timed out in %u.%03u secs",
name, msecs/1000, msecs%1000);
http_client_connection_debug(conn, "%s", error);
http_client_peer_connection_failure(conn->peer, error);
http_client_connection_close(&conn);
}
static void
http_client_connection_tunnel_response(const struct http_response *response,
struct http_client_connection *conn)
{
struct http_client_tunnel tunnel;
const char *name = http_client_peer_addr2str(&conn->peer->addr);
if (response->status != 200) {
http_client_peer_connection_failure(conn->peer, t_strdup_printf(
"Tunnel connect(%s) failed: %d %s", name,
response->status, response->reason));
conn->connect_request = NULL;
return;
}
http_client_request_start_tunnel(conn->connect_request, &tunnel);
conn->connect_request = NULL;
connection_init_from_streams
(conn->client->conn_list, &conn->conn, name, tunnel.input, tunnel.output);
i_stream_unref(&tunnel.input);
o_stream_unref(&tunnel.output);
conn->connect_initialized = TRUE;
}
static void
http_client_connection_connect_tunnel(struct http_client_connection *conn,
const struct ip_addr *ip, in_port_t port)
{
unsigned int msecs;
conn->connect_start_timestamp = ioloop_timeval;
conn->connect_request = http_client_request_connect_ip
(conn->client, ip, port, http_client_connection_tunnel_response, conn);
http_client_request_set_urgent(conn->connect_request);
http_client_request_submit(conn->connect_request);
/* don't use connection.h timeout because we want this timeout
to include also the SSL handshake */
msecs = conn->client->set.connect_timeout_msecs;
if (msecs == 0)
msecs = conn->client->set.request_timeout_msecs;
if (msecs > 0) {
conn->to_connect =
timeout_add(msecs, http_client_connect_tunnel_timeout, conn);
}
}
struct http_client_connection *
http_client_connection_create(struct http_client_peer *peer)
{
struct http_client_connection *conn;
static unsigned int id = 0;
const struct http_client_peer_addr *addr = &peer->addr;
const char *conn_type = "UNKNOWN";
switch (peer->addr.type) {
case HTTP_CLIENT_PEER_ADDR_HTTP:
conn_type = "HTTP";
break;
case HTTP_CLIENT_PEER_ADDR_HTTPS:
conn_type = "HTTPS";
break;
case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL:
conn_type = "Tunneled HTTPS";
break;
case HTTP_CLIENT_PEER_ADDR_RAW:
conn_type = "Raw";
break;
case HTTP_CLIENT_PEER_ADDR_UNIX:
conn_type = "Unix";
break;
}
conn = i_new(struct http_client_connection, 1);
conn->refcount = 1;
conn->client = peer->client;
conn->id = id++;
conn->peer = peer;
if (peer->addr.type != HTTP_CLIENT_PEER_ADDR_RAW)
i_array_init(&conn->request_wait_list, 16);
switch (peer->addr.type) {
case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL:
http_client_connection_connect_tunnel
(conn, &addr->a.tcp.ip, addr->a.tcp.port);
break;
case HTTP_CLIENT_PEER_ADDR_UNIX:
connection_init_client_unix(peer->client->conn_list, &conn->conn,
addr->a.un.path);
conn->connect_initialized = TRUE;
http_client_connection_connect(conn);
break;
default:
connection_init_client_ip(peer->client->conn_list, &conn->conn,
&addr->a.tcp.ip, addr->a.tcp.port);
conn->connect_initialized = TRUE;
http_client_connection_connect(conn);
}
array_append(&peer->conns, &conn, 1);
http_client_connection_debug(conn,
"%s connection created (%d parallel connections exist)%s",
conn_type, array_count(&peer->conns),
(conn->to_input == NULL ? "" : " [broken]"));
return conn;
}
void http_client_connection_ref(struct http_client_connection *conn)
{
i_assert(conn->refcount > 0);
conn->refcount++;
}
static void
http_client_connection_disconnect(struct http_client_connection *conn)
{
conn->closing = TRUE;
conn->connected = FALSE;
if (conn->connect_request != NULL)
http_client_request_abort(&conn->connect_request);
if (conn->incoming_payload != NULL) {
/* the stream is still accessed by lib-http caller. */
i_stream_remove_destroy_callback(conn->incoming_payload,
http_client_payload_destroyed);
conn->incoming_payload = NULL;
}
if (conn->connect_initialized)
connection_disconnect(&conn->conn);
if (conn->io_req_payload != NULL)
io_remove(&conn->io_req_payload);
if (conn->to_requests != NULL)
timeout_remove(&conn->to_requests);
if (conn->to_connect != NULL)
timeout_remove(&conn->to_connect);
if (conn->to_input != NULL)
timeout_remove(&conn->to_input);
if (conn->to_idle != NULL)
timeout_remove(&conn->to_idle);
if (conn->to_response != NULL)
timeout_remove(&conn->to_response);
}
void http_client_connection_unref(struct http_client_connection **_conn)
{
struct http_client_connection *conn = *_conn;
struct http_client_connection *const *conn_idx;
ARRAY_TYPE(http_client_connection) *conn_arr;
struct http_client_peer *peer = conn->peer;
struct http_client_request **req;
i_assert(conn->refcount > 0);
if (--conn->refcount > 0)
return;
http_client_connection_debug(conn, "Connection destroy");
http_client_connection_disconnect(conn);
/* abort all pending requests (not supposed to happen here) */
if (array_is_created(&conn->request_wait_list)) {
array_foreach_modifiable(&conn->request_wait_list, req) {
i_assert((*req)->submitted);
http_client_request_error(*req,
HTTP_CLIENT_REQUEST_ERROR_ABORTED,
"Aborting");
http_client_request_unref(req);
}
array_free(&conn->request_wait_list);
}
if (conn->pending_request != NULL) {
struct http_client_request *pending_req = conn->pending_request;
conn->pending_request = NULL;
http_client_request_error(pending_req,
HTTP_CLIENT_REQUEST_ERROR_ABORTED,
"Aborting");
http_client_request_unref(&pending_req);
}
if (conn->http_parser != NULL)
http_response_parser_deinit(&conn->http_parser);
if (conn->ssl_iostream != NULL)
ssl_iostream_unref(&conn->ssl_iostream);
if (conn->connect_initialized)
connection_deinit(&conn->conn);
/* remove this connection from the list */
conn_arr = &conn->peer->conns;
array_foreach(conn_arr, conn_idx) {
if (*conn_idx == conn) {
array_delete(conn_arr, array_foreach_idx(conn_arr, conn_idx), 1);
break;
}
}
if (conn->connect_succeeded)
http_client_peer_connection_lost(peer);
i_free(conn);
*_conn = NULL;
}
void http_client_connection_close(struct http_client_connection **_conn)
{
struct http_client_connection *conn = *_conn;
http_client_connection_debug(conn, "Connection close");
http_client_connection_disconnect(conn);
http_client_connection_unref(_conn);
}
void http_client_connection_switch_ioloop(struct http_client_connection *conn)
{
if (conn->io_req_payload != NULL)
conn->io_req_payload = io_loop_move_io(&conn->io_req_payload);
if (conn->to_requests != NULL)
conn->to_requests = io_loop_move_timeout(&conn->to_requests);
if (conn->to_connect != NULL)
conn->to_connect = io_loop_move_timeout(&conn->to_connect);
if (conn->to_input != NULL)
conn->to_input = io_loop_move_timeout(&conn->to_input);
if (conn->to_idle != NULL)
conn->to_idle = io_loop_move_timeout(&conn->to_idle);
if (conn->to_response != NULL)
conn->to_response = io_loop_move_timeout(&conn->to_response);
if (conn->incoming_payload != NULL)
i_stream_switch_ioloop(conn->incoming_payload);
connection_switch_ioloop(&conn->conn);
}