http-client-peer.c revision 1be73c463dba613f52c586013cf271330ff6e4e1
/* Copyright (c) 2013-2017 Dovecot authors, see the included COPYING file */
#include "lib.h"
#include "net.h"
#include "time-util.h"
#include "str.h"
#include "hash.h"
#include "array.h"
#include "llist.h"
#include "istream.h"
#include "ostream.h"
#include "iostream-ssl.h"
#include "http-response-parser.h"
#include "http-client-private.h"
/*
* Logging
*/
/* Peer pool */
static inline void
static inline void
const char *format, ...)
{
i_debug("http-client: peer %s: %s",
}
}
/* Peer */
static inline void
static inline void
const char *format, ...)
{
i_debug("http-client: peer %s: %s",
}
}
/*
* Peer address
*/
unsigned int http_client_peer_addr_hash
(const struct http_client_peer_addr *peer)
{
/* fall through */
break;
break;
}
return hash;
}
(const struct http_client_peer_addr *peer1,
const struct http_client_peer_addr *peer2)
{
int ret;
/* Queues are created with peer addresses that have an uninitialized
IP value, because that is assigned later when the host lookup completes.
In all other other contexts, the IP is always initialized, so we do not
compare IPs when one of them is unassigned. */
return ret;
return 0;
return null_strcmp
}
i_unreached();
}
/*
* Peer pool
*/
static struct http_client_peer_pool *
{
struct http_client_peer_pool *ppool;
return ppool;
}
{
return;
}
{
struct http_client_connection **conn;
/* make a copy of the connection array; freed connections modify it */
}
{
return;
return;
}
/*
* Peer
*/
static void
const char *
{
break;
default:
}
}
}
static struct http_client_peer *
const struct http_client_peer_addr *addr)
{
struct http_client_peer *peer;
break;
break;
break;
default:
break;
}
return peer;
}
{
}
static void
{
struct http_client_connection **conn;
struct http_client_queue *const *queue;
if (peer->disconnected)
return;
/* make a copy of the connection array; freed connections modify it */
}
/* unlist in client */
/* unlink all queues */
}
static void
unsigned int count)
{
struct http_client_connection *const *idle_conns;
unsigned int i, idle_count;
if (count == 0)
return;
for (i = 0; i < count && i < idle_count; i++) {
"Using idle connection (connections=%u)",
}
for (; i < count; i++) {
(void)http_client_connection_create(peer);
}
}
static void
{
"Backoff timer expired");
return;
}
}
static bool
{
return TRUE;
int backoff_time_spent =
"Starting backoff timer for %d msecs",
return TRUE;
}
"Backoff time already exceeded by %d msecs",
}
return FALSE;
}
static void
{
if (peer->backoff_time_msecs == 0)
else
}
static void
{
peer->backoff_time_msecs = 0;
}
static void
{
return;
}
{
struct http_client_connection *const *conn_idx;
return TRUE;
return TRUE;
}
return FALSE;
}
static void
{
struct http_client_connection **conn;
/* make a copy of the connection array; freed connections modify it */
if (!http_client_connection_is_active(*conn))
}
}
static unsigned int
unsigned int *num_urgent_r)
{
struct http_client_queue *const *queue;
num_requests += requests;
num_urgent += urgent;
}
return num_requests;
}
{
struct http_client_connection *const *conn_idx;
unsigned int num_urgent = 0;
/* no connections or pending requests; disconnect immediately */
return;
}
/* check all connections for idle status */
}
}
static void
{
struct _conn_available {
struct http_client_connection *conn;
unsigned int pending_requests;
};
struct http_client_connection *const *conn_idx;
struct _conn_available *conn_avail_idx;
struct http_client_peer *tmp_peer;
bool statistics_dirty = TRUE;
/* FIXME: limit the number of requests handled in one run to prevent
I/O starvation. */
/* disconnect pending connections if we're not linked to any queue
anymore */
/* peer is completely unused and inactive; drop it immediately */
return;
}
"Peer no longer used; will now cancel pending connections "
return;
}
/* don't do anything unless we have pending requests */
if (num_pending == 0) {
"No requests to service for this peer "
return;
}
do {
/* gather connection statistics */
int ret;
break;
} else if (ret > 0) {
struct _conn_available *conn_avail;
unsigned int insert_idx, pending_requests;
/* compile sorted availability list */
if (array_count(&conns_avail) == 0) {
insert_idx = 0;
} else {
break;
}
}
}
if (pending_requests == 0)
idle++;
}
/* count the number of connecting and closing connections */
closing++;
connecting++;
}
if (conn_lost) {
/* connection array changed while iterating; retry */
continue;
}
/* use idle connections right away */
if (idle > 0) {
"Using %u idle connections to handle %u requests "
"(%u total connections ready)",
break;
idle--;
} else {
/* update statistics */
if (num_urgent > 0)
num_urgent--;
num_pending--;
}
}
}
/* don't continue unless we have more pending requests */
if (num_pending == 0) {
"No more requests to service for this peer "
break;
}
} while (statistics_dirty);
if (!http_client_peer_unref(&tmp_peer))
return;
if (num_pending == 0)
return;
/* determine how many new connections we can set up */
working_conn_count == connecting) {
/* don't create new connections until the existing ones have
finished connecting successfully. */
new_connections = 0;
} else {
/* only create connections for urgent requests */
} else if (num_pending <= connecting) {
/* there are already enough connections being made */
new_connections = 0;
} else if (working_conn_count == connecting) {
/* no connections succeeded so far, don't hammer the server with more
than one connection attempt unless its urgent */
if (num_urgent > 0) {
} else {
}
} else if (num_pending - connecting >
/* create maximum allowed connections */
} else {
/* create as many connections as we need */
}
}
/* create connections */
if (new_connections > 0) {
"Creating %u new connections to handle requests "
"(already %u usable, connecting to %u, closing %u)",
return;
}
/* cannot create new connections for normal request; attempt pipelining */
if (working_conn_count - connecting >=
if (!peer->allows_pipelining) {
"Will not pipeline until peer has shown support");
return;
}
/* fill pipelines */
do {
handled = 0;
/* fill smallest pipelines first,
until all pipelines are filled to the same level */
continue;
if (pipeline_level == 0) {
break; /* restart from least busy connection */
}
/* pipeline it */
/* connection now unavailable */
} else {
/* successfully pipelined */
num_pending--;
handled++;
}
}
total_handled += handled;
"Pipelined %u requests (filled pipelines up to %u requests)",
return;
}
/* still waiting for connections to finish */
"No request handled; waiting for new connections");
return;
}
{
T_BEGIN {
} T_END;
}
{
/* trigger request handling through timeout */
}
}
{
return TRUE;
return FALSE;
}
{
(void)http_client_peer_unref(_peer);
}
{
unsigned int conns_active =
if (conns_active > 0) {
"Not dropping peer (%d connections active)",
return;
}
return;
"Dropping peer (waiting for backof timeout)");
/* will disconnect any pending connections */
} else {
"Dropping peer now");
/* drop peer immediately */
}
}
struct http_client_peer *
const struct http_client_peer_addr *addr)
{
struct http_client_peer *peer;
return peer;
}
struct http_client_queue *queue)
{
struct http_client_queue *const *queue_idx;
return TRUE;
}
return FALSE;
}
struct http_client_queue *queue)
{
"Linked queue %s (%d queues linked)",
}
}
struct http_client_queue *queue)
{
struct http_client_queue *const *queue_idx;
"Unlinked queue %s (%d queues linked)",
return;
}
}
}
struct http_client_request *
{
struct http_client_queue *const *queue_idx;
struct http_client_request *req;
return req;
}
}
return NULL;
}
{
struct http_client_queue *const *queue;
"Successfully connected (connections=%u)",
}
}
const char *reason)
{
struct http_client_queue *const *queue;
unsigned int pending;
/* count number of pending connections */
"Failed to make connection "
"(connections=%u, connecting=%u)",
/* manage backoff timer only when this was the only attempt */
if (pending == 1)
if (pending > 1) {
/* if there are other connections attempting to connect, wait
for them before failing the requests. remember that we had
trouble with connecting so in future we don't try to create
more than one connection until connects work again. */
} else {
failed. a second connect will probably also fail, so just
try another IP for the hosts(s) or abort all requests if this
}
}
}
bool premature)
{
unsigned int num_pending, num_urgent;
/* we get here when an already connected connection fails. if the
connect itself fails, http_client_peer_connection_failure() is
called instead. */
if (peer->disconnected)
return;
"Lost a connection%s (%u queues linked, %u connections left, "
"%u requests pending, %u requests urgent)",
/* update backoff timer if the connection was lost prematurely.
this prevents reconnecting immediately to a server that is
misbehaving by disconnecting before sending a response.
*/
if (premature) {
}
if (peer->handling_requests) {
/* we got here from the request handler loop */
"Lost a connection while handling requests");
return;
}
/* if there are pending requests for this peer, create a new connection
for them. if not, this peer will wind itself down. */
}
unsigned int
{
struct http_client_connection *const *conn_idx;
unsigned int idle = 0;
/* find idle connections */
idle++;
}
return idle;
}
unsigned int
{
struct http_client_connection *const *conn_idx;
unsigned int active = 0;
/* find idle connections */
active++;
}
return active;
}
unsigned int
{
struct http_client_connection *const *conn_idx;
unsigned int pending = 0;
/* find idle connections */
pending++;
}
return pending;
}
{
}
peer->to_backoff =
}
}