child.c revision f95bb67198df2d264e130a7f4edac3262644c76e
/* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifdef WIN32
#include "httpd.h"
#include "http_main.h"
#include "http_log.h"
#include "http_config.h" /* for read_config */
#include "http_core.h" /* for get_remote_host */
#include "http_connection.h"
#include "apr_portable.h"
#include "apr_thread_proc.h"
#include "apr_getopt.h"
#include "apr_strings.h"
#include "apr_lib.h"
#include "apr_shm.h"
#include "apr_thread_mutex.h"
#include "ap_mpm.h"
#include "ap_config.h"
#include "ap_listen.h"
#include "mpm_default.h"
#include "mpm_winnt.h"
#include "mpm_common.h"
#include <malloc.h>
#include "apr_atomic.h"
#ifdef __MINGW32__
#include <mswsock.h>
#endif
/* shared with mpm_winnt.c */
/* used by parent to signal the child to start and exit */
/* shared with mpm_winnt.c, but should be private to child.c */
/* Queue for managing the passing of winnt_conn_ctx_t between
* the accept and worker threads.
*/
typedef struct winnt_conn_ctx_t_s {
struct winnt_conn_ctx_t_s *next;
int sa_server_len;
int sa_client_len;
short socket_family;
typedef enum {
IOCP_SHUTDOWN = 4
} io_state_e;
static apr_pool_t *pchild;
static int shutdown_in_progress = 0;
static int workers_may_exit = 0;
static unsigned int g_blocked_threads = 0;
static HANDLE max_requests_per_child_event;
static apr_thread_mutex_t *child_lock;
static apr_thread_mutex_t *qlock;
static int num_completion_contexts = 0;
static int max_num_completion_contexts = 0;
{
/* Recycle the completion context.
* - clear the ptrans pool
* - put the context on the queue to be consumed by the accept thread
* Note:
* context->accept_socket may be in a disconnected but reusable
* state so -don't- close it.
*/
if (context) {
if (qtail) {
} else {
}
}
}
static winnt_conn_ctx_t *mpm_get_completion_context(void)
{
while (1) {
/* Grab a context off the queue */
if (qhead) {
if (!qhead)
} else {
}
if (!context) {
/* We failed to grab a context off the queue, consider allocating
* a new one out of the child pool. There may be up to
* (ap_threads_per_child + num_listeners) contexts in the system
* at once.
*/
/* All workers are busy, need to wait for one */
static int reported = 0;
if (!reported) {
"Server ran out of threads to serve "
"requests. Consider raising the "
"ThreadsPerChild setting");
reported = 1;
}
/* Wait for a worker to free a context. Once per second, give
* the caller a chance to check for shutdown. If the wait
* succeeds, get the context off the queue. It must be
* available, since there's only one consumer.
*/
if (rv == WAIT_OBJECT_0)
continue;
else /* Hopefully, WAIT_TIMEOUT */
return NULL;
} else {
/* Allocate another context.
* Note: Multiple failures in the next two steps will cause
* the pchild pool to 'leak' storage. I don't think this
* is worth fixing...
*/
sizeof(winnt_conn_ctx_t));
/* Hopefully this is a temporary condition ... */
"mpm_get_completion_context: "
"CreateEvent failed.");
return NULL;
}
/* Create the transaction pool */
if (rv != APR_SUCCESS) {
"mpm_get_completion_context: Failed "
"to create the transaction pool.");
return NULL;
}
break;
}
} else {
/* Got a context from the queue */
break;
}
}
return context;
}
{
if (context)
else
pOverlapped = NULL;
return APR_SUCCESS;
}
/*
* find_ready_listener()
* Only used by Win9* and should go away when the win9*_accept() function is
* reimplemented using apr_poll().
*/
static ap_listen_rec *head_listener;
{
lr = head_listener;
do {
if (!head_listener) {
}
return lr;
}
if (!lr) {
lr = ap_listeners;
}
} while (lr != head_listener);
return NULL;
}
/* Windows NT/2000 specific code...
* model. An accept thread accepts connections off the network then issues
* PostQueuedCompletionStatus() to awake a thread blocked on the ThreadDispatch
* IOCompletionPort.
*
* winnt_accept()
* One or more accept threads run in this function, each of which accepts
* connections off the network and calls PostQueuedCompletionStatus() to
* queue an io completion packet to the ThreadDispatch IOCompletionPort.
* winnt_get_connection()
* Worker threads block on the ThreadDispatch IOCompletionPort awaiting
* connections to service.
*/
#define MAX_ACCEPTEX_ERR_COUNT 100
{
#if APR_HAVE_IPV6
#endif
#if APR_HAVE_IPV6
"winnt_accept: getsockname error on listening socket, "
"is IPv6 available?");
return 1;
}
#endif
"Child %d: Starting thread to listen on port %d.",
while (!shutdown_in_progress) {
if (!context) {
if (!context) {
/* Temporary resource constraint? */
Sleep(0);
continue;
}
}
/* Create and initialize the accept socket */
#if APR_HAVE_IPV6
}
}
"winnt_accept: Failed to allocate an accept socket. "
"Temporary resource constraint? Try again.");
Sleep(100);
continue;
}
#else
/* Another temporary condition? */
"winnt_accept: Failed to allocate an accept "
"socket. Temporary resource constraint? "
"Retrying.");
Sleep(100);
continue;
}
}
#endif
/* AcceptEx on the completion context. The completion context will be
* signaled when a connection is accepted.
*/
0,
&context->Overlapped)) {
rv = apr_get_netos_error();
/* We can get here when:
* 1) the client disconnects early
* 2) TransmitFile does not properly recycle the accept socket (typically
* because the client disconnected)
* 3) there is VPN or Firewall software installed with buggy AcceptEx implementation
* 4) the webserver is using a dynamic address that has changed
*/
++err_count;
if (err_count > MAX_ACCEPTEX_ERR_COUNT) {
"Child %d: Encountered too many AcceptEx "
"faults accepting client connections. "
"Possible causes: dynamic address renewal, "
"or incompatible VPN or firewall software. "
"Try the directive 'AcceptFilter none'.",
my_pid);
err_count = 0;
}
continue;
}
++err_count;
if (err_count > MAX_ACCEPTEX_ERR_COUNT) {
"Child %d: Encountered too many AcceptEx "
"faults accepting client connections. "
"Possible causes: Unknown. "
"Try the directive 'AcceptFilter none'.",
my_pid);
err_count = 0;
}
continue;
}
err_count = 0;
/* Wait for pending i/o.
* Wake up once per second to check for shutdown .
* XXX: We should be waiting on exit_event instead of polling
*/
while (1) {
if (rv == WAIT_OBJECT_0) {
/* socket already closed */
break;
}
"winnt_accept: Asynchronous AcceptEx failed.");
}
break;
}
/* WAIT_TIMEOUT */
if (shutdown_in_progress) {
break;
}
}
continue;
}
}
err_count = 0;
/* Inherit the listen socket settings. Required for
* shutdown() to work
*/
SO_UPDATE_ACCEPT_CONTEXT, (char *)&nlsd,
sizeof(nlsd))) {
"setsockopt(SO_UPDATE_ACCEPT_CONTEXT) failed.");
/* Not a failure condition. Keep running. */
}
/* Get the local & remote address */
0,
&context->sa_client_len);
/* When a connection is received, send an io completion notification to
* the ThreadDispatchIOCP. This function could be replaced by
* mpm_post_completion_context(), but why do an extra function call...
*/
&context->Overlapped);
}
if (!shutdown_in_progress) {
/* Yow, hit an irrecoverable error! Tell the child to die. */
}
"Child %d: Accept thread exiting.", my_pid);
return 0;
}
{
int rc;
#ifdef _WIN64
#else
#endif
while (1) {
if (workers_may_exit) {
return NULL;
}
if (!rc) {
rc = apr_get_os_error();
"Child %d: GetQueuedComplationStatus returned %d",
continue;
}
switch (CompKey) {
case IOCP_CONNECTION_ACCEPTED:
break;
case IOCP_SHUTDOWN:
return NULL;
default:
return NULL;
}
break;
}
return context;
}
/*
* worker_main()
* Main entry point for the worker threads. Worker threads block in
* win*_get_connection() awaiting a connection to service.
*/
{
static int requests_this_child = 0;
int thread_num = (int)thread_num_val;
while (1) {
conn_rec *c;
/* Grab a connection off the network */
if (!context) {
/* Time for the thread to exit */
break;
}
/* Have we hit MaxRequestPerChild connections? */
if (ap_max_requests_per_child) {
}
}
if (c) {
&disconnected);
if (!disconnected) {
}
else {
/* If the socket is disconnected but we are not using acceptex,
* we cannot reuse the socket. Disconnected sockets are removed
* from the apr_socket_t struct by apr_sendfile() to prevent the
* socket descriptor from being inadvertently closed by a call
* to apr_socket_close(), so close it directly.
*/
/* XXX Study me for NT;
* closesocket(context->accept_socket);
* context->accept_socket = INVALID_SOCKET;
*/
}
}
else {
/* ap_run_create_connection closes the socket on failure */
}
}
(request_rec *) NULL);
return 0;
}
int thread_to_clean)
{
int i;
(*thread_cnt)--;
}
/*
* child_main()
* Entry point for the main control thread for the child process.
* This thread creates the accept thread, worker threads and
* monitors the child process for maintenance and shutdown
* events.
*/
static void create_listener_thread()
{
int tid;
int num_listeners = 0;
/* Start an accept thread per listener
* XXX: Why would we have a NULL sd in our listeners?
*/
/* Number of completion_contexts allowed in the system is
* (ap_threads_per_child + num_listeners). We need the additional
* completion contexts to prevent server hangs when ThreadsPerChild
* is configured to something less than or equal to the number
* of listeners. This is not a usual case, but people have
* encountered it.
*/
}
/* Now start a thread per listener */
/* A smaller stack is sufficient.
* To convert to CreateThread, the returned handle cannot be
*/
}
}
}
{
apr_hash_t *ht;
int listener_started = 0;
int threads_created = 0;
int watch_thread;
int time_remains;
int cld;
int tid;
int rv;
int i;
/* Initialize the child_events */
if (!max_requests_per_child_event) {
"Child %d: Failed to create a max_requests event.",
my_pid);
}
child_events[0] = exit_event;
/*
* Wait until we have permission to start accepting connections.
* start_mutex is used to ensure that only one child ever
*/
if (status != APR_SUCCESS) {
"Child %d: Failed to acquire the start_mutex. "
"Process will exit.", my_pid);
}
"Child %d: Acquired the start mutex.", my_pid);
/*
* Create the worker thread dispatch IOCompletionPort
*/
/* Create the worker thread dispatch IOCP */
NULL, 0, 0);
if (!qwait_event) {
"Child %d: Failed to create a qwait event.", my_pid);
}
/*
* Create the pool of worker threads
*/
"Child %d: Starting %d worker threads.",
* sizeof(HANDLE));
while (1) {
for (i = 0; i < ap_threads_per_child; i++) {
int *score_idx;
continue;
}
worker_main, (void *) i,
stack_res_flag, &tid);
if (child_handles[i] == 0) {
"Child %d: CreateThread failed. Unable to "
"create all worker threads. Created %d of the %d "
"threads requested with the ThreadsPerChild "
"configuration directive.",
goto shutdown;
}
/* Save the score board index in ht keyed to the thread handle.
* We need this when cleaning up threads down below...
*/
*score_idx = i;
}
/* Start the listener only when workers are available */
if (!listener_started && threads_created) {
listener_started = 1;
}
if (threads_created == ap_threads_per_child) {
break;
}
/* Check to see if the child has been told to exit */
break;
}
/* wait for previous generation to clean up an entry in the scoreboard
*/
}
/* Wait for one of three events:
* exit_event:
* The exit_event is signaled by the parent process to notify
* the child that it is time to exit.
*
* max_requests_per_child_event:
* This event is signaled by the worker threads to indicate that
* the process has handled MaxRequestsPerChild connections.
*
* TIMEOUT:
* To do periodic maintenance on the server (check for thread exits,
* number of completion contexts, etc.)
*
* XXX: thread exits *aren't* being checked.
*
* XXX: other_child - we need the process handles to the other children
* in order to map them to apr_proc_other_child_read (which is not
* named well, it's more like a_p_o_c_died.)
*
* XXX: however - if we get a_p_o_c handle inheritance working, and
* the parent process creates other children and passes the pipes
* to our worker processes, then we have no business doing such
* things in the child_main loop, but should happen in master_main.
*/
while (1) {
#if !APR_HAS_OTHER_CHILD
#else
if (rv == WAIT_TIMEOUT) {
}
else
#endif
if (rv == WAIT_FAILED) {
/* Something serious is wrong */
"Child %d: WAIT_FAILED -- shutting down server",
my_pid);
break;
}
else if (cld == 0) {
/* Exit event was signaled */
"Child %d: Exit event signaled. Child process is "
"ending.", my_pid);
break;
}
else {
/* MaxRequestsPerChild event set by the worker threads.
* Signal the parent to restart
*/
"Child %d: Process exiting because it reached "
"MaxRequestsPerChild. Signaling the parent to "
"restart a new child process.", my_pid);
break;
}
}
/*
* Time to shutdown the child process
*/
/* Close the listening sockets. Note, we must close the listeners
* before closing any accept sockets pending in AcceptEx to prevent
* memory leaks in the kernel.
*/
}
/* Shutdown listener threads and pending AcceptEx socksts
* but allow the worker threads to continue consuming from
* the queue of accepted connections.
*/
shutdown_in_progress = 1;
Sleep(1000);
/* Tell the worker threads to exit */
workers_may_exit = 1;
/* Release the start_mutex to let the new process (in the restart
* scenario) a chance to begin accepting and servicing requests
*/
if (rv == APR_SUCCESS) {
"Child %d: Released the start mutex", my_pid);
}
else {
"Child %d: Failure releasing the start mutex", my_pid);
}
/* Shutdown the worker threads
* Post worker threads blocked on the ThreadDispatch IOCompletion port
*/
while (g_blocked_threads > 0) {
"Child %d: %d threads blocked on the completion port",
for (i=g_blocked_threads; i > 0; i--) {
}
Sleep(1000);
}
/* Empty the accept queue of completion contexts */
while (qhead) {
}
/* Give busy threads a chance to service their connections,
* (no more than the global server timeout period which
* we track in msec remaining).
*/
watch_thread = 0;
while (threads_created)
{
int nFailsafe = MAXIMUM_WAIT_OBJECTS;
/* Every time we roll over to wait on the first group
* of MAXIMUM_WAIT_OBJECTS threads, take a breather,
* and infrequently update the error log.
*/
if (watch_thread >= threads_created) {
if ((time_remains -= 100) < 0)
break;
/* Every 30 seconds give an update */
if ((time_remains % 30000) == 0) {
"Child %d: Waiting %d more seconds "
"for %d worker threads to finish.",
}
/* We'll poll from the top, 10 times per second */
Sleep(100);
watch_thread = 0;
}
/* Fairness, on each iteration we will pick up with the thread
* after the one we just removed, even if it's a single thread.
* We don't block here.
*/
child_handles + watch_thread, 0, 0);
if (dwRet == WAIT_FAILED) {
break;
}
if (dwRet == WAIT_TIMEOUT) {
/* none ready */
continue;
}
else if (dwRet >= WAIT_ABANDONED_0) {
/* We just got the ownership of the object, which
* should happen at most MAXIMUM_WAIT_OBJECTS times.
* It does NOT mean that the object is signaled.
*/
if ((nFailsafe--) < 1)
break;
}
else {
if (watch_thread >= threads_created)
break;
}
}
/* Kill remaining threads off the hard way */
if (threads_created) {
"Child %d: Terminating %d threads that failed to exit.",
}
for (i = 0; i < threads_created; i++) {
int *score_idx;
CloseHandle(child_handles[i]);
/* Reset the scoreboard entry for the thread we just whacked */
if (score_idx) {
}
}
"Child %d: All worker threads have exited.", my_pid);
}
#endif /* def WIN32 */