mpm/winnt/child.c

	child.c revision a2c036f0ca71e35c085b4cd9451a6d3718bc65da
d3ed5b56cb6b58f87ffd125bed48f7668f13de1edirkx/* ====================================================================
893328ef6ff86d0ca27774778d84410353789fb0fielding * The Apache Software License, Version 1.1
842ae4bd224140319ae7feec1872b93dfd491143fielding *
842ae4bd224140319ae7feec1872b93dfd491143fielding * Copyright (c) 2000-2003 The Apache Software Foundation.  All rights
842ae4bd224140319ae7feec1872b93dfd491143fielding * reserved.
842ae4bd224140319ae7feec1872b93dfd491143fielding *
842ae4bd224140319ae7feec1872b93dfd491143fielding * Redistribution and use in source and binary forms, with or without
842ae4bd224140319ae7feec1872b93dfd491143fielding * modification, are permitted provided that the following conditions
893328ef6ff86d0ca27774778d84410353789fb0fielding * are met:
0202d2114cc6d7042995100519cce45c808c153bnd *
893328ef6ff86d0ca27774778d84410353789fb0fielding * 1. Redistributions of source code must retain the above copyright
0202d2114cc6d7042995100519cce45c808c153bnd *    notice, this list of conditions and the following disclaimer.
0202d2114cc6d7042995100519cce45c808c153bnd *
0202d2114cc6d7042995100519cce45c808c153bnd * 2. Redistributions in binary form must reproduce the above copyright
0202d2114cc6d7042995100519cce45c808c153bnd *    notice, this list of conditions and the following disclaimer in
0202d2114cc6d7042995100519cce45c808c153bnd *    the documentation and/or other materials provided with the
893328ef6ff86d0ca27774778d84410353789fb0fielding *    distribution.
893328ef6ff86d0ca27774778d84410353789fb0fielding *
893328ef6ff86d0ca27774778d84410353789fb0fielding * 3. The end-user documentation included with the redistribution,
893328ef6ff86d0ca27774778d84410353789fb0fielding *    if any, must include the following acknowledgment:
0d50a692ff2ac7bdb42e417737ed86ebf0a41671ben *       "This product includes software developed by the
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *        Apache Software Foundation (http://www.apache.org/)."
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *    Alternately, this acknowledgment may appear in the software itself,
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *    if and wherever such third-party acknowledgments normally appear.
90b402e944318ae02afd50911eae6da1910f661dpquerna *
928f622b54e87afbbaba6add8aef8066ca16a040wrowe * 4. The names "Apache" and "Apache Software Foundation" must
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *    not be used to endorse or promote products derived from this
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *    software without prior written permission. For written
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *    permission, please contact apache@apache.org.
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * 5. Products derived from this software may not be called "Apache",
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *    nor may "Apache" appear in their name, without prior written
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *    permission of the Apache Software Foundation.
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * SUCH DAMAGE.
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * ====================================================================
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz *
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * This software consists of voluntary contributions made by many
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * individuals on behalf of the Apache Software Foundation.  For more
893328ef6ff86d0ca27774778d84410353789fb0fielding * information on the Apache Software Foundation, please see
893328ef6ff86d0ca27774778d84410353789fb0fielding * <http://www.apache.org/>.
893328ef6ff86d0ca27774778d84410353789fb0fielding *
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz * Portions of this software are based upon public domain software
0b7037d0efc1cd510ee12e8a35c982c3f2da0353pquerna * originally written at the National Center for Supercomputing Applications,
90b402e944318ae02afd50911eae6da1910f661dpquerna * University of Illinois, Urbana-Champaign.
90b402e944318ae02afd50911eae6da1910f661dpquerna */
0b7037d0efc1cd510ee12e8a35c982c3f2da0353pquerna
90b402e944318ae02afd50911eae6da1910f661dpquerna#ifdef WIN32
90b402e944318ae02afd50911eae6da1910f661dpquerna
c5694b1d7dca4f561ebce416b5ffacf856f825aawrowe#define CORE_PRIVATE
90b402e944318ae02afd50911eae6da1910f661dpquerna#include "httpd.h"
90b402e944318ae02afd50911eae6da1910f661dpquerna#include "http_main.h"
90b402e944318ae02afd50911eae6da1910f661dpquerna#include "http_log.h"
90b402e944318ae02afd50911eae6da1910f661dpquerna#include "http_config.h"    /* for read_config */
90b402e944318ae02afd50911eae6da1910f661dpquerna#include "http_core.h"      /* for get_remote_host */
90b402e944318ae02afd50911eae6da1910f661dpquerna#include "http_connection.h"
90b402e944318ae02afd50911eae6da1910f661dpquerna#include "apr_portable.h"
90b402e944318ae02afd50911eae6da1910f661dpquerna#include "apr_thread_proc.h"
893328ef6ff86d0ca27774778d84410353789fb0fielding#include "apr_getopt.h"
90b402e944318ae02afd50911eae6da1910f661dpquerna#include "apr_strings.h"
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz#include "apr_lib.h"
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz#include "apr_shm.h"
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz#include "apr_thread_mutex.h"
893328ef6ff86d0ca27774778d84410353789fb0fielding#include "ap_mpm.h"
f14fe1f44cb7018927ee0e31eb959cba8dc3700bjorton#include "ap_config.h"
893328ef6ff86d0ca27774778d84410353789fb0fielding#include "ap_listen.h"
0b7037d0efc1cd510ee12e8a35c982c3f2da0353pquerna#include "mpm_default.h"
893328ef6ff86d0ca27774778d84410353789fb0fielding#include "mpm_winnt.h"
90b402e944318ae02afd50911eae6da1910f661dpquerna#include "mpm_common.h"
928f622b54e87afbbaba6add8aef8066ca16a040wrowe#include <malloc.h>
928f622b54e87afbbaba6add8aef8066ca16a040wrowe#include "apr_atomic.h"
928f622b54e87afbbaba6add8aef8066ca16a040wrowe
928f622b54e87afbbaba6add8aef8066ca16a040wrowe/* shared with mpm_winnt.c */
928f622b54e87afbbaba6add8aef8066ca16a040wroweextern DWORD my_pid;
928f622b54e87afbbaba6add8aef8066ca16a040wrowe
90b402e944318ae02afd50911eae6da1910f661dpquerna/* used by parent to signal the child to start and exit */
90b402e944318ae02afd50911eae6da1910f661dpquerna/* shared with mpm_winnt.c, but should be private to child.c */
c5694b1d7dca4f561ebce416b5ffacf856f825aawroweapr_proc_mutex_t *start_mutex;
90b402e944318ae02afd50911eae6da1910f661dpquernaHANDLE exit_event;
90b402e944318ae02afd50911eae6da1910f661dpquerna
90b402e944318ae02afd50911eae6da1910f661dpquerna/* child_main() should never need to modify is_graceful!?! */
90b402e944318ae02afd50911eae6da1910f661dpquernaextern int volatile is_graceful;
90b402e944318ae02afd50911eae6da1910f661dpquerna
90b402e944318ae02afd50911eae6da1910f661dpquerna
90b402e944318ae02afd50911eae6da1910f661dpquerna/* Queue for managing the passing of COMP_CONTEXTs between
90b402e944318ae02afd50911eae6da1910f661dpquerna * the accept and worker threads.
893328ef6ff86d0ca27774778d84410353789fb0fielding */
90b402e944318ae02afd50911eae6da1910f661dpquernastatic apr_pool_t *pchild;
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantzstatic int shutdown_in_progress = 0;
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantzstatic int workers_may_exit = 0;
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantzstatic unsigned int g_blocked_threads = 0;
893328ef6ff86d0ca27774778d84410353789fb0fieldingstatic HANDLE max_requests_per_child_event;
f14fe1f44cb7018927ee0e31eb959cba8dc3700bjorton
893328ef6ff86d0ca27774778d84410353789fb0fielding
0b7037d0efc1cd510ee12e8a35c982c3f2da0353pquernastatic apr_thread_mutex_t  *qlock;
0b7037d0efc1cd510ee12e8a35c982c3f2da0353pquernastatic PCOMP_CONTEXT qhead = NULL;
0b7037d0efc1cd510ee12e8a35c982c3f2da0353pquernastatic PCOMP_CONTEXT qtail = NULL;
0b7037d0efc1cd510ee12e8a35c982c3f2da0353pquernastatic int num_completion_contexts = 0;
893328ef6ff86d0ca27774778d84410353789fb0fieldingstatic HANDLE ThreadDispatchIOCP = NULL;
893328ef6ff86d0ca27774778d84410353789fb0fieldingstatic HANDLE qwait_event = NULL;
893328ef6ff86d0ca27774778d84410353789fb0fielding
893328ef6ff86d0ca27774778d84410353789fb0fielding
893328ef6ff86d0ca27774778d84410353789fb0fieldingAP_DECLARE(void) mpm_recycle_completion_context(PCOMP_CONTEXT context)
893328ef6ff86d0ca27774778d84410353789fb0fielding{
893328ef6ff86d0ca27774778d84410353789fb0fielding    /* Recycle the completion context.
623882f55701a4c5a6bded180f60d86a524ac341sascha     * - clear the ptrans pool
5a3fb2c128b4671fb5091eaea8861c759f50a4e6brianp     * - put the context on the queue to be consumed by the accept thread
623882f55701a4c5a6bded180f60d86a524ac341sascha     * Note:
e07615ab3b59412d79eebb8d5b12f488e77c14a7jorton     * context->accept_socket may be in a disconnected but reusable
e07615ab3b59412d79eebb8d5b12f488e77c14a7jorton     * state so -don't- close it.
c066cdcb75ca262e424345b347b86e7ce2285869jerenkrantz     */
c15906bab97b9b71b6dc08c1aa2f5c0b72b5ca4edreid    if (context) {
c15906bab97b9b71b6dc08c1aa2f5c0b72b5ca4edreid        apr_pool_clear(context->ptrans);
96e6cafca226a8a2a64a7bbdc634b5b2679c9e0csascha        context->next = NULL;
96e6cafca226a8a2a64a7bbdc634b5b2679c9e0csascha        ResetEvent(context->Overlapped.hEvent);
893328ef6ff86d0ca27774778d84410353789fb0fielding        apr_thread_mutex_lock(qlock);
893328ef6ff86d0ca27774778d84410353789fb0fielding        if (qtail) {
893328ef6ff86d0ca27774778d84410353789fb0fielding            qtail->next = context;
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz        } else {
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz            qhead = context;
893328ef6ff86d0ca27774778d84410353789fb0fielding            SetEvent(qwait_event);
893328ef6ff86d0ca27774778d84410353789fb0fielding        }
893328ef6ff86d0ca27774778d84410353789fb0fielding        qtail = context;
893328ef6ff86d0ca27774778d84410353789fb0fielding        apr_thread_mutex_unlock(qlock);
893328ef6ff86d0ca27774778d84410353789fb0fielding    }
c5694b1d7dca4f561ebce416b5ffacf856f825aawrowe}
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantzAP_DECLARE(PCOMP_CONTEXT) mpm_get_completion_context(void)
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz{
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz    apr_status_t rv;
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz    PCOMP_CONTEXT context = NULL;
59b1b6c3fd51c83c3bb9f02a8f08751335f9fb1dminfrin
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz    while (1) {
893328ef6ff86d0ca27774778d84410353789fb0fielding        /* Grab a context off the queue */
c5694b1d7dca4f561ebce416b5ffacf856f825aawrowe        apr_thread_mutex_lock(qlock);
c5694b1d7dca4f561ebce416b5ffacf856f825aawrowe        if (qhead) {
c5694b1d7dca4f561ebce416b5ffacf856f825aawrowe            context = qhead;
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz            qhead = qhead->next;
c5694b1d7dca4f561ebce416b5ffacf856f825aawrowe            if (!qhead)
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz                qtail = NULL;
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz        } else {
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz            ResetEvent(qwait_event);
59b1b6c3fd51c83c3bb9f02a8f08751335f9fb1dminfrin        }
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz        apr_thread_mutex_unlock(qlock);
893328ef6ff86d0ca27774778d84410353789fb0fielding
c5694b1d7dca4f561ebce416b5ffacf856f825aawrowe        if (!context) {
c5694b1d7dca4f561ebce416b5ffacf856f825aawrowe            /* We failed to grab a context off the queue, consider allocating a
1e5333ef1a0edb888d75c0cdd90b33fa7e89fc31wsanchez             * new one out of the child pool. There may be up to ap_threads_per_child
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz             * contexts in the system at once.
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz             */
99d360dcbb5ac2be27694be74cc6124dbadf3315jerenkrantz            if (num_completion_contexts >= ap_threads_per_child) {
1e5333ef1a0edb888d75c0cdd90b33fa7e89fc31wsanchez                /* All workers are busy, need to wait for one */
1d13cbde60ace1b56ca57b9f0f74168bb1288174trawick                static int reported = 0;
1d13cbde60ace1b56ca57b9f0f74168bb1288174trawick                if (!reported) {
80b1e2ff2e3515fdab14675684b2640ceb6ce1c7trawick                    ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
1d13cbde60ace1b56ca57b9f0f74168bb1288174trawick                                 "Server ran out of threads to serve requests. Consider "
9045837071eae7a4b55204a21f31ccee17749af3jerenkrantz                                 "raising the ThreadsPerChild setting");
ab44eb80e084a02a66a58336d6fcfbbe48310439trawick                    reported = 1;
ab44eb80e084a02a66a58336d6fcfbbe48310439trawick                }
9045837071eae7a4b55204a21f31ccee17749af3jerenkrantz
ab44eb80e084a02a66a58336d6fcfbbe48310439trawick                /* Wait for a worker to free a context. Once per second, give
ab44eb80e084a02a66a58336d6fcfbbe48310439trawick                 * the caller a chance to check for shutdown. If the wait
9045837071eae7a4b55204a21f31ccee17749af3jerenkrantz                 * succeeds, get the context off the queue. It must be available,
9045837071eae7a4b55204a21f31ccee17749af3jerenkrantz                 * since there's only one consumer.
893328ef6ff86d0ca27774778d84410353789fb0fielding                 */
753fdde6734149ee551065c7c3cd74838dc27cc3brianp                rv = WaitForSingleObject(qwait_event, 1000);
893328ef6ff86d0ca27774778d84410353789fb0fielding                if (rv == WAIT_OBJECT_0)
893328ef6ff86d0ca27774778d84410353789fb0fielding                    continue;
893328ef6ff86d0ca27774778d84410353789fb0fielding                else /* Hopefully, WAIT_TIMEOUT */
544c23d79a3a7596a3612224ed3ac9fa8372929djerenkrantz                    return NULL;
893328ef6ff86d0ca27774778d84410353789fb0fielding            } else {
893328ef6ff86d0ca27774778d84410353789fb0fielding                /* Allocate another context.
893328ef6ff86d0ca27774778d84410353789fb0fielding                 * Note:
544c23d79a3a7596a3612224ed3ac9fa8372929djerenkrantz                 * Multiple failures in the next two steps will cause the pchild pool
96e6cafca226a8a2a64a7bbdc634b5b2679c9e0csascha                 * to 'leak' storage. I don't think this is worth fixing...
e07615ab3b59412d79eebb8d5b12f488e77c14a7jorton                 */
e07615ab3b59412d79eebb8d5b12f488e77c14a7jorton                context = (PCOMP_CONTEXT) apr_pcalloc(pchild, sizeof(COMP_CONTEXT));
0464cb46220c1025cbfe1365a6bd352e67d2e766jorton
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                context->Overlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                if (context->Overlapped.hEvent == NULL) {
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                    /* Hopefully this is a temporary condition ... */
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                    ap_log_error(APLOG_MARK,APLOG_WARNING, apr_get_os_error(), ap_server_conf,
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                                 "mpm_get_completion_context: CreateEvent failed.");
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                    return NULL;
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                }
4028d805f3ab215e39bae405d35e001a538bc50eminfrin
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                /* Create the tranaction pool */
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                if ((rv = apr_pool_create(&context->ptrans, pchild)) != APR_SUCCESS) {
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                    ap_log_error(APLOG_MARK,APLOG_WARNING, rv, ap_server_conf,
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                                 "mpm_get_completion_context: Failed to create the transaction pool.");
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                    CloseHandle(context->Overlapped.hEvent);
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                    return NULL;
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                }
4028d805f3ab215e39bae405d35e001a538bc50eminfrin                apr_pool_tag(context->ptrans, "ptrans");
4028d805f3ab215e39bae405d35e001a538bc50eminfrin
f0528865c2e6c22ecc51eeecc496251489029bddjorton                context->accept_socket = INVALID_SOCKET;
f0528865c2e6c22ecc51eeecc496251489029bddjorton                context->ba = apr_bucket_alloc_create(pchild);
f0528865c2e6c22ecc51eeecc496251489029bddjorton                apr_atomic_inc(&num_completion_contexts);
f0528865c2e6c22ecc51eeecc496251489029bddjorton                break;
f0528865c2e6c22ecc51eeecc496251489029bddjorton            }
f0528865c2e6c22ecc51eeecc496251489029bddjorton        } else {
f0528865c2e6c22ecc51eeecc496251489029bddjorton            /* Got a context from the queue */
f0528865c2e6c22ecc51eeecc496251489029bddjorton            break;
f0528865c2e6c22ecc51eeecc496251489029bddjorton        }
f0528865c2e6c22ecc51eeecc496251489029bddjorton    }
f0528865c2e6c22ecc51eeecc496251489029bddjorton
893328ef6ff86d0ca27774778d84410353789fb0fielding    return context;
}

AP_DECLARE(apr_status_t) mpm_post_completion_context(PCOMP_CONTEXT context,
                                                     io_state_e state)
{
    LPOVERLAPPED pOverlapped;
    if (context)
        pOverlapped = &context->Overlapped;
    else
        pOverlapped = NULL;

    PostQueuedCompletionStatus(ThreadDispatchIOCP, 0, state, pOverlapped);
    return APR_SUCCESS;
}


/*
 * find_ready_listener()
 * Only used by Win9* and should go away when the win9*_accept() function is
 * reimplemented using apr_poll().
 */
static ap_listen_rec *head_listener;

static APR_INLINE ap_listen_rec *find_ready_listener(fd_set * main_fds)
{
    ap_listen_rec *lr;
    SOCKET nsd;

    for (lr = head_listener; lr ; lr = lr->next) {
        apr_os_sock_get(&nsd, lr->sd);
    if (FD_ISSET(nsd, main_fds)) {
        head_listener = lr->next;
            if (head_listener == NULL)
                head_listener = ap_listeners;

        return (lr);
    }
    }
    return NULL;
}


/* Windows 9x specific code...
 * Accept processing for on Windows 95/98 uses a producer/consumer queue
 * model. A single thread accepts connections and queues the accepted socket
 * to the accept queue for consumption by a pool of worker threads.
 *
 * win9x_accept()
 *    The accept threads runs this function, which accepts connections off
 *    the network and calls add_job() to queue jobs to the accept_queue.
 * add_job()/remove_job()
 *    Add or remove an accepted socket from the list of sockets
 *    connected to clients. allowed_globals.jobmutex protects
 *    against multiple concurrent access to the linked list of jobs.
 * win9x_get_connection()
 *    Calls remove_job() to pull a job from the accept queue. All the worker
 *    threads block on remove_job.
 */

typedef struct joblist_s {
    struct joblist_s *next;
    int sock;
} joblist;

typedef struct globals_s {
    HANDLE jobsemaphore;
    joblist *jobhead;
    joblist *jobtail;
    apr_thread_mutex_t *jobmutex;
    int jobcount;
} globals;

globals allowed_globals = {NULL, NULL, NULL, NULL, 0};

#define MAX_SELECT_ERRORS 100


static void add_job(int sock)
{
    joblist *new_job;

    new_job = (joblist *) malloc(sizeof(joblist));
    if (new_job == NULL) {
    ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
                     "Ouch!  Out of memory in add_job()!");
        return;
    }
    new_job->next = NULL;
    new_job->sock = sock;

    apr_thread_mutex_lock(allowed_globals.jobmutex);

    if (allowed_globals.jobtail != NULL)
    allowed_globals.jobtail->next = new_job;
    allowed_globals.jobtail = new_job;
    if (!allowed_globals.jobhead)
    allowed_globals.jobhead = new_job;
    allowed_globals.jobcount++;
    ReleaseSemaphore(allowed_globals.jobsemaphore, 1, NULL);

    apr_thread_mutex_unlock(allowed_globals.jobmutex);
}


static int remove_job(void)
{
    joblist *job;
    int sock;

    WaitForSingleObject(allowed_globals.jobsemaphore, INFINITE);
    apr_thread_mutex_lock(allowed_globals.jobmutex);

    if (shutdown_in_progress && !allowed_globals.jobhead) {
        apr_thread_mutex_unlock(allowed_globals.jobmutex);
    return (INVALID_SOCKET);
    }
    job = allowed_globals.jobhead;
    ap_assert(job);
    allowed_globals.jobhead = job->next;
    if (allowed_globals.jobhead == NULL)
    allowed_globals.jobtail = NULL;
    apr_thread_mutex_unlock(allowed_globals.jobmutex);
    sock = job->sock;
    free(job);

    return (sock);
}


static void win9x_accept(void * dummy)
{
    struct timeval tv;
    fd_set main_fds;
    int wait_time = 1;
    int csd;
    SOCKET nsd = INVALID_SOCKET;
    struct sockaddr_in sa_client;
    int count_select_errors = 0;
    int rc;
    int clen;
    ap_listen_rec *lr;
    struct fd_set listenfds;
    SOCKET listenmaxfd = INVALID_SOCKET;

    /* Setup the listeners
     * ToDo: Use apr_poll()
     */
    FD_ZERO(&listenfds);
    for (lr = ap_listeners; lr; lr = lr->next) {
        if (lr->sd != NULL) {
            apr_os_sock_get(&nsd, lr->sd);
            FD_SET(nsd, &listenfds);
            if (listenmaxfd == INVALID_SOCKET || nsd > listenmaxfd) {
                listenmaxfd = nsd;
            }
        }
    }

    head_listener = ap_listeners;

    while (!shutdown_in_progress) {
    tv.tv_sec = wait_time;
    tv.tv_usec = 0;
    memcpy(&main_fds, &listenfds, sizeof(fd_set));

    rc = select(listenmaxfd + 1, &main_fds, NULL, NULL, &tv);

        if (rc == 0 || (rc == SOCKET_ERROR && APR_STATUS_IS_EINTR(apr_get_netos_error()))) {
            count_select_errors = 0;    /* reset count of errors */
            continue;
        }
        else if (rc == SOCKET_ERROR) {
            /* A "real" error occurred, log it and increment the count of
             * select errors. This count is used to ensure we don't go into
             * a busy loop of continuous errors.
             */
            ap_log_error(APLOG_MARK, APLOG_INFO, apr_get_netos_error(), ap_server_conf,
                         "select failed with error %d", apr_get_netos_error());
            count_select_errors++;
            if (count_select_errors > MAX_SELECT_ERRORS) {
                shutdown_in_progress = 1;
                ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(), ap_server_conf,
                             "Too many errors in select loop. Child process exiting.");
                break;
            }
    } else {
        ap_listen_rec *lr;

        lr = find_ready_listener(&main_fds);
        if (lr != NULL) {
                /* fetch the native socket descriptor */
                apr_os_sock_get(&nsd, lr->sd);
        }
    }

    do {
            clen = sizeof(sa_client);
            csd = accept(nsd, (struct sockaddr *) &sa_client, &clen);
        } while (csd < 0 && APR_STATUS_IS_EINTR(apr_get_netos_error()));

    if (csd < 0) {
            if (APR_STATUS_IS_ECONNABORTED(apr_get_netos_error())) {
        ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(), ap_server_conf,
                "accept: (client socket)");
            }
    }
    else {
        add_job(csd);
    }
    }
    SetEvent(exit_event);
}


static PCOMP_CONTEXT win9x_get_connection(PCOMP_CONTEXT context)
{
    apr_os_sock_info_t sockinfo;
    int len;

    if (context == NULL) {
        /* allocate the completion context and the transaction pool */
        context = apr_pcalloc(pchild, sizeof(COMP_CONTEXT));
        apr_pool_create(&context->ptrans, pchild);
        apr_pool_tag(context->ptrans, "ptrans");
        context->ba = apr_bucket_alloc_create(pchild);
    }

    while (1) {
        apr_pool_clear(context->ptrans);
        context->accept_socket = remove_job();
        if (context->accept_socket == INVALID_SOCKET) {
            return NULL;
        }
    len = sizeof(struct sockaddr);
        context->sa_server = apr_palloc(context->ptrans, len);
        if (getsockname(context->accept_socket,
                        context->sa_server, &len)== SOCKET_ERROR) {
            ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), ap_server_conf,
                         "getsockname failed");
            continue;
        }
        len = sizeof(struct sockaddr);
        context->sa_client = apr_palloc(context->ptrans, len);
        if ((getpeername(context->accept_socket,
                         context->sa_client, &len)) == SOCKET_ERROR) {
            ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), ap_server_conf,
                         "getpeername failed");
            memset(&context->sa_client, '\0', sizeof(context->sa_client));
        }
        sockinfo.os_sock = &context->accept_socket;
        sockinfo.local   = context->sa_server;
        sockinfo.remote  = context->sa_client;
        sockinfo.family  = APR_INET;
        sockinfo.type    = SOCK_STREAM;
        apr_os_sock_make(&context->sock, &sockinfo, context->ptrans);

        return context;
    }
}


/* Windows NT/2000 specific code...
 * Accept processing for on Windows NT uses a producer/consumer queue
 * model. An accept thread accepts connections off the network then issues
 * PostQueuedCompletionStatus() to awake a thread blocked on the ThreadDispatch
 * IOCompletionPort.
 *
 * winnt_accept()
 *    One or more accept threads run in this function, each of which accepts
 *    connections off the network and calls PostQueuedCompletionStatus() to
 *    queue an io completion packet to the ThreadDispatch IOCompletionPort.
 * winnt_get_connection()
 *    Worker threads block on the ThreadDispatch IOCompletionPort awaiting
 *    connections to service.
 */
static void winnt_accept(void *lr_)
{
    ap_listen_rec *lr = (ap_listen_rec *)lr_;
    apr_os_sock_info_t sockinfo;
    PCOMP_CONTEXT context = NULL;
    DWORD BytesRead;
    SOCKET nlsd;
    int rv, err_count = 0;

    apr_os_sock_get(&nlsd, lr->sd);

    while (!shutdown_in_progress) {
        if (!context) {
            context = mpm_get_completion_context();
            if (!context) {
                /* Temporary resource constraint? */
                Sleep(0);
                continue;
            }
        }

        /* Create and initialize the accept socket */
        if (context->accept_socket == INVALID_SOCKET) {
            context->accept_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
            if (context->accept_socket == INVALID_SOCKET) {
                /* Another temporary condition? */
                ap_log_error(APLOG_MARK,APLOG_WARNING, apr_get_netos_error(), ap_server_conf,
                             "winnt_accept: Failed to allocate an accept socket. "
                             "Temporary resource constraint? Try again.");
                Sleep(100);
                continue;
            }
        }

        /* AcceptEx on the completion context. The completion context will be
         * signaled when a connection is accepted.
         */
        if (!AcceptEx(nlsd, context->accept_socket,
                      context->buff,
                      0,
                      PADDED_ADDR_SIZE,
                      PADDED_ADDR_SIZE,
                      &BytesRead,
                      &context->Overlapped)) {
            rv = apr_get_netos_error();
            if ((rv == APR_FROM_OS_ERROR(WSAEINVAL)) ||
                (rv == APR_FROM_OS_ERROR(WSAENOTSOCK))) {
                /* Hack alert, we can get here because:
                 * 1) Occasionally, TransmitFile will not recycle the accept socket
                 *    (usually when the client disconnects early).
                 * 2) There is VPN or Firewall software installed with buggy AcceptEx implementation
                 * 3) The webserver is using a dynamic address and it has changed
                 */
                Sleep(0);
                if (++err_count > 1000) {
                    apr_int32_t disconnected;

                    /* abitrary socket call to test if the Listening socket is still valid */
                    apr_status_t listen_rv =  apr_socket_opt_get(lr->sd, APR_SO_DISCONNECTED, &disconnected);

                    if (listen_rv == APR_SUCCESS) {
                        ap_log_error(APLOG_MARK,APLOG_ERR, listen_rv, ap_server_conf,
                                     "AcceptEx error: If this occurs constantly and NO requests are being served "
                                     "try using the WindowsSocketsWorkaround directive set to 'on'.");
                        err_count = 0;
                    }
                    else {
                        ap_log_error(APLOG_MARK,APLOG_ERR, listen_rv, ap_server_conf,
                                     "The Listening socket is no longer valid. Dynamic address changed?");
                        break;
                    }
                }

                closesocket(context->accept_socket);
                context->accept_socket = INVALID_SOCKET;
                ap_log_error(APLOG_MARK, APLOG_DEBUG, rv, ap_server_conf,
                       "winnt_accept: AcceptEx failed, either early client disconnect, "
                       "dynamic address renewal, or incompatible VPN or Firewall software.");

                continue;
            }
            else if ((rv != APR_FROM_OS_ERROR(ERROR_IO_PENDING)) &&
                     (rv != APR_FROM_OS_ERROR(WSA_IO_PENDING))) {
                ap_log_error(APLOG_MARK,APLOG_ERR, rv, ap_server_conf,
                             "winnt_accept: AcceptEx failed. Attempting to recover.");
                closesocket(context->accept_socket);
                context->accept_socket = INVALID_SOCKET;
                Sleep(100);
                continue;
            }
            err_count = 0;

            /* Wait for pending i/o.
             * Wake up once per second to check for shutdown .
             * XXX: We should be waiting on exit_event instead of polling
             */
            while (1) {
                rv = WaitForSingleObject(context->Overlapped.hEvent, 1000);
                if (rv == WAIT_OBJECT_0) {
                    if (context->accept_socket == INVALID_SOCKET) {
                        /* socket already closed */
                        break;
                    }
                    if (!GetOverlappedResult((HANDLE)context->accept_socket,
                                             &context->Overlapped,
                                             &BytesRead, FALSE)) {
                        ap_log_error(APLOG_MARK, APLOG_WARNING,
                                     apr_get_os_error(), ap_server_conf,
                             "winnt_accept: Asynchronous AcceptEx failed.");
                        closesocket(context->accept_socket);
                        context->accept_socket = INVALID_SOCKET;
                    }
                    break;
                }
                /* WAIT_TIMEOUT */
                if (shutdown_in_progress) {
                    closesocket(context->accept_socket);
                    context->accept_socket = INVALID_SOCKET;
                    break;
                }
            }
            if (context->accept_socket == INVALID_SOCKET) {
                continue;
            }
        }

        /* Inherit the listen socket settings. Required for
         * shutdown() to work
         */
        if (setsockopt(context->accept_socket, SOL_SOCKET,
                       SO_UPDATE_ACCEPT_CONTEXT, (char *)&nlsd,
                       sizeof(nlsd))) {
            ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), ap_server_conf,
                         "setsockopt(SO_UPDATE_ACCEPT_CONTEXT) failed.");
            /* Not a failure condition. Keep running. */
        }

        /* Get the local & remote address */
        GetAcceptExSockaddrs(context->buff,
                             0,
                             PADDED_ADDR_SIZE,
                             PADDED_ADDR_SIZE,
                             &context->sa_server,
                             &context->sa_server_len,
                             &context->sa_client,
                             &context->sa_client_len);

        sockinfo.os_sock = &context->accept_socket;
        sockinfo.local   = context->sa_server;
        sockinfo.remote  = context->sa_client;
        sockinfo.family  = APR_INET;
        sockinfo.type    = SOCK_STREAM;
        apr_os_sock_make(&context->sock, &sockinfo, context->ptrans);

        /* When a connection is received, send an io completion notification to
         * the ThreadDispatchIOCP. This function could be replaced by
         * mpm_post_completion_context(), but why do an extra function call...
         */
        PostQueuedCompletionStatus(ThreadDispatchIOCP, 0, IOCP_CONNECTION_ACCEPTED,
                                   &context->Overlapped);
        context = NULL;
    }
    if (!shutdown_in_progress) {
        /* Yow, hit an irrecoverable error! Tell the child to die. */
        SetEvent(exit_event);
    }
    ap_log_error(APLOG_MARK, APLOG_INFO, APR_SUCCESS, ap_server_conf,
                 "Child %d: Accept thread exiting.", my_pid);
}


static PCOMP_CONTEXT winnt_get_connection(PCOMP_CONTEXT context)
{
    int rc;
    DWORD BytesRead;
    DWORD CompKey;
    LPOVERLAPPED pol;

    mpm_recycle_completion_context(context);

    apr_atomic_inc(&g_blocked_threads);
    while (1) {
        if (workers_may_exit) {
            apr_atomic_dec(&g_blocked_threads);
            return NULL;
        }
        rc = GetQueuedCompletionStatus(ThreadDispatchIOCP, &BytesRead, &CompKey,
                                       &pol, INFINITE);
        if (!rc) {
            rc = apr_get_os_error();
            ap_log_error(APLOG_MARK,APLOG_DEBUG, rc, ap_server_conf,
                             "Child %d: GetQueuedComplationStatus returned %d", my_pid, rc);
            continue;
        }

        switch (CompKey) {
        case IOCP_CONNECTION_ACCEPTED:
            context = CONTAINING_RECORD(pol, COMP_CONTEXT, Overlapped);
            break;
        case IOCP_SHUTDOWN:
            apr_atomic_dec(&g_blocked_threads);
            return NULL;
        default:
            apr_atomic_dec(&g_blocked_threads);
            return NULL;
        }
        break;
    }
    apr_atomic_dec(&g_blocked_threads);

    return context;
}


/*
 * worker_main()
 * Main entry point for the worker threads. Worker threads block in
 * win*_get_connection() awaiting a connection to service.
 */
static void worker_main(long thread_num)
{
    static int requests_this_child = 0;
    PCOMP_CONTEXT context = NULL;
    ap_sb_handle_t *sbh;

    ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf,
                 "Child %d: Worker thread %d starting.", my_pid, thread_num);
    while (1) {
        conn_rec *c;
        apr_int32_t disconnected;

        ap_update_child_status_from_indexes(0, thread_num, SERVER_READY, NULL);

        /* Grab a connection off the network */
        if (osver.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS || windows_sockets_workaround == 1) {
            context = win9x_get_connection(context);
        }
        else {
            context = winnt_get_connection(context);
        }
        if (!context) {
            /* Time for the thread to exit */
            break;
        }

        /* Have we hit MaxRequestPerChild connections? */
        if (ap_max_requests_per_child) {
            requests_this_child++;
            if (requests_this_child > ap_max_requests_per_child) {
                SetEvent(max_requests_per_child_event);
            }
        }

        ap_create_sb_handle(&sbh, context->ptrans, 0, thread_num);
        c = ap_run_create_connection(context->ptrans, ap_server_conf,
                                     context->sock, thread_num, sbh,
                                     context->ba);

        if (c) {
            ap_process_connection(c, context->sock);
            apr_socket_opt_get(context->sock, APR_SO_DISCONNECTED,
                               &disconnected);
            if (!disconnected) {
                context->accept_socket = INVALID_SOCKET;
                ap_lingering_close(c);
            }
        }
        else {
            /* ap_run_create_connection closes the socket on failure */
            context->accept_socket = INVALID_SOCKET;
        }
    }

    ap_update_child_status_from_indexes(0, thread_num, SERVER_DEAD,
                                        (request_rec *) NULL);

    ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf,
                 "Child %d: Worker thread %d exiting.", my_pid, thread_num);
}


static void cleanup_thread(HANDLE *handles, int *thread_cnt, int thread_to_clean)
{
    int i;

    CloseHandle(handles[thread_to_clean]);
    for (i = thread_to_clean; i < ((*thread_cnt) - 1); i++)
    handles[i] = handles[i + 1];
    (*thread_cnt)--;
}


/*
 * child_main()
 * Entry point for the main control thread for the child process.
 * This thread creates the accept thread, worker threads and
 * monitors the child process for maintenance and shutdown
 * events.
 */
static void create_listener_thread()
{
    int tid;
    if (osver.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS || windows_sockets_workaround == 1) {
        _beginthreadex(NULL, 0, (LPTHREAD_START_ROUTINE) win9x_accept,
                       NULL, 0, &tid);
    } else {
        /* Start an accept thread per listener
         * XXX: Why would we have a NULL sd in our listeners?
         */
        ap_listen_rec *lr;
        for (lr = ap_listeners; lr; lr = lr->next) {
            if (lr->sd != NULL) {
                _beginthreadex(NULL, 1000, (LPTHREAD_START_ROUTINE) winnt_accept,
                               (void *) lr, 0, &tid);
            }
        }
    }
}


void child_main(apr_pool_t *pconf)
{
    apr_status_t status;
    apr_hash_t *ht;
    ap_listen_rec *lr;
    HANDLE child_events[2];
    int threads_created = 0;
    int listener_started = 0;
    int tid;
    HANDLE *child_handles;
    int rv;
    time_t end_time;
    int i;
    int cld;

    apr_pool_create(&pchild, pconf);
    apr_pool_tag(pchild, "pchild");

    ap_run_child_init(pchild, ap_server_conf);
    ht = apr_hash_make(pchild);

    /* Initialize the child_events */
    max_requests_per_child_event = CreateEvent(NULL, TRUE, FALSE, NULL);
    if (!max_requests_per_child_event) {
        ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(), ap_server_conf,
                     "Child %d: Failed to create a max_requests event.", my_pid);
        exit(APEXIT_CHILDINIT);
    }
    child_events[0] = exit_event;
    child_events[1] = max_requests_per_child_event;

    allowed_globals.jobsemaphore = CreateSemaphore(NULL, 0, 1000000, NULL);
    apr_thread_mutex_create(&allowed_globals.jobmutex,
                            APR_THREAD_MUTEX_DEFAULT, pchild);

    /*
     * Wait until we have permission to start accepting connections.
     * start_mutex is used to ensure that only one child ever
     * goes into the listen/accept loop at once.
     */
    status = apr_proc_mutex_lock(start_mutex);
    if (status != APR_SUCCESS) {
        ap_log_error(APLOG_MARK,APLOG_ERR, status, ap_server_conf,
                     "Child %d: Failed to acquire the start_mutex. Process will exit.", my_pid);
        exit(APEXIT_CHILDINIT);
    }
    ap_log_error(APLOG_MARK,APLOG_NOTICE, APR_SUCCESS, ap_server_conf,
                 "Child %d: Acquired the start mutex.", my_pid);

    /*
     * Create the worker thread dispatch IOCompletionPort
     * on Windows NT/2000
     */
    if (osver.dwPlatformId != VER_PLATFORM_WIN32_WINDOWS && windows_sockets_workaround != 1) {
        /* Create the worker thread dispatch IOCP */
        ThreadDispatchIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE,
                                                    NULL,
                                                    0,
                                                    0); /* CONCURRENT ACTIVE THREADS */
        apr_thread_mutex_create(&qlock, APR_THREAD_MUTEX_DEFAULT, pchild);
        qwait_event = CreateEvent(NULL, TRUE, FALSE, NULL);
        if (!qwait_event) {
            ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(), ap_server_conf,
                         "Child %d: Failed to create a qwait event.", my_pid);
            exit(APEXIT_CHILDINIT);
        }
    }

    /*
     * Create the pool of worker threads
     */
    ap_log_error(APLOG_MARK,APLOG_NOTICE, APR_SUCCESS, ap_server_conf,
                 "Child %d: Starting %d worker threads.", my_pid, ap_threads_per_child);
    child_handles = (HANDLE) apr_pcalloc(pchild, ap_threads_per_child * sizeof(int));
    while (1) {
        for (i = 0; i < ap_threads_per_child; i++) {
            int *score_idx;
            int status = ap_scoreboard_image->servers[0][i].status;
            if (status != SERVER_GRACEFUL && status != SERVER_DEAD) {
                continue;
            }
            ap_update_child_status_from_indexes(0, i, SERVER_STARTING, NULL);
            child_handles[i] = (HANDLE) _beginthreadex(NULL, 0, (LPTHREAD_START_ROUTINE) worker_main,
                                                       (void *) i, 0, &tid);
            if (child_handles[i] == 0) {
                ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(), ap_server_conf,
                             "Child %d: _beginthreadex failed. Unable to create all worker threads. "
                             "Created %d of the %d threads requested with the ThreadsPerChild configuration directive.",
                             threads_created, ap_threads_per_child);
                ap_signal_parent(SIGNAL_PARENT_SHUTDOWN);
                goto shutdown;
            }
            threads_created++;
            /* Save the score board index in ht keyed to the thread handle. We need this
             * when cleaning up threads down below...
             */
            score_idx = apr_pcalloc(pchild, sizeof(int));
            *score_idx = i;
            apr_hash_set(ht, &child_handles[i], sizeof(HANDLE), score_idx);
        }
        /* Start the listener only when workers are available */
        if (!listener_started && threads_created) {
            create_listener_thread();
            listener_started = 1;
        }
        if (threads_created == ap_threads_per_child) {
            break;
        }
        /* Check to see if the child has been told to exit */
        if (WaitForSingleObject(exit_event, 0) != WAIT_TIMEOUT) {
            break;
        }
        /* wait for previous generation to clean up an entry in the scoreboard */
        apr_sleep(1 * APR_USEC_PER_SEC);
    }

    /* Wait for one of three events:
     * exit_event:
     *    The exit_event is signaled by the parent process to notify
     *    the child that it is time to exit.
     *
     * max_requests_per_child_event:
     *    This event is signaled by the worker threads to indicate that
     *    the process has handled MaxRequestsPerChild connections.
     *
     * TIMEOUT:
     *    To do periodic maintenance on the server (check for thread exits,
     *    number of completion contexts, etc.)
     *
     * XXX: thread exits *aren't* being checked.
     *
     * XXX: other_child - we need the process handles to the other children
     *      in order to map them to apr_proc_other_child_read (which is not
     *      named well, it's more like a_p_o_c_died.)
     *
     * XXX: however - if we get a_p_o_c handle inheritance working, and
     *      the parent process creates other children and passes the pipes
     *      to our worker processes, then we have no business doing such
     *      things in the child_main loop, but should happen in master_main.
     */
    while (1) {
#if !APR_HAS_OTHER_CHILD
        rv = WaitForMultipleObjects(2, (HANDLE *) child_events, FALSE, INFINITE);
        cld = rv - WAIT_OBJECT_0;
#else
        rv = WaitForMultipleObjects(2, (HANDLE *) child_events, FALSE, 1000);
        cld = rv - WAIT_OBJECT_0;
        if (rv == WAIT_TIMEOUT) {
            apr_proc_other_child_check();
        }
        else
#endif
            if (rv == WAIT_FAILED) {
            /* Something serious is wrong */
            ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(), ap_server_conf,
                         "Child %d: WAIT_FAILED -- shutting down server");
            break;
        }
        else if (cld == 0) {
            /* Exit event was signaled */
            ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf,
                         "Child %d: Exit event signaled. Child process is ending.", my_pid);
            break;
        }
        else {
            /* MaxRequestsPerChild event set by the worker threads.
             * Signal the parent to restart
             */
            ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf,
                         "Child %d: Process exiting because it reached "
                         "MaxRequestsPerChild. Signaling the parent to "
                         "restart a new child process.", my_pid);
            ap_signal_parent(SIGNAL_PARENT_RESTART);
            break;
        }
    }

    /*
     * Time to shutdown the child process
     */

 shutdown:
    /* Setting is_graceful will cause threads handling keep-alive connections
     * to close the connection after handling the current request.
     */
    is_graceful = 1;

    /* Close the listening sockets. Note, we must close the listeners
     * before closing any accept sockets pending in AcceptEx to prevent
     * memory leaks in the kernel.
     */
    for (lr = ap_listeners; lr ; lr = lr->next) {
        apr_socket_close(lr->sd);
    }

    /* Shutdown listener threads and pending AcceptEx socksts
     * but allow the worker threads to continue consuming from
     * the queue of accepted connections.
     */
    shutdown_in_progress = 1;

    Sleep(1000);

    /* Tell the worker threads to exit */
    workers_may_exit = 1;

    /* Release the start_mutex to let the new process (in the restart
     * scenario) a chance to begin accepting and servicing requests
     */
    rv = apr_proc_mutex_unlock(start_mutex);
    if (rv == APR_SUCCESS) {
        ap_log_error(APLOG_MARK,APLOG_NOTICE, rv, ap_server_conf,
                     "Child %d: Released the start mutex", my_pid);
    }
    else {
        ap_log_error(APLOG_MARK,APLOG_ERR, rv, ap_server_conf,
                     "Child %d: Failure releasing the start mutex", my_pid);
    }

    /* Shutdown the worker threads */
    if (osver.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS || windows_sockets_workaround == 1) {
        for (i = 0; i < threads_created; i++) {
            add_job(INVALID_SOCKET);
        }
    }
    else { /* Windows NT/2000 */
        /* Post worker threads blocked on the ThreadDispatch IOCompletion port */
        while (g_blocked_threads > 0) {
            ap_log_error(APLOG_MARK,APLOG_INFO, APR_SUCCESS, ap_server_conf,
                         "Child %d: %d threads blocked on the completion port", my_pid, g_blocked_threads);
            for (i=g_blocked_threads; i > 0; i--) {
                PostQueuedCompletionStatus(ThreadDispatchIOCP, 0, IOCP_SHUTDOWN, NULL);
            }
            Sleep(1000);
        }
        /* Empty the accept queue of completion contexts */
        apr_thread_mutex_lock(qlock);
        while (qhead) {
            CloseHandle(qhead->Overlapped.hEvent);
            closesocket(qhead->accept_socket);
            qhead = qhead->next;
        }
        apr_thread_mutex_unlock(qlock);
    }

    /* Give busy worker threads a chance to service their connections */
    ap_log_error(APLOG_MARK,APLOG_NOTICE, APR_SUCCESS, ap_server_conf,
                 "Child %d: Waiting for %d worker threads to exit.", my_pid, threads_created);
    end_time = time(NULL) + 180;
    while (threads_created) {
        rv = wait_for_many_objects(threads_created, child_handles, end_time - time(NULL));
        if (rv != WAIT_TIMEOUT) {
            rv = rv - WAIT_OBJECT_0;
            ap_assert((rv >= 0) && (rv < threads_created));
            cleanup_thread(child_handles, &threads_created, rv);
            continue;
        }
        break;
    }

    /* Kill remaining threads off the hard way */
    if (threads_created) {
        ap_log_error(APLOG_MARK,APLOG_NOTICE, APR_SUCCESS, ap_server_conf,
                     "Child %d: Terminating %d threads that failed to exit.", my_pid);
    }
    for (i = 0; i < threads_created; i++) {
        int *score_idx;
        TerminateThread(child_handles[i], 1);
        CloseHandle(child_handles[i]);
        /* Reset the scoreboard entry for the thread we just whacked */
        score_idx = apr_hash_get(ht, &child_handles[i], sizeof(HANDLE));
        ap_update_child_status_from_indexes(0, *score_idx, SERVER_DEAD, NULL);
    }
    ap_log_error(APLOG_MARK,APLOG_NOTICE, APR_SUCCESS, ap_server_conf,
                 "Child %d: All worker threads have exited.", my_pid);

    CloseHandle(allowed_globals.jobsemaphore);
    apr_thread_mutex_destroy(allowed_globals.jobmutex);
    if (osver.dwPlatformId != VER_PLATFORM_WIN32_WINDOWS && windows_sockets_workaround != 1) {
        apr_thread_mutex_destroy(qlock);
        CloseHandle(qwait_event);
    }

    apr_pool_destroy(pchild);
    CloseHandle(exit_event);
}

#endif /* def WIN32 */