src/lib/ioloop-epoll.c

	ioloop-epoll.c revision 80cfaba9e253545cbcd08f84939b27cdbb4a50d0
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen/* Copyright (c) 2004-2016 Dovecot authors, see the included COPYING file */
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#include "lib.h"
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#include "array.h"
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#include "fd-close-on-exec.h"
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#include "ioloop-private.h"
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#include "ioloop-iolist.h"
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#ifdef IOLOOP_EPOLL
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#include <sys/epoll.h>
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#include <unistd.h>
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainenstruct ioloop_handler_context {
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    int epfd;
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    unsigned int deleted_count;
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    ARRAY(struct io_list *) fd_index;
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    ARRAY(struct epoll_event) events;
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen};
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainenvoid io_loop_handler_init(struct ioloop *ioloop, unsigned int initial_fd_count)
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen{
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    struct ioloop_handler_context *ctx;
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    ioloop->handler_context = ctx = i_new(struct ioloop_handler_context, 1);
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
6759ba36f2bac9e83547cfdcf233382a139f1283Timo Sirainen    i_array_init(&ctx->events, initial_fd_count);
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    i_array_init(&ctx->fd_index, initial_fd_count);
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
6759ba36f2bac9e83547cfdcf233382a139f1283Timo Sirainen    ctx->epfd = epoll_create(initial_fd_count);
6759ba36f2bac9e83547cfdcf233382a139f1283Timo Sirainen    if (ctx->epfd < 0) {
6759ba36f2bac9e83547cfdcf233382a139f1283Timo Sirainen        if (errno != EMFILE)
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen            i_fatal("epoll_create(): %m");
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen        else {
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen            i_fatal("epoll_create(): %m (you may need to increase "
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen                "/proc/sys/fs/epoll/max_user_instances)");
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen        }
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    }
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    fd_close_on_exec(ctx->epfd, TRUE);
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen}
6759ba36f2bac9e83547cfdcf233382a139f1283Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainenvoid io_loop_handler_deinit(struct ioloop *ioloop)
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen{
6759ba36f2bac9e83547cfdcf233382a139f1283Timo Sirainen    struct ioloop_handler_context *ctx = ioloop->handler_context;
b60ba688ee49df2c2efba9071b717c8d2363cdc3Timo Sirainen    struct io_list **list;
6759ba36f2bac9e83547cfdcf233382a139f1283Timo Sirainen    unsigned int i, count;
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    list = array_get_modifiable(&ctx->fd_index, &count);
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    for (i = 0; i < count; i++)
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen        i_free(list[i]);
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    if (close(ctx->epfd) < 0)
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen        i_error("close(epoll) failed: %m");
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    array_free(&ioloop->handler_context->fd_index);
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    array_free(&ioloop->handler_context->events);
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen    i_free(ioloop->handler_context);
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen}
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#define IO_EPOLL_ERROR (EPOLLERR | EPOLLHUP)
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#define IO_EPOLL_INPUT (EPOLLIN | EPOLLPRI | IO_EPOLL_ERROR)
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen#define IO_EPOLL_OUTPUT (EPOLLOUT | IO_EPOLL_ERROR)
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainen
deed0441882796b3d985ab867e96b9c56d9fc0d9Timo Sirainenstatic int epoll_event_mask(struct io_list *list)
{
    int events = 0, i;
    struct io_file *io;

    for (i = 0; i < IOLOOP_IOLIST_IOS_PER_FD; i++) {
        io = list->ios[i];

        if (io == NULL)
            continue;

        if ((io->io.condition & IO_READ) != 0)
            events |= IO_EPOLL_INPUT;
        if ((io->io.condition & IO_WRITE) != 0)
            events |= IO_EPOLL_OUTPUT;
        if ((io->io.condition & IO_ERROR) != 0)
            events |= IO_EPOLL_ERROR;
    }

    return events;
}

void io_loop_handle_add(struct io_file *io)
{
    struct ioloop_handler_context *ctx = io->io.ioloop->handler_context;
    struct io_list **list;
    struct epoll_event event;
    int op;
    bool first;

    list = array_idx_modifiable(&ctx->fd_index, io->fd);
    if (*list == NULL)
        *list = i_new(struct io_list, 1);

    first = ioloop_iolist_add(*list, io);

    memset(&event, 0, sizeof(event));
    event.data.ptr = *list;
    event.events = epoll_event_mask(*list);

    op = first ? EPOLL_CTL_ADD : EPOLL_CTL_MOD;

    if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) {
        if (errno == EPERM && op == EPOLL_CTL_ADD) {
            i_panic("epoll_ctl(add, %d) failed: %m "
                "(fd doesn't support epoll%s)", io->fd,
                io->fd != STDIN_FILENO ? "" :
                " - instead of '<file', try 'cat file|'");
        }
        i_panic("epoll_ctl(%s, %d) failed: %m",
            op == EPOLL_CTL_ADD ? "add" : "mod", io->fd);
    }

    if (first) {
        /* allow epoll_wait() to return the maximum number of events
           by keeping space allocated for each file descriptor */
        if (ctx->deleted_count > 0)
            ctx->deleted_count--;
        else
            array_append_zero(&ctx->events);
    }
}

void io_loop_handle_remove(struct io_file *io, bool closed)
{
    struct ioloop_handler_context *ctx = io->io.ioloop->handler_context;
    struct io_list **list;
    struct epoll_event event;
    int op;
    bool last;

    list = array_idx_modifiable(&ctx->fd_index, io->fd);
    last = ioloop_iolist_del(*list, io);

    if (!closed) {
        memset(&event, 0, sizeof(event));
        event.data.ptr = *list;
        event.events = epoll_event_mask(*list);

        op = last ? EPOLL_CTL_DEL : EPOLL_CTL_MOD;

        if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) {
            const char *errstr = t_strdup_printf(
                "epoll_ctl(%s, %d) failed: %m",
                op == EPOLL_CTL_DEL ? "del" : "mod", io->fd);
            if (errno == EBADF)
                i_panic("%s", errstr);
            else
                i_error("%s", errstr);
        }
    }
    if (last) {
        /* since we're not freeing memory in any case, just increase
           deleted counter so next handle_add() can just decrease it
           insteading of appending to the events array */
        ctx->deleted_count++;
    }
    i_free(io);
}

void io_loop_handler_run_internal(struct ioloop *ioloop)
{
    struct ioloop_handler_context *ctx = ioloop->handler_context;
    struct epoll_event *events;
    const struct epoll_event *event;
    struct io_list *list;
    struct io_file *io;
    struct timeval tv;
    unsigned int events_count;
    int msecs, ret, i, j;
    bool call;

    i_assert(ctx != NULL);

        /* get the time left for next timeout task */
    msecs = io_loop_get_wait_time(ioloop, &tv);

    events = array_get_modifiable(&ctx->events, &events_count);
    if (ioloop->io_files != NULL && events_count > ctx->deleted_count) {
        ret = epoll_wait(ctx->epfd, events, events_count, msecs);
        if (ret < 0 && errno != EINTR)
            i_fatal("epoll_wait(): %m");
    } else {
        /* no I/Os, but we should have some timeouts.
           just wait for them. */
        if (msecs < 0)
            i_panic("BUG: No IOs or timeouts set. Not waiting for infinity.");
        usleep(msecs*1000);
        ret = 0;
    }

    /* execute timeout handlers */
        io_loop_handle_timeouts(ioloop);

    if (!ioloop->running)
        return;

    for (i = 0; i < ret; i++) {
        /* io_loop_handle_add() may cause events array reallocation,
           so we have use array_idx() */
        event = array_idx(&ctx->events, i);
        list = event->data.ptr;

        for (j = 0; j < IOLOOP_IOLIST_IOS_PER_FD; j++) {
            io = list->ios[j];
            if (io == NULL)
                continue;

            call = FALSE;
            if ((event->events & (EPOLLHUP | EPOLLERR)) != 0)
                call = TRUE;
            else if ((io->io.condition & IO_READ) != 0)
                call = (event->events & EPOLLIN) != 0;
            else if ((io->io.condition & IO_WRITE) != 0)
                call = (event->events & EPOLLOUT) != 0;
            else if ((io->io.condition & IO_ERROR) != 0)
                call = (event->events & IO_EPOLL_ERROR) != 0;

            if (call)
                io_loop_call_io(&io->io);
        }
    }
}

#endif  /* IOLOOP_EPOLL */