src/lib/ioloop-epoll.c

	ioloop-epoll.c revision 193f5296d2a6b847970c222d8a261b89aae46331
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen/*
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen * Linux epoll() based ioloop handler.
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen *
345648b341f228bd7f0b89f8aa3ecb9c470d817eTimo Sirainen * Copyright (c) 2004 Andrey Panin <pazke@donpac.ru>
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen *
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen * This software is released under the MIT license.
5a2cb3d097a2d9a9e930af997e7bf3400a8d840dTimo Sirainen */
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "lib.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "array.h"
97c339398f1aba6f315b55a9b6ee6b020e33bea4Timo Sirainen#include "fd-close-on-exec.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "ioloop-internal.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "ioloop-iolist.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#ifdef IOLOOP_EPOLL
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include <sys/epoll.h>
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include <unistd.h>
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainenstruct ioloop_handler_context {
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    int epfd;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    unsigned int deleted_count;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    ARRAY_DEFINE(fd_index, struct io_list *);
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    ARRAY_DEFINE(events, struct epoll_event);
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen};
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainenvoid io_loop_handler_init(struct ioloop *ioloop)
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen{
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    struct ioloop_handler_context *ctx;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    ioloop->handler_context = ctx = i_new(struct ioloop_handler_context, 1);
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    i_array_init(&ctx->events, IOLOOP_INITIAL_FD_COUNT);
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    i_array_init(&ctx->fd_index, IOLOOP_INITIAL_FD_COUNT);
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    ctx->epfd = epoll_create(IOLOOP_INITIAL_FD_COUNT);
73bfdbe28c2ce6d143eadf0bab8ccfbe4cab0faeTimo Sirainen    if (ctx->epfd < 0)
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        i_fatal("epoll_create(): %m");
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen    fd_close_on_exec(ctx->epfd, TRUE);
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen}
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen
c57776c06ec99ba9b0dafdbf9475ea72ea8ca134Timo Sirainenvoid io_loop_handler_deinit(struct ioloop *ioloop)
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen{
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen    struct ioloop_handler_context *ctx = ioloop->handler_context;
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen    struct io_list **list;
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen    unsigned int i, count;
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen    list = array_get_modifiable(&ctx->fd_index, &count);
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen    for (i = 0; i < count; i++)
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen        i_free(list[i]);
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen    if (close(ctx->epfd) < 0)
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen        i_error("close(epoll) failed: %m");
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen    array_free(&ioloop->handler_context->fd_index);
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen    array_free(&ioloop->handler_context->events);
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen    i_free(ioloop->handler_context);
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen}
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen#define IO_EPOLL_ERROR (EPOLLERR | EPOLLHUP)
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen#define IO_EPOLL_INPUT (EPOLLIN | EPOLLPRI | IO_EPOLL_ERROR)
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen#define IO_EPOLL_OUTPUT (EPOLLOUT | IO_EPOLL_ERROR)
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainen
e70d5895795732b8247ab9abb045b438e954bc46Timo Sirainenstatic int epoll_event_mask(struct io_list *list)
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen{
33c6d5807b449463e9b81db5ec99fe027cc1b984Timo Sirainen    int events = 0, i;
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen    struct io_file *io;
94a78eb438622fa53abef1e1726714dacad4b61cTimo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    for (i = 0; i < IOLOOP_IOLIST_IOS_PER_FD; i++) {
c7480644202e5451fbed448508ea29a25cffc99cTimo Sirainen        io = list->ios[i];
a4ac325c2802693c6b761e5a8fda961e5d7490eaTimo Sirainen
b7c2065b3f10f9ae27787a9db5aaefbfc70d4502Timo Sirainen        if (io == NULL)
0469ed17dafcc56589ce00960a23f4f06817dfb5Timo Sirainen            continue;
0469ed17dafcc56589ce00960a23f4f06817dfb5Timo Sirainen
0469ed17dafcc56589ce00960a23f4f06817dfb5Timo Sirainen        if (io->io.condition & IO_READ)
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen            events |= IO_EPOLL_INPUT;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen        if (io->io.condition & IO_WRITE)
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen            events |= IO_EPOLL_OUTPUT;
97c339398f1aba6f315b55a9b6ee6b020e33bea4Timo Sirainen        if (io->io.condition & IO_ERROR)
97c339398f1aba6f315b55a9b6ee6b020e33bea4Timo Sirainen            events |= IO_EPOLL_ERROR;
97c339398f1aba6f315b55a9b6ee6b020e33bea4Timo Sirainen    }
97c339398f1aba6f315b55a9b6ee6b020e33bea4Timo Sirainen
97c339398f1aba6f315b55a9b6ee6b020e33bea4Timo Sirainen    return events;
97c339398f1aba6f315b55a9b6ee6b020e33bea4Timo Sirainen}
97c339398f1aba6f315b55a9b6ee6b020e33bea4Timo Sirainen
97c339398f1aba6f315b55a9b6ee6b020e33bea4Timo Sirainenvoid io_loop_handle_add(struct io_file *io)
0469ed17dafcc56589ce00960a23f4f06817dfb5Timo Sirainen{
0469ed17dafcc56589ce00960a23f4f06817dfb5Timo Sirainen    struct ioloop_handler_context *ctx = io->io.ioloop->handler_context;
0469ed17dafcc56589ce00960a23f4f06817dfb5Timo Sirainen    struct io_list **list;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    struct epoll_event event;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    int op;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    bool first;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    list = array_idx_modifiable(&ctx->fd_index, io->fd);
33c6d5807b449463e9b81db5ec99fe027cc1b984Timo Sirainen    if (*list == NULL)
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen        *list = i_new(struct io_list, 1);
94a78eb438622fa53abef1e1726714dacad4b61cTimo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    first = ioloop_iolist_add(*list, io);
c7480644202e5451fbed448508ea29a25cffc99cTimo Sirainen
a4ac325c2802693c6b761e5a8fda961e5d7490eaTimo Sirainen    memset(&event, 0, sizeof(event));
b7c2065b3f10f9ae27787a9db5aaefbfc70d4502Timo Sirainen    event.data.ptr = *list;
0469ed17dafcc56589ce00960a23f4f06817dfb5Timo Sirainen    event.events = epoll_event_mask(*list);
0469ed17dafcc56589ce00960a23f4f06817dfb5Timo Sirainen
0469ed17dafcc56589ce00960a23f4f06817dfb5Timo Sirainen    op = first ? EPOLL_CTL_ADD : EPOLL_CTL_MOD;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
    if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) {
        i_fatal("io_loop_handle_add: epoll_ctl(%d, %d): %m",
            op, io->fd);
    }

    if (first) {
        /* allow epoll_wait() to return the maximum number of events
           by keeping space allocated for each file descriptor */
        if (ctx->deleted_count > 0)
            ctx->deleted_count--;
        else
            (void)array_append_space(&ctx->events);
    }
}

void io_loop_handle_remove(struct io_file *io, bool closed)
{
    struct ioloop_handler_context *ctx = io->io.ioloop->handler_context;
    struct io_list **list;
    struct epoll_event event;
    int op;
    bool last;

    list = array_idx_modifiable(&ctx->fd_index, io->fd);
    last = ioloop_iolist_del(*list, io);

    if (!closed) {
        memset(&event, 0, sizeof(event));
        event.data.ptr = *list;
        event.events = epoll_event_mask(*list);

        op = last ? EPOLL_CTL_DEL : EPOLL_CTL_MOD;

        if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) {
            i_error("io_loop_handle_remove: epoll_ctl(%d, %d): %m",
                op, io->fd);
        }
    }
    if (last) {
        /* since we're not freeing memory in any case, just increase
           deleted counter so next handle_add() can just decrease it
           insteading of appending to the events array */
        ctx->deleted_count++;
    }
    i_free(io);
}

void io_loop_handler_run(struct ioloop *ioloop)
{
    struct ioloop_handler_context *ctx = ioloop->handler_context;
    struct epoll_event *events;
    const struct epoll_event *event;
    struct io_list *list;
    struct io_file *io;
    struct timeval tv;
    unsigned int events_count, t_id;
    int msecs, ret, i, j;
    bool call;

        /* get the time left for next timeout task */
    msecs = io_loop_get_wait_time(ioloop, &tv, NULL);

    events = array_get_modifiable(&ctx->events, &events_count);
    ret = epoll_wait(ctx->epfd, events, events_count, msecs);
    if (ret < 0 && errno != EINTR)
        i_fatal("epoll_wait(): %m");

    /* execute timeout handlers */
        io_loop_handle_timeouts(ioloop);

    if (!ioloop->running)
        return;

    for (i = 0; i < ret; i++) {
        /* io_loop_handle_add() may cause events array reallocation,
           so we have use array_idx() */
        event = array_idx(&ctx->events, i);
        list = event->data.ptr;

        for (j = 0; j < IOLOOP_IOLIST_IOS_PER_FD; j++) {
            io = list->ios[j];
            if (io == NULL)
                continue;

            call = FALSE;
            if ((event->events & (EPOLLHUP | EPOLLERR)) != 0)
                call = TRUE;
            else if ((io->io.condition & IO_READ) != 0)
                call = (event->events & EPOLLIN) != 0;
            else if ((io->io.condition & IO_WRITE) != 0)
                call = (event->events & EPOLLOUT) != 0;
            else if ((io->io.condition & IO_ERROR) != 0)
                call = (event->events & IO_EPOLL_ERROR) != 0;

            if (call) {
                t_id = t_push();
                io->io.callback(io->io.context);
                if (t_pop() != t_id) {
                    i_panic("Leaked a t_pop() call in "
                        "I/O handler %p",
                        (void *)io->io.callback);
                }
            }
        }
    }
}

#endif  /* IOLOOP_EPOLL */