ioloop-epoll.c revision 20a3870db4f78717574ee94bca1512994391b2ab
89a126810703c666309310d0f3189e9834d70b5bTimo Sirainen/*
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen * Linux epoll() based ioloop handler.
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen *
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen * Copyright (c) 2004 Andrey Panin <pazke@donpac.ru>
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen *
65988f5a8abed57e9894fec77105941e046d3490Timo Sirainen * This software is released under the MIT license.
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen */
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen#include "lib.h"
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen#include "array.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "fd-close-on-exec.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "ioloop-internal.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "ioloop-iolist.h"
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#ifdef IOLOOP_EPOLL
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include <sys/epoll.h>
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include <unistd.h>
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainenstruct ioloop_handler_context {
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen int epfd;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen unsigned int deleted_count;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen ARRAY_DEFINE(fd_index, struct io_list *);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen ARRAY_DEFINE(events, struct epoll_event);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen};
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainenvoid io_loop_handler_init(struct ioloop *ioloop)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen{
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen struct ioloop_handler_context *ctx;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen ioloop->handler_context = ctx = i_new(struct ioloop_handler_context, 1);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen i_array_init(&ctx->events, IOLOOP_INITIAL_FD_COUNT);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen i_array_init(&ctx->fd_index, IOLOOP_INITIAL_FD_COUNT);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen ctx->epfd = epoll_create(IOLOOP_INITIAL_FD_COUNT);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen if (ctx->epfd < 0)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen i_fatal("epoll_create(): %m");
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen fd_close_on_exec(ctx->epfd, TRUE);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen}
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainenvoid io_loop_handler_deinit(struct ioloop *ioloop)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen{
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen struct ioloop_handler_context *ctx = ioloop->handler_context;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen struct io_list **list;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen unsigned int i, count;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen list = array_get_modifiable(&ctx->fd_index, &count);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen for (i = 0; i < count; i++)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen i_free(list[i]);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen if (close(ctx->epfd) < 0)
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen i_error("close(epoll) failed: %m");
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen array_free(&ioloop->handler_context->fd_index);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen array_free(&ioloop->handler_context->events);
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen i_free(ioloop->handler_context);
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen}
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
7c424aa51c956c628e3512055841aa2f9eef4833Timo Sirainen#define IO_EPOLL_ERROR (EPOLLERR | EPOLLHUP)
f923659c0e5298263d80622c99f4dc4132b4675bTimo Sirainen#define IO_EPOLL_INPUT (EPOLLIN | EPOLLPRI | IO_EPOLL_ERROR)
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen#define IO_EPOLL_OUTPUT (EPOLLOUT | IO_EPOLL_ERROR)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainenstatic int epoll_event_mask(struct io_list *list)
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen{
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen int events = 0, i;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen struct io_file *io;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen for (i = 0; i < IOLOOP_IOLIST_IOS_PER_FD; i++) {
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen io = list->ios[i];
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen if (io == NULL)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen continue;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen if (io->io.condition & IO_READ)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen events |= IO_EPOLL_INPUT;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen if (io->io.condition & IO_WRITE)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen events |= IO_EPOLL_OUTPUT;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen if (io->io.condition & IO_ERROR)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen events |= IO_EPOLL_ERROR;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen }
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen return events;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen}
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
fab050cbfdf3da692441d2e2fb4b2a4c6ac9e0daTimo Sirainenvoid io_loop_handle_add(struct ioloop *ioloop, struct io_file *io)
fab050cbfdf3da692441d2e2fb4b2a4c6ac9e0daTimo Sirainen{
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen struct ioloop_handler_context *ctx = ioloop->handler_context;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen struct io_list **list;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen struct epoll_event event;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen int op;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen bool first;
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen list = array_idx_modifiable(&ctx->fd_index, io->fd);
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen if (*list == NULL)
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen *list = i_new(struct io_list, 1);
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen first = ioloop_iolist_add(*list, io);
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen memset(&event, 0, sizeof(event));
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen event.data.ptr = *list;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen event.events = epoll_event_mask(*list);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen op = first ? EPOLL_CTL_ADD : EPOLL_CTL_MOD;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) {
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen i_fatal("io_loop_handle_add: epoll_ctl(%d, %d): %m",
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen op, io->fd);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen }
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen if (first) {
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen /* allow epoll_wait() to return the maximum number of events
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen by keeping space allocated for each file descriptor */
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen if (ctx->deleted_count > 0)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen ctx->deleted_count--;
d65a556a5ec078cd7f1d0060adb16fc860d66b27Timo Sirainen else
d65a556a5ec078cd7f1d0060adb16fc860d66b27Timo Sirainen (void)array_append_space(&ctx->events);
d65a556a5ec078cd7f1d0060adb16fc860d66b27Timo Sirainen }
e4ded29bff0662a590c2439ef2df8cda8a7cdd9bTimo Sirainen}
e4ded29bff0662a590c2439ef2df8cda8a7cdd9bTimo Sirainen
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainenvoid io_loop_handle_remove(struct ioloop *ioloop, struct io_file *io)
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen{
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen struct ioloop_handler_context *ctx = ioloop->handler_context;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen struct io_list **list;
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen struct epoll_event event;
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen int op;
345253fb28498b2e0a60f4a2a8644c65feee7e75Timo Sirainen bool last;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen list = array_idx_modifiable(&ctx->fd_index, io->fd);
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen last = ioloop_iolist_del(*list, io);
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen memset(&event, 0, sizeof(event));
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen event.data.ptr = *list;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen event.events = epoll_event_mask(*list);
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
a4ee24a4d5eefa80bbefc5acba16587ae36c3b5bTimo Sirainen op = last ? EPOLL_CTL_DEL : EPOLL_CTL_MOD;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) {
a93de780c3b78cfaace287026e468f3c3e34683aTimo Sirainen i_error("io_loop_handle_remove: epoll_ctl(%d, %d): %m",
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen op, io->fd);
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen }
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen if (last) {
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen /* since we're not freeing memory in any case, just increase
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen deleted counter so next handle_add() can just decrease it
345253fb28498b2e0a60f4a2a8644c65feee7e75Timo Sirainen insteading of appending to the events array */
65988f5a8abed57e9894fec77105941e046d3490Timo Sirainen ctx->deleted_count++;
65988f5a8abed57e9894fec77105941e046d3490Timo Sirainen }
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen i_free(io);
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen}
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainenvoid io_loop_handler_run(struct ioloop *ioloop)
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen{
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen struct ioloop_handler_context *ctx = ioloop->handler_context;
a4ee24a4d5eefa80bbefc5acba16587ae36c3b5bTimo Sirainen struct epoll_event *events;
a4ee24a4d5eefa80bbefc5acba16587ae36c3b5bTimo Sirainen const struct epoll_event *event;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen struct io_list *list;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen struct io_file *io;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen struct timeval tv;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen unsigned int events_count, t_id;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen int msecs, ret, i, j;
055f4599bba1874fa1148a8fa488517fa077619cTimo Sirainen bool call;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen /* get the time left for next timeout task */
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen msecs = io_loop_get_wait_time(ioloop, &tv, NULL);
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen events = array_get_modifiable(&ctx->events, &events_count);
345253fb28498b2e0a60f4a2a8644c65feee7e75Timo Sirainen ret = epoll_wait(ctx->epfd, events, events_count, msecs);
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen if (ret < 0 && errno != EINTR)
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen i_fatal("epoll_wait(): %m");
a93de780c3b78cfaace287026e468f3c3e34683aTimo Sirainen
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen /* execute timeout handlers */
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen io_loop_handle_timeouts(ioloop);
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
345253fb28498b2e0a60f4a2a8644c65feee7e75Timo Sirainen if (!ioloop->running)
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen return;
a93de780c3b78cfaace287026e468f3c3e34683aTimo Sirainen
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen for (i = 0; i < ret; i++) {
/* io_loop_handle_add() may cause events array reallocation,
so we have use array_idx() */
event = array_idx(&ctx->events, i);
list = event->data.ptr;
for (j = 0; j < IOLOOP_IOLIST_IOS_PER_FD; j++) {
io = list->ios[j];
if (io == NULL)
continue;
call = FALSE;
if ((event->events & (EPOLLHUP | EPOLLERR)) != 0)
call = TRUE;
else if ((io->io.condition & IO_READ) != 0)
call = (event->events & EPOLLIN) != 0;
else if ((io->io.condition & IO_WRITE) != 0)
call = (event->events & EPOLLOUT) != 0;
else if ((io->io.condition & IO_ERROR) != 0)
call = (event->events & IO_EPOLL_ERROR) != 0;
if (call) {
t_id = t_push();
io->io.callback(io->io.context);
if (t_pop() != t_id) {
i_panic("Leaked a t_pop() call in "
"I/O handler %p",
(void *)io->io.callback);
}
}
}
}
}
#endif /* IOLOOP_EPOLL */