ioloop-epoll.c revision 637f9883a385abb03fd1211e79cc68df696cc387
5a580c3a38ced62d4bcc95b8ac7c4f2935b5d294Timo Sirainen/*
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch * Linux epoll() based ioloop handler.
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch *
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch * Copyright (c) 2004 Andrey Panin <pazke@donpac.ru>
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch *
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch * This software is released under the MIT license.
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch */
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch#include "lib.h"
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch#include "array.h"
6d573191bea1a64d6046be070487a5705a2d0204Stephan Bosch#include "fd-close-on-exec.h"
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch#include "ioloop-private.h"
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen#include "ioloop-iolist.h"
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch#ifdef IOLOOP_EPOLL
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen#include <sys/epoll.h>
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen#include <unistd.h>
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainenstruct ioloop_handler_context {
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen int epfd;
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen unsigned int deleted_count;
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen ARRAY_DEFINE(fd_index, struct io_list *);
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen ARRAY_DEFINE(events, struct epoll_event);
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen};
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen
7384b4e78eaab44693c985192276e31322155e32Stephan Boschvoid io_loop_handler_init(struct ioloop *ioloop, unsigned int initial_fd_count)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch{
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct ioloop_handler_context *ctx;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch ioloop->handler_context = ctx = i_new(struct ioloop_handler_context, 1);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_array_init(&ctx->events, initial_fd_count);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_array_init(&ctx->fd_index, initial_fd_count);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch ctx->epfd = epoll_create(initial_fd_count);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (ctx->epfd < 0) {
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (errno != EMFILE)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_fatal("epoll_create(): %m");
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch else {
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_fatal("epoll_create(): %m (you may need to increase "
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch "/proc/sys/fs/epoll/max_user_instances)");
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch }
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch }
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch fd_close_on_exec(ctx->epfd, TRUE);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch}
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Boschvoid io_loop_handler_deinit(struct ioloop *ioloop)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch{
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct ioloop_handler_context *ctx = ioloop->handler_context;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct io_list **list;
c6494255de7b934281dd052960fd8ab5aa48e79eTimo Sirainen unsigned int i, count;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch list = array_get_modifiable(&ctx->fd_index, &count);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch for (i = 0; i < count; i++)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_free(list[i]);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (close(ctx->epfd) < 0)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_error("close(epoll) failed: %m");
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch array_free(&ioloop->handler_context->fd_index);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch array_free(&ioloop->handler_context->events);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_free(ioloop->handler_context);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch}
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch#define IO_EPOLL_ERROR (EPOLLERR | EPOLLHUP)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch#define IO_EPOLL_INPUT (EPOLLIN | EPOLLPRI | IO_EPOLL_ERROR)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch#define IO_EPOLL_OUTPUT (EPOLLOUT | IO_EPOLL_ERROR)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Boschstatic int epoll_event_mask(struct io_list *list)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch{
069def4dc35022852d569b7ab75a3b19d2cb0f1cTimo Sirainen int events = 0, i;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct io_file *io;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch for (i = 0; i < IOLOOP_IOLIST_IOS_PER_FD; i++) {
6d573191bea1a64d6046be070487a5705a2d0204Stephan Bosch io = list->ios[i];
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (io == NULL)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch continue;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (io->io.condition & IO_READ)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch events |= IO_EPOLL_INPUT;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (io->io.condition & IO_WRITE)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch events |= IO_EPOLL_OUTPUT;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (io->io.condition & IO_ERROR)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch events |= IO_EPOLL_ERROR;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch }
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch return events;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch}
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Boschvoid io_loop_handle_add(struct io_file *io)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch{
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct ioloop_handler_context *ctx = io->io.ioloop->handler_context;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct io_list **list;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct epoll_event event;
fe681e6db72f30bd754b622005bbe298e5ca775aTimo Sirainen int op;
fe681e6db72f30bd754b622005bbe298e5ca775aTimo Sirainen bool first;
fe681e6db72f30bd754b622005bbe298e5ca775aTimo Sirainen
fe681e6db72f30bd754b622005bbe298e5ca775aTimo Sirainen list = array_idx_modifiable(&ctx->fd_index, io->fd);
fe681e6db72f30bd754b622005bbe298e5ca775aTimo Sirainen if (*list == NULL)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch *list = i_new(struct io_list, 1);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch first = ioloop_iolist_add(*list, io);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch memset(&event, 0, sizeof(event));
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch event.data.ptr = *list;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch event.events = epoll_event_mask(*list);
fb1be3de0159d6a10e916ad992e2bc53be64c6d5Timo Sirainen
fb1be3de0159d6a10e916ad992e2bc53be64c6d5Timo Sirainen op = first ? EPOLL_CTL_ADD : EPOLL_CTL_MOD;
fb1be3de0159d6a10e916ad992e2bc53be64c6d5Timo Sirainen
6bc9fb43cc1ac24693d030a6cbfa43bc7cbc82cbTimo Sirainen if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) {
6bc9fb43cc1ac24693d030a6cbfa43bc7cbc82cbTimo Sirainen if (errno == EPERM && op == EPOLL_CTL_ADD) {
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen i_fatal("epoll_ctl(add, %d) failed: %m "
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen "(fd doesn't support epoll%s)", io->fd,
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen io->fd != STDIN_FILENO ? "" :
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen " - instead of '<file', try 'cat file|'");
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch }
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_panic("epoll_ctl(%s, %d) failed: %m",
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch op == EPOLL_CTL_ADD ? "add" : "mod", io->fd);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch }
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (first) {
fc94140acba51adafedafbc8491a3223a51db7a8Stephan Bosch /* allow epoll_wait() to return the maximum number of events
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch by keeping space allocated for each file descriptor */
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (ctx->deleted_count > 0)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch ctx->deleted_count--;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch else
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch array_append_zero(&ctx->events);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch }
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch}
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Boschvoid io_loop_handle_remove(struct io_file *io, bool closed)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch{
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct ioloop_handler_context *ctx = io->io.ioloop->handler_context;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct io_list **list;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch struct epoll_event event;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch int op;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch bool last;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch list = array_idx_modifiable(&ctx->fd_index, io->fd);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch last = ioloop_iolist_del(*list, io);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (!closed) {
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch memset(&event, 0, sizeof(event));
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch event.data.ptr = *list;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch event.events = epoll_event_mask(*list);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch op = last ? EPOLL_CTL_DEL : EPOLL_CTL_MOD;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) {
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_error("epoll_ctl(%s, %d) failed: %m",
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch op == EPOLL_CTL_DEL ? "del" : "mod", io->fd);
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch }
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch }
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch if (last) {
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch /* since we're not freeing memory in any case, just increase
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch deleted counter so next handle_add() can just decrease it
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch insteading of appending to the events array */
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch ctx->deleted_count++;
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch }
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch i_free(io);
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch}
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Boschvoid io_loop_handler_run(struct ioloop *ioloop)
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch{
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch struct ioloop_handler_context *ctx = ioloop->handler_context;
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch struct epoll_event *events;
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch const struct epoll_event *event;
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch struct io_list *list;
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch struct io_file *io;
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch struct timeval tv;
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch unsigned int events_count;
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch int msecs, ret, i, j;
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch bool call;
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch /* get the time left for next timeout task */
b99130e4cf4af4e6b103b949456222f3a2dff424Timo Sirainen msecs = io_loop_get_wait_time(ioloop, &tv);
b99130e4cf4af4e6b103b949456222f3a2dff424Timo Sirainen
b99130e4cf4af4e6b103b949456222f3a2dff424Timo Sirainen events = array_get_modifiable(&ctx->events, &events_count);
b99130e4cf4af4e6b103b949456222f3a2dff424Timo Sirainen if (events_count > 0) {
87c121a4c05b9cee46f1f757ec6999d441519abfStephan Bosch ret = epoll_wait(ctx->epfd, events, events_count, msecs);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch if (ret < 0 && errno != EINTR)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch i_fatal("epoll_wait(): %m");
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch } else {
6d573191bea1a64d6046be070487a5705a2d0204Stephan Bosch /* no I/Os, but we should have some timeouts.
6d573191bea1a64d6046be070487a5705a2d0204Stephan Bosch just wait for them. */
6d573191bea1a64d6046be070487a5705a2d0204Stephan Bosch if (msecs > 0)
6d573191bea1a64d6046be070487a5705a2d0204Stephan Bosch usleep(msecs*1000);
6d573191bea1a64d6046be070487a5705a2d0204Stephan Bosch ret = 0;
6d573191bea1a64d6046be070487a5705a2d0204Stephan Bosch }
6d573191bea1a64d6046be070487a5705a2d0204Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch /* execute timeout handlers */
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch io_loop_handle_timeouts(ioloop);
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
4521d35c263add6af3f1ae55b3760291767ce50cTimo Sirainen if (!ioloop->running)
4521d35c263add6af3f1ae55b3760291767ce50cTimo Sirainen return;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen for (i = 0; i < ret; i++) {
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch /* io_loop_handle_add() may cause events array reallocation,
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch so we have use array_idx() */
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen event = array_idx(&ctx->events, i);
4521d35c263add6af3f1ae55b3760291767ce50cTimo Sirainen list = event->data.ptr;
4521d35c263add6af3f1ae55b3760291767ce50cTimo Sirainen
4521d35c263add6af3f1ae55b3760291767ce50cTimo Sirainen for (j = 0; j < IOLOOP_IOLIST_IOS_PER_FD; j++) {
4521d35c263add6af3f1ae55b3760291767ce50cTimo Sirainen io = list->ios[j];
4521d35c263add6af3f1ae55b3760291767ce50cTimo Sirainen if (io == NULL)
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen continue;
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen call = FALSE;
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen if ((event->events & (EPOLLHUP | EPOLLERR)) != 0)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch call = TRUE;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch else if ((io->io.condition & IO_READ) != 0)
65c0e43da8cfc730eeb4634f8aa384081bbfa4e7Timo Sirainen call = (event->events & EPOLLIN) != 0;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch else if ((io->io.condition & IO_WRITE) != 0)
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch call = (event->events & EPOLLOUT) != 0;
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch else if ((io->io.condition & IO_ERROR) != 0)
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen call = (event->events & IO_EPOLL_ERROR) != 0;
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen if (call)
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen io_loop_call_io(&io->io);
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen }
1d048c5050f03c24251e5af8087e640de21b2d62Timo Sirainen }
eb325a5a90c1d2655e74972bde0de6a699d2c864Stephan Bosch}
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch#endif /* IOLOOP_EPOLL */
7384b4e78eaab44693c985192276e31322155e32Stephan Bosch