/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _ASYNCIO_H
#define _ASYNCIO_H
#pragma ident "%Z%%M% %I% %E% SMI"
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <thread.h>
#include <pthread.h>
#include <setjmp.h>
#include <signal.h>
#include <siginfo.h>
#include <aio.h>
#include <limits.h>
#include <ucontext.h>
#include <sys/asynch.h>
#include <sys/mman.h>
#if !defined(_LP64)
#define AIOSTKSIZE (64 * 1024)
#else
#define AIOSTKSIZE (128 * 1024)
#endif
#define SIGAIOCANCEL SIGLWP /* special aio cancelation signal */
#define AIO_WAITN_MAXIOCBS 32768 /* max. iocbs per system call */
/*
* Declare structure types. The structures themselves are defined below.
*/
typedef struct aio_args aio_args_t;
typedef struct aio_lio aio_lio_t;
typedef struct notif_param notif_param_t;
typedef struct aio_req aio_req_t;
typedef struct aio_worker aio_worker_t;
typedef struct aio_hash aio_hash_t;
struct aio_args {
int fd;
caddr_t buf;
size_t bufsz;
offset_t offset;
};
/*
* list head for UFS list I/O
*/
struct aio_lio {
mutex_t lio_mutex; /* list mutex */
cond_t lio_cond_cv; /* list notification for I/O done */
aio_lio_t *lio_next; /* pointer to next on freelist */
char lio_mode; /* LIO_WAIT/LIO_NOWAIT */
char lio_canned; /* lio was canceled */
char lio_largefile; /* largefile operation */
char lio_waiting; /* waiting in __lio_listio() */
int lio_nent; /* Number of list I/O's */
int lio_refcnt; /* outstanding I/O's */
int lio_event; /* Event number for notification */
int lio_port; /* Port number for notification */
int lio_signo; /* Signal number for notification */
union sigval lio_sigval; /* Signal parameter */
uintptr_t lio_object; /* for SIGEV_THREAD or SIGEV_PORT */
struct sigevent *lio_sigevent; /* Notification function and attr. */
};
/*
* Notification parameters
*/
struct notif_param {
int np_signo; /* SIGEV_SIGNAL */
int np_port; /* SIGEV_THREAD or SIGEV_PORT */
void *np_user;
int np_event;
uintptr_t np_object;
int np_lio_signo; /* listio: SIGEV_SIGNAL */
int np_lio_port; /* listio: SIGEV_THREAD or SIGEV_PORT */
void *np_lio_user;
int np_lio_event;
uintptr_t np_lio_object;
};
struct aio_req {
/*
* fields protected by _aio_mutex lock.
*/
aio_req_t *req_link; /* hash/freelist chain link */
/*
* when req is on the doneq, then req_next is protected by
* the _aio_mutex lock. when the req is on a work q, then
* req_next is protected by a worker's work_qlock1 lock.
*/
aio_req_t *req_next; /* request/done queue link */
aio_req_t *req_prev; /* double linked list */
/*
* fields protected by a worker's work_qlock1 lock.
*/
char req_state; /* AIO_REQ_QUEUED, ... */
/*
* fields require no locking.
*/
char req_type; /* AIO_POSIX_REQ or not */
char req_largefile; /* largefile operation */
char req_op; /* AIOREAD, etc. */
aio_worker_t *req_worker; /* associate request with worker */
aio_result_t *req_resultp; /* address of result buffer */
aio_args_t req_args; /* arglist */
aio_lio_t *req_head; /* list head for LIO */
struct sigevent req_sigevent;
void *req_aiocbp; /* ptr to aiocb or aiocb64 */
notif_param_t req_notify; /* notification parameters */
};
/* special lio type that destroys itself when lio refcnt becomes zero */
#define LIO_FSYNC LIO_WAIT+1
#define LIO_DESTROY LIO_FSYNC+1
/* lio flags */
#define LIO_FSYNC_CANCELED 0x1
/* values for aio_state */
#define AIO_REQ_QUEUED 1
#define AIO_REQ_INPROGRESS 2
#define AIO_REQ_CANCELED 3
#define AIO_REQ_DONE 4
#define AIO_REQ_FREE 5
#define AIO_REQ_DONEQ 6
/* use KAIO in _aio_rw() */
#define AIO_NO_KAIO 0x0
#define AIO_KAIO 0x1
#define AIO_NO_DUPS 0x2
#define AIO_POSIX_REQ 0x1
#define CHECK 1
#define NOCHECK 2
#define CHECKED 3
#define USERAIO 4
#define USERAIO_DONE 5
/* values for _aio_flags */
/* if set, _aiodone() notifies aio_waitn about done requests */
#define AIO_WAIT_INPROGRESS 0x1
/* if set, _aiodone() wakes up functions waiting for completed I/Os */
#define AIO_IO_WAITING 0x2
#define AIO_LIB_WAITN 0x4 /* aio_waitn in progress */
#define AIO_LIB_WAITN_PENDING 0x8 /* aio_waitn requests pending */
/*
* Before a kaio() system call, the fd will be checked
* to ensure that kernel async. I/O is supported for this file.
* The only way to find out is if a kaio() call returns ENOTSUP,
* so the default will always be to try the kaio() call. Only in
* the specific instance of a kaio() call returning ENOTSUP
* will we stop submitting kaio() calls for that fd.
* If the fd is outside the array bounds, we will allow the kaio()
* call.
*
* The only way that an fd entry can go from ENOTSUP to supported
* is if that fd is freed up by a close(), and close will clear
* the entry for that fd.
*
* Each fd gets a bit in the array _kaio_supported[].
*
* uint32_t _kaio_supported[MAX_KAIO_FDARRAY_SIZE];
*
* Array is MAX_KAIO_ARRAY_SIZE of 32-bit elements, for 8kb.
* If more than (MAX_KAIO_FDARRAY_SIZE * KAIO_FDARRAY_ELEM_SIZE)
* files are open, this can be expanded.
*/
#define MAX_KAIO_FDARRAY_SIZE 2048
#define KAIO_FDARRAY_ELEM_SIZE WORD_BIT /* uint32_t */
#define MAX_KAIO_FDS (MAX_KAIO_FDARRAY_SIZE * KAIO_FDARRAY_ELEM_SIZE)
#define VALID_FD(fdes) ((fdes) >= 0 && (fdes) < MAX_KAIO_FDS)
#define KAIO_SUPPORTED(fdes) \
(!VALID_FD(fdes) || \
((_kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] & \
(uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE))) == 0))
#define SET_KAIO_NOT_SUPPORTED(fdes) \
if (VALID_FD(fdes)) \
_kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] |= \
(uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE))
#define CLEAR_KAIO_SUPPORTED(fdes) \
if (VALID_FD(fdes)) \
_kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] &= \
~(uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE))
struct aio_worker {
aio_worker_t *work_forw; /* forward link in list of workers */
aio_worker_t *work_backw; /* backwards link in list of workers */
mutex_t work_qlock1; /* lock for work queue 1 */
cond_t work_idle_cv; /* place to sleep when idle */
aio_req_t *work_head1; /* head of work request queue 1 */
aio_req_t *work_tail1; /* tail of work request queue 1 */
aio_req_t *work_next1; /* work queue one's next pointer */
aio_req_t *work_prev1; /* last request done from queue 1 */
aio_req_t *work_req; /* active work request */
thread_t work_tid; /* worker's thread-id */
int work_count1; /* length of work queue one */
int work_done1; /* number of requests done */
int work_minload1; /* min length of queue */
int work_idleflg; /* when set, worker is idle */
sigjmp_buf work_jmp_buf; /* cancellation point */
};
struct aio_hash { /* resultp hash table */
mutex_t hash_lock;
aio_req_t *hash_ptr;
#if !defined(_LP64)
void *hash_pad; /* ensure sizeof (aio_hash_t) == 32 */
#endif
};
extern aio_hash_t *_aio_hash;
#define HASHSZ 2048 /* power of 2 */
#define AIOHASH(resultp) ((((uintptr_t)(resultp) >> 17) ^ \
((uintptr_t)(resultp) >> 2)) & (HASHSZ - 1))
#define POSIX_AIO(x) ((x)->req_type == AIO_POSIX_REQ)
extern int __uaio_init(void);
extern void _kaio_init(void);
extern intptr_t _kaio(int, ...);
extern int _aiorw(int, caddr_t, int, offset_t, int, aio_result_t *, int);
extern int _aio_rw(aiocb_t *, aio_lio_t *, aio_worker_t **, int, int);
#if !defined(_LP64)
extern int _aio_rw64(aiocb64_t *, aio_lio_t *, aio_worker_t **, int, int);
#endif
extern int _aio_create_worker(aio_req_t *, int);
extern int _aio_cancel_req(aio_worker_t *, aio_req_t *, int *, int *);
extern int aiocancel_all(int);
extern void aio_panic(const char *);
extern aio_req_t *_aio_hash_find(aio_result_t *);
extern aio_req_t *_aio_hash_del(aio_result_t *);
extern void _aio_req_mark_done(aio_req_t *);
extern void _aio_waitn_wakeup(void);
extern aio_worker_t *_aio_worker_alloc(void);
extern void _aio_worker_free(void *);
extern aio_req_t *_aio_req_alloc(void);
extern void _aio_req_free(aio_req_t *);
extern aio_lio_t *_aio_lio_alloc(void);
extern void _aio_lio_free(aio_lio_t *);
extern int _aio_idle(aio_worker_t *);
extern void *_aio_do_request(void *);
extern void *_aio_do_notify(void *);
extern void _lio_remove(aio_req_t *);
extern aio_req_t *_aio_req_remove(aio_req_t *);
extern int _aio_get_timedelta(timespec_t *, timespec_t *);
extern aio_result_t *_aio_req_done(void);
extern void _aio_set_result(aio_req_t *, ssize_t, int);
extern int _aio_sigev_thread_init(struct sigevent *);
extern int _aio_sigev_thread(aiocb_t *);
#if !defined(_LP64)
extern int _aio_sigev_thread64(aiocb64_t *);
#endif
extern aio_worker_t *_kaiowp; /* points to kaio cleanup thread */
extern aio_worker_t *__workers_rw; /* list of all rw workers */
extern aio_worker_t *__nextworker_rw; /* worker chosen for next rw request */
extern int __rw_workerscnt; /* number of rw workers */
extern aio_worker_t *__workers_no; /* list of all notification workers */
extern aio_worker_t *__nextworker_no; /* worker chosen, next notification */
extern int __no_workerscnt; /* number of notification workers */
extern mutex_t __aio_initlock; /* makes aio initialization atomic */
extern cond_t __aio_initcv;
extern int __aio_initbusy;
extern mutex_t __aio_mutex; /* global aio lock */
extern cond_t _aio_iowait_cv; /* wait for userland I/Os */
extern cond_t _aio_waitn_cv; /* wait for end of aio_waitn */
extern int _max_workers; /* max number of workers permitted */
extern int _min_workers; /* min number of workers */
extern sigset_t _worker_set; /* worker's signal mask */
extern int _aio_worker_cnt; /* number of AIO workers */
extern int _sigio_enabled; /* when set, send SIGIO signal */
extern pid_t __pid; /* process's PID */
extern int __uaio_ok; /* indicates if aio is initialized */
extern int _kaio_ok; /* indicates if kaio is initialized */
extern pthread_key_t _aio_key; /* for thread-specific data */
extern aio_req_t *_aio_done_tail; /* list of done requests */
extern aio_req_t *_aio_done_head;
extern aio_req_t *_aio_doneq;
extern int _aio_freelist_cnt;
extern int _aio_allocated_cnt;
extern int _aio_donecnt;
extern int _aio_doneq_cnt;
extern int _aio_waitncnt; /* # of requests for aio_waitn */
extern int _aio_outstand_cnt; /* # of outstanding requests */
extern int _kaio_outstand_cnt; /* # of outstanding kaio requests */
extern int _aio_req_done_cnt; /* req. done but not in "done queue" */
extern int _aio_kernel_suspend; /* active kernel kaio calls */
extern int _aio_suscv_cnt; /* aio_suspend calls waiting on cv's */
extern int _aiowait_flag; /* when set, aiowait() is inprogress */
extern int _aio_flags; /* see defines, above */
extern uint32_t *_kaio_supported;
extern const sigset_t maskset; /* all maskable signals */
#ifdef __cplusplus
}
#endif
#endif /* _ASYNCIO_H */