/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <sys/nsc_thread.h>
#include <vm/seg_kmem.h>
#include "sd_bcache.h"
#include "sd_trace.h"
#include "sd_io.h"
#include "sd_iob.h"
#include "sd_misc.h"
#if defined(_SD_DEBUG) /* simulate disk errors */
#include "sd_tdaemon.h"
#endif
#ifndef DS_DDICT
#endif
int sdbc_do_page = 0;
static int sdbc_bio_count;
#ifdef _SD_BIO_STATS
static __start_io_count = 0;
#endif /* _SD_BIO_STATS */
/*
* Forward declare all statics that are used before defined to enforce
* parameter checking. Also forward-declare all functions that have 64-bit
* argument types to enforce correct parameter checking.
*
* Some (if not all) of these could be removed if the code were reordered
*/
#ifdef DEBUG
static int _sdbc_ioj_lookup(dev_t);
static void _sdbc_ioj_clear_err(int);
#endif
static int SD_WRITES_TOT = 0;
/*
* _sd_add_vm_to_bp_plist - add the page corresponding to the
* virtual address "v" (kernel virtaddr) to the pagelist linked
* to buffer "bp".
*
* The virtual address "v" is "known" to be allocated by segkmem
* and we can look up the page by using the segkmem vnode kvp.
*
*
*/
static void
{
if (!pp) {
"_sd_add_vm_to_bp_plist: couldn't find page for 0x%p",
(void *)v);
}
}
#ifdef _SD_BIO_STATS
static int
{
int cnt = 0;
return (cnt);
;
return (cnt);
}
#endif /* _SD_BIO_STATS */
/*
* _sdbc_iobuf_load - load time initialization of io bufs structures.
*
*
* RETURNS:
* 0 - success.
* -1 - failure.
*
* USAGE:
* This routine initializes load time buf structures.
* Should be called when the cache is loaded.
*/
int
_sdbc_iobuf_load(void)
{
/*
* HACK add a ref to kvp, to prevent VN_RELE on it from panicing
* the system
*/
return (0);
}
/*
* _sdbc_iobuf_unload - unload time cleanup of io buf structures.
*
*
* USAGE:
* This routine removes load time buf structures.
* Should be called when the cache is unloaded.
*/
void
_sdbc_iobuf_unload(void)
{
/* Undo our VN_HOLD hack, by putting ref count back to normal state */
}
/*
* _sdbc_iobuf_configure - configure a list of io bufs for later use.
*
* ARGUMENTS:
* num_bufs - number of buffers. (from the configuration file)
*
* RETURNS:
* 0 - success.
* <0 - failure.
*
* USAGE:
* This routine configures the buf structures for io.
* Should be called when the cache is configured.
*/
int
{
int i;
return (-1);
}
buflist = &_sd_buflist;
if (!hook->iob_drv_iodone) {
return (-2);
}
}
for (i = 0; i < MAX_HOOK_LOCKS; i++)
NULL);
_sd_buflist.hook_waiters = 0;
sdbc_bio_count = 0;
SD_WRITES_TOT = 0;
/* pagelist i/o pages must be done in cache_init */
return (0);
}
/*
* _sdbc_iobuf_deconfigure - release all memory allocated for buf list
*
* ARGUMENTS:
* None.
*
* RETURNS:
* 0
*/
void
_sdbc_iobuf_deconfigure(void)
{
ushort_t i;
if (_sd_buflist.hooks) {
for (i = 0; i < _sd_buflist.bl_init_count; i ++) {
}
for (i = 0; i < MAX_HOOK_LOCKS; i ++) {
}
}
#ifdef DEBUG
{
void _sdbc_ioj_clear_err(int);
}
#endif
}
/*
* _sd_pending_iobuf()
*
* Return the number of I/O bufs outstanding
*/
int
_sd_pending_iobuf(void)
{
return (sdbc_bio_count);
}
/*
* _sd_get_iobuf - allocate a buf.
*
* ARGUMENTS:
* None.
*
* RETURNS:
* NULL - failure.
* buf ptr otherwise.
*
* ASSUMPTIONS - process could block if we run out.
*
*/
/*ARGSUSED*/
static struct buf *
{
/* Get a buffer, ready for page list i/o */
if (DO_PAGE_LIST)
else
return (NULL);
return (bp);
}
/*
* _sd_put_iobuf - put a buf back in the freelist.
*
* ARGUMENTS:
* bp - buf pointer.
*
* RETURNS:
* 0
*
*/
static void
{
if (DO_PAGE_LIST)
else
}
/* use for ORing only */
#define B_KERNBUF 0
static void
{
/*
* if pagelist i/o, _sd_get_iobuf()/pageio_setup() has already
* set b_flags to
* B_KERNBUF | B_PAGEIO | B_NOCACHE | B_BUSY (sol 6,7,8)
* or
* B_PAGEIO | B_NOCACHE | B_BUSY (sol 9)
*/
}
/*
* _sd_get_hook - get an iob hook from the free list.
*
* ARGUMENTS:
* none
*
* RETURNS:
* the newly allocated iob_hook.
*
*/
static iob_hook_t *
_sd_get_hook(void)
{
if (ret)
else {
goto retry;
}
#ifdef _SD_BIO_STATS
ret->NORM_IO_SIZE = 0;
ret->PAGE_COMBINED = 0;
#endif /* _SD_BIO_STATS */
return (ret);
}
/*
* _sd_put_hook - put an iob hook back on the free list.
*
* ARGUMENTS:
* hook - an iob_hook to be returned to the freelist.
*
*
*/
static void
{
if (_sd_buflist.hook_waiters) {
}
}
/*
* _sd_extend_iob - the i/o block we are handling needs a new struct buf to
* describe the next hunk of i/o. Get a new struct buf initialize it based
* on the state in the struct buf we are passed as an arg.
* ARGUMENTS:
* head_bp - a buffer header in the current i/o block we are handling.
* (generally the initial header but in fact could be any
* of the ones [if any] that were chained to the initial
* one).
*/
static struct buf *
{
if (!(bp = _sd_get_iobuf(0)))
return (0);
if (!DO_PAGE_LIST)
/*
* associated with this block of i/o.
* hook->tail points to the last buffer in the chain.
*/
return (bp);
}
/*
* sd_alloc_iob - start processing a block of i/o. This allocates an initial
* buffer header for describing the i/o and a iob_hook for collecting
* information about all the i/o requests added to this buffer.
*
* ARGUMENTS:
* dev - the device all the i/o is destined for.
* fba_pos - the initial disk block to read.
* blks - ignored
* flag - signal whether this is a read or write request.
*
* RETURNS:
* pointer to free struct buf which will be used to describe i/o request.
*/
/* ARGSUSED */
struct buf *
{
if (!(bp = _sd_get_iobuf(0)))
return (0);
hook = _sd_get_hook();
if (!hook) {
/* can't see how this could happen */
return (0);
}
/*
* pick an arbitrary lock
*/
(MAX_HOOK_LOCKS - 1)];
return (bp);
}
/*
* _sd_pack_pages - produce i/o requests that will perform the type of i/o
* buf pointer to by list to minimize the number of bufs required.
*
* ARGUMENTS:
* bp - is the i/o description i.e. head
* list - is where to start adding this i/o request (null if we should extend)
* addr - address describing where the data is.
* offset - offset from addr where data begins
* size - size of the i/o request.
*/
static void
{
int page_end_aligned;
#ifdef _SD_BIO_STATS
#endif /* _SD_BIO_STATS */
/*
* we're hosed since we have no error return...
* though we could ignore stuff from here on out
* and return ENOMEM when we get to sd_start_io.
* This will do for now.
*/
}
/*
* We only want to do pagelist i/o if we end on a page boundary.
* If we don't end on a page boundary we won't combine with the
* next request and so we may as well do it as normal as it
* will only use one buffer.
*/
if (DO_PAGE_LIST && page_end_aligned) {
if (start_addr & page_offset_mask) {
/*
* handle the partial page
*/
/*
* we're hosed since we have no error
* return though we could ignore stuff
* from here on out and return ENOMEM
* when we get to sd_start_io.
* This will do for now.
*/
"_sd_pack_pages: couldn't extend iob");
}
}
#ifdef _SD_BIO_STATS
#endif /* _SD_BIO_STATS */
(unsigned char *) start_addr);
}
/*
* Now fill with all the full pages remaining.
*/
#ifdef _SD_BIO_STATS
#endif /* _SD_BIO_STATS */
(unsigned char *) start_addr);
start_addr += page_size;
#ifdef _SD_BIO_STATS
hook->PAGE_COMBINED++;
#endif /* _SD_BIO_STATS */
}
if (size)
} else {
/*
* Wasn't worth it as pagelist i/o, do as normal
*/
/*
* we're hosed since we have no error return...
* though we could ignore stuff from here on out
* and return ENOMEM when we get to sd_start_io.
* This will do for now.
*/
"_sd_pack_pages: couldn't extend iob");
}
/* kernel virtual */
#ifdef _SD_BIO_STATS
#endif /* _SD_BIO_STATS */
}
}
/*
* perform same function as _sd_pack_pages() when not doing pageio
*/
static void
{
#ifdef _SD_BIO_STATS
#endif /* _SD_BIO_STATS */
/*
* we're hosed since we have no error return...
* though we could ignore stuff from here on out
* and return ENOMEM when we get to sd_start_io.
* This will do for now.
*/
"extend iob");
}
/* contiguous */
} else {
/*
* not contiguous mem (extend) or first buffer (bufsize == 0).
*/
/*
* we're hosed since we have no error return...
* though we could ignore stuff from here on out
* and return ENOMEM when we get to sd_start_io.
* This will do for now.
*/
"extend iob");
}
}
#ifdef _SD_BIO_STATS
#endif /* _SD_BIO_STATS */
}
/*
* sd_add_fba - add an i/o request to the block of i/o described by bp.
* We try and combine this request with the previous request. In
* Addition we try and do the i/o as PAGELIST_IO if it satisfies
* the restrictions for it. If the i/o request can't be combined
* we extend the i/o description with a new buffer header and add
* it to the chain headed by bp.
*
* ARGUMENTS:
* bp - the struct buf describing the block i/o we are collecting.
* A NULL indicates that this i/o request doesn't need to actually
* happen. Used to mark reads when the fba is already in cache and
* dirty.
*
* fba_pos - offset from address in addr where the i/o is to start.
*
* fba_len - number of consecutive fbas to transfer.
*
* NOTE: It is assumed that the memory is physically contiguous but may span
* multiple pages (should a cache block be larger than a page).
*
*/
void
{
if (addr) {
/*
* See if this can be combined with previous request(s)
*/
if (DO_PAGE_LIST)
else
size);
} else {
if (DO_PAGE_LIST) {
/*
* Last buffer was a pagelist. Unless a
* skip was detected the last request
* ended on a page boundary. If this
* one starts on one we combine the
* best we can.
*/
else
} else {
/*
* Last buffer was vanilla i/o or worse
* (sd_add_mem)
*/
size);
}
} else {
else
}
}
} else {
/* Must be a read of dirty block we want to discard */
#ifdef _SD_BIO_STATS
#endif /* _SD_BIO_STATS */
}
}
/*
* sd_add_mem - add an i/o request to the block of i/o described by bp.
* The memory target for this i/o may span multiple pages and may
* not be physically contiguous.
* also the len might also not be a multiple of an fba.
*
* ARGUMENTS:
* bp - the struct buf describing the block i/o we are collecting.
*
* buf - target of this i/o request.
*
* len - number of bytes to transfer.
*
*/
void
{
nsc_size_t n;
/*
* i/o size must be multiple of an FBA since we can't
* count on lower level drivers to understand b_offset
*/
if (BLK_FBA_OFF(n) != 0) {
"!sdbc(sd_add_mem) i/o request not FBA sized (%"
NSC_SZFMT ")", n);
}
/* first request */
} else {
/* we're hosed */
"sd_add_mem: couldn't extend iob");
}
}
}
}
/*
* sd_start_io - start all the i/o needed to satisfy the i/o request described
* by bp. If supplied the a non-NULL fn then this is an async request
* and we will return NSC_PENDING and call fn when all the i/o complete.
* Otherwise this is a synchronous request and we sleep until all the
* i/o is complete. If any buffer in the chain gets an error we return
* the first error we see (once all the i/o is complete).
*
* ARGUMENTS:
* bp - the struct buf describing the block i/o we are collecting.
*
* strategy - strategy function to call if known by the user, or NULL.
*
* fn - user's callback function. NULL implies synchronous request.
*
* arg - an argument passed to user's callback function.
*
*/
int
{
int err;
#ifdef _SD_BIO_STATS
static int total_bufs;
#endif /* _SD_BIO_STATS */
else
ea_fn = _sd_sync_ea;
#ifdef _SD_BIO_STATS
#endif /* _SD_BIO_STATS */
(sizeof (SD_WRITES_LEN)/sizeof (int))]++;
}
#ifdef _SD_BIO_STATS
total_bufs ++;
int i;
if (i > max_run_r)
max_run_r = i;
total_xpages_r += i;
} else {
if (i > max_run_w)
max_run_w = i;
total_xpages_w += i;
}
}
#endif /* _SD_BIO_STATS */
/*
* It's possible for us to be told to read a dirty block
* where all the i/o can go away (e.g. read one fba, it's
* in cache and dirty) so we really have nothing to do but
* say we're done.
*/
if (!strategy) {
strategy =
}
if (!strategy) {
} else
#ifdef DEBUG
/* inject i/o error for testing */
} else
#endif
{
}
} else {
}
}
#ifdef _SD_BIO_STATS
if (__start_io_count == 2000) {
__start_io_count = 0;
"!sdbc(sd_start_io) t_bufs %d pages %d "
total_bufs = 0;
total_pages = 0;
total_pages_combined = 0;
total_norm = 0;
total_norm_combined = 0;
total_skipped = 0;
total_norm_size = 0;
"!sdbc(sd_start_io)(r) max_run %d, total_xp %d total yp %d",
total_xpages_r = 0;
total_ypages_r = 0;
max_run_r = 0;
"!sdbc(sd_start_io)(w) max_run %d, total_xp %d total yp %d",
total_xpages_w = 0;
total_ypages_w = 0;
max_run_w = 0;
}
#endif /* _SD_BIO_STATS */
if (ea_fn == _sd_async_ea) {
return (NSC_PENDING);
}
}
return (err);
}
/*
* _sd_sync_ea - called when a single i/o operation is complete. If this
* is the last outstanding i/o we wakeup the sleeper.
* If this i/o had an error then we store the error result in the
* iob_hook if this was the first error.
*
* ARGUMENTS:
* bp - the struct buf describing the block i/o that just completed.
*
* Comments:
* This routine is called at interrupt level when the io is done.
*/
static int
{
int error;
int done;
/*
* We get called for each buf that completes. When they are all done.
* we wakeup the waiter.
*/
if (done) {
/* remember the last buffer so we can free it later */
}
/*
* let sd_start_io free the final buffer so the hook can be returned
* first.
*/
if (!done)
return (0);
}
/*
* static int
*
* ARGUMENTS:
* bp - io buf pointer.
*
* RETURNS:
* NONE.
*
* Comments:
* This routine is called at interrupt level when the io is done.
* This is only called when the operation is asynchronous.
*/
static int
{
/*
* We get called for each buf that completes. When they are all done.
* we call the requestor's callback function.
*/
if (done) {
int error;
#if defined(_SD_DEBUG) /* simulate disk errors */
#endif
/* MAKE SURE b_lblkno, b_count never changes!! */
} else
return (0);
}
#ifdef DEBUG
typedef struct ioerr_inject_s {
int ioj_err;
int ioj_cnt;
void
{
}
void
{
if (ioerr_inject_table != NULL) {
sdbc_max_devs * sizeof (ioerr_inject_t));
}
}
static int
{
int cd;
return (0);
} else {
}
}
return (0);
}
void
{
int i;
for (i = 0; i < sdbc_max_devs; ++i) {
}
} else
}
static
void
{
int i;
for (i = 0; i < sdbc_max_devs; ++i) {
}
} else {
}
}
static void
{
_sdbc_ioj_set_err(cd, 0, 0);
}
int
{
return (EINVAL);
return (0);
}
int
{
return (EINVAL);
return (0);
}
#endif