sd_bcache.c revision fcf3ce441efd61da9bb2884968af01cb7c1452cc
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2008 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <sys/nsc_thread.h>
#include "sd_bcache.h"
#include "sd_trace.h"
#include "sd_io.h"
#include "sd_bio.h"
#include "sd_ft.h"
#include "sd_misc.h"
#include "sd_pcu.h"
#ifndef DS_DDICT
#include <sys/ddi_impldefs.h>
#endif
/*
* kstat interface
*/
static kstat_t *sdbc_global_stats_kstat;
typedef struct {
#ifdef DEBUG
#endif
static sdbc_global_stats_t sdbc_global_stats = {
#ifdef DEBUG
#endif
};
static kstat_t **sdbc_cd_kstats;
static kstat_t **sdbc_cd_io_kstats;
static kmutex_t *sdbc_cd_io_kstats_mutexes;
static kstat_t *sdbc_global_io_kstat;
static kmutex_t sdbc_global_io_kstat_mutex;
static int cd_kstat_add(int cd);
static int cd_kstat_remove(int cd);
typedef struct {
static sdbc_cd_stats_t sdbc_cd_stats = {
#ifdef NSC_MULTI_TERABYTE
#else
#endif
};
#ifdef DEBUG
/*
* dynmem kstat interface
*/
static kstat_t *sdbc_dynmem_kstat_dm;
static int simplect_dm;
typedef struct {
static sdbc_dynmem_dm_t sdbc_dynmem_dm = {
};
#endif
/* End of dynmem kstats */
#ifdef DEBUG
int *dmchainpull_table; /* dmchain wastage stats */
#endif
/*
* dynmem process vars
*/
/* metadata for volumes */
/* metadata for cache write blocks */
static ss_centry_info_t *_sdbc_gl_centry_info;
/* wblocks * sizeof(ss_centry_info_t) */
static size_t _sdbc_gl_centry_info_size;
static int _SD_DELAY_QUEUE = 1;
static int sdbc_allocb_pageio1, sdbc_allocb_pageio2;
static int sdbc_dmchain_not_avail;
static int sdbc_allocb_deallocd;
static int sdbc_centry_deallocd;
static int sdbc_check_cot;
static int sdbc_ra_hash; /* 1-block read-ahead fails due to hash hit */
static int sdbc_ra_none; /* 1-block read-ahead fails due to "would block" */
/*
* Set the following variable to 1 to enable pagelist io mutual
* exclusion on all _sd_alloc_buf() operations.
*
* This is set to ON to prevent front end / back end races between new
* NSC_WRTHRU io operations coming in through _sd_alloc_buf(), and
* previously written data being flushed out to disk by the sdbc
* flusher at the back end.
* -- see bugtraq 4287564
* -- Simon Crosland, Mon Nov 8 16:34:09 GMT 1999
*/
static int sdbc_pageio_always = 1;
int sdbc_use_dmchain = 0; /* start time switch for dm chaining */
/*
* if sdbc_static_cache is 1 allocate all cache memory at startup.
* deallocate only at shutdown.
*/
int sdbc_static_cache = 1;
#ifdef DEBUG
/*
* Pagelist io mutual exclusion debug facility.
*/
#define SDBC_PAGEIO_OFF 0 /* no debug */
static int sdbc_pageio_debug = SDBC_PAGEIO_OFF;
#endif
/*
* INF SD cache global data
*/
int _sd_cctl_groupsz;
extern krwlock_t sdbc_queue_lock;
unsigned int _sd_node_hint;
int BLK_FBAS; /* number of FBA's in a cache block */
int CACHE_BLOCK_SIZE; /* size in bytes of a cache block */
int CBLOCKS;
static int sdbc_prefetch_valid_cnt;
static int sdbc_prefetch_busy_cnt;
static int sdbc_prefetch_trailing;
static int sdbc_prefetch_deallocd;
static int sdbc_prefetch_pageio1;
static int sdbc_prefetch_pageio2;
static int sdbc_prefetch_hit;
static int sdbc_prefetch_lost;
_sd_bitmap_t _fba_bits[] = {
0x0000, 0x0001, 0x0003, 0x0007,
0x000f, 0x001f, 0x003f, 0x007f,
0x00ff,
#if defined(_SD_8K_BLKSIZE)
0x01ff, 0x03ff, 0x07ff,
0x0fff, 0x1fff, 0x3fff, 0x7fff,
0xffff,
#endif
};
static int _sd_ccsync_cnt = 256;
static _sd_cctl_sync_t *_sd_ccent_sync;
#ifdef _MULTI_DATAMODEL
#endif
#ifdef DEBUG
#else
#endif
/*
* Forward declare all statics that are used before defined to enforce
* parameter checking
* Some (if not all) of these could be removed if the code were reordered
*/
static void _sdbc_stats_deconfigure(void);
static int _sdbc_stats_configure(int cblocks);
static int _sdbc_lruq_configure(_sd_queue_t *);
static void _sdbc_lruq_deconfigure(void);
static void _sdbc_mem_deconfigure(int cblocks);
static int _sd_flush_cd(int cd);
_sd_buf_handle_t **hp);
static int _sdbc_gl_centry_configure(spcs_s_info_t);
static int _sdbc_gl_file_configure(spcs_s_info_t);
static void _sdbc_gl_centry_deconfigure(void);
static void _sdbc_gl_file_deconfigure(void);
/* dynmem support */
static int sdbc_check_cctl_cot(_sd_cctl_t *);
static int sdbc_dmqueues_configure();
static void sdbc_dmqueues_deconfigure();
static _sd_cctl_t *sdbc_get_dmchain(int, int *, int);
static int sdbc_dmchain_avail(_sd_cctl_t *);
void sdbc_requeue_head_dm_try(_sd_cctl_t *);
sdbc_allocbuf_t *, int);
int);
static void sdbc_centry_init_dm(_sd_cctl_t *);
static int sdbc_centry_memalloc_dm(_sd_cctl_t *, int, int);
static void sdbc_centry_alloc_end(sdbc_allocbuf_t *);
/* _SD_DEBUG */
static int _sd_cctl_valid(_sd_cctl_t *);
#endif
static
nsc_def_t _sdbc_fd_def[] = {
0, 0, 0
};
/*
* _sdbc_cache_configure - initialize cache blocks, queues etc.
*
* ARGUMENTS:
* cblocks - Number of cache blocks
*
* RETURNS:
* 0 on success.
* SDBC_EENABLEFAIL or SDBC_EMEMCONFIG on failure.
*
*/
int
{
_sd_cache_files = (_sd_cd_info_t *)
KM_SLEEP);
if (_sdbc_stats_configure(cblocks))
return (SDBC_EENABLEFAIL);
if (sdbc_use_dmchain) {
if (sdbc_dmqueues_configure())
return (SDBC_EENABLEFAIL);
} else {
return (SDBC_EENABLEFAIL);
}
return (SDBC_EMEMCONFIG);
sdbc_allocb_pageio1 = 0;
sdbc_allocb_pageio2 = 0;
sdbc_allocb_hit = 0;
sdbc_allocb_inuse = 0;
sdbc_allocb_lost = 0;
sdbc_centry_inuse = 0;
sdbc_centry_lost = 0;
sdbc_centry_hit = 0;
sdbc_centry_deallocd = 0;
sdbc_allocb_deallocd = 0;
sdbc_prefetch_hit = 0;
sdbc_prefetch_lost = 0;
sdbc_check_cot = 0;
sdbc_prefetch1 = 1;
sdbc_ra_hash = 0;
sdbc_ra_none = 0;
return (0);
}
/*
* _sdbc_cache_deconfigure - cache is being deconfigured. Release any
* memory that we acquired during the configuration process and return
* to the unconfigured state.
*
* NOTE: all users of the cache should be inactive at this point,
* gone.
*
*/
void
_sdbc_cache_deconfigure(void)
{
/* CCIO shutdown must happen before memory is free'd */
if (_sd_cache_files) {
sdbc_max_devs * sizeof (_sd_cd_info_t));
}
BLK_FBA_BITS = 0;
BLK_FBAS = 0;
CACHE_BLOCK_SIZE = 0;
if (sdbc_use_dmchain)
else
CBLOCKS = 0;
}
/*
* _sdbc_stats_deconfigure - cache is being deconfigured turn off
* stats. This could seemingly do more but we leave most of the
* data intact until cache is configured again.
*
*/
static void
_sdbc_stats_deconfigure(void)
{
int i;
#ifdef DEBUG
if (sdbc_dynmem_kstat_dm) {
}
#endif
if (sdbc_global_stats_kstat) {
}
if (sdbc_cd_kstats) {
for (i = 0; i < sdbc_max_devs; i++) {
if (sdbc_cd_kstats[i]) {
sdbc_cd_kstats[i] = NULL;
}
}
}
if (sdbc_global_io_kstat) {
}
if (sdbc_cd_io_kstats) {
for (i = 0; i < sdbc_max_devs; i++) {
if (sdbc_cd_io_kstats[i]) {
sdbc_cd_io_kstats[i] = NULL;
}
}
}
if (sdbc_cd_io_kstats_mutexes) {
/* mutexes are already destroyed in cd_kstat_remove() */
}
if (_sd_cache_stats) {
sizeof (_sd_stats_t) +
}
#ifdef _MULTI_DATAMODEL
if (_sd_cache_stats32) {
}
#endif
}
static int
{
_sd_cache_stats->st_wrcancelns = 0;
_sd_cache_stats->st_destaged = 0;
#ifdef _MULTI_DATAMODEL
#endif
/* kstat implementation - global stats */
sizeof (sdbc_global_stats)/sizeof (kstat_named_t),
if (sdbc_global_stats_kstat != NULL) {
} else {
}
/* global I/O kstats */
if (sdbc_global_io_kstat) {
NULL);
}
/*
* kstat implementation - cd stats
* NOTE: one kstat instance for each open cache descriptor
*/
KM_SLEEP);
/*
* kstat implementation - i/o kstats per cache descriptor
* NOTE: one I/O kstat instance for each cd
*/
KM_SLEEP);
#ifdef DEBUG
/* kstat implementation - dynamic memory stats */
sizeof (sdbc_dynmem_dm)/sizeof (kstat_named_t),
if (sdbc_dynmem_kstat_dm != NULL) {
} else {
}
#endif
return (0);
}
/*
* sdbc_dmqueues_configure()
* initialize the queues of dynamic memory chains.
*/
static int max_dm_queues;
static int
{
int i;
/*
* CAUTION! this code depends on max_dyn_list not changing
* if it does change behavior may be incorrect, as cc_alloc_size_dm
* depends on max_dyn_list and indexes to dmqueues are derived from
* cc_alloc_size_dm.
* see _sd_setup_category_on_type() and _sd_dealloc_dm()
* TODO: prevent max_dyn_list from on-the-fly modification (easy) or
* allow for on-the-fly changes to number of dm queues (hard).
*/
++max_dm_queues; /* need a "0" queue for centrys with no memory */
sdbc_dm_queues = (_sd_queue_t *)
KM_SLEEP);
#ifdef DEBUG
max_dm_queues * sizeof (int),
KM_SLEEP);
#endif
for (i = 0; i < max_dm_queues; ++i) {
(void) _sdbc_lruq_configure(&sdbc_dm_queues[i]);
sdbc_dm_queues[i].sq_dmchain_cblocks = i;
}
return (0);
}
static void
{
/* CAUTION! this code depends on max_dyn_list not changing */
if (sdbc_dm_queues)
max_dm_queues = 0;
}
/*
* _sdbc_lruq_configure - initialize the lru queue
*
* ARGUMENTS: NONE
* RETURNS: 0
*
*/
static int
{
return (0);
}
/*
* _sdbc_lruq_deconfigure - deconfigure the lru queue
*
* ARGUMENTS: NONE
*
*/
static void
_sdbc_lruq_deconfigure(void)
{
}
/*
* _sdbc_mem_configure - initialize the cache memory.
* Create and initialize the hash table.
* Create cache control blocks and fill them with relevent
* information and enqueue onto the lru queue.
* Initialize the Write control blocks (blocks that contain
* information as to where the data will be mirrored)
* Initialize the Fault tolerant blocks (blocks that contain
* information about the mirror nodes dirty writes)
*
* ARGUMENTS:
* cblocks - Number of cache blocks.
* RETURNS: 0
*
*/
static int
{
return (-1);
}
((cblocks % _SD_CCTL_GROUPS) != 0);
for (i = 0; i < _SD_CCTL_GROUPS; i++) {
_sd_cctl[i] = (_sd_cctl_t *)
return (-1);
}
}
if (_sd_ccent_sync == NULL) {
return (-1);
}
for (i = 0; i < _sd_ccsync_cnt; i++) {
NULL);
}
blk = 0;
netc = &_sd_net_config;
prev_entry_dm = 0;
first_entry_dm = 0;
centry->cc_iocount = 0;
if (!first_entry_dm)
if (prev_entry_dm)
{
_sd_queue_t *q;
if (sdbc_use_dmchain) {
q = &sdbc_dm_queues[0];
centry->cc_cblocks = 0;
} else
q = _SD_LRU_Q;
_sd_ins_queue(q, centry);
}
}
if (_sdbc_gl_centry_configure(kstatus) != 0)
return (-1);
if (_sdbc_gl_file_configure(kstatus) != 0)
return (-1);
return (0);
}
/*
* _sdbc_gl_file_configure()
* allocate and initialize space for the global filename data.
*
*/
static int
{
int err = 0;
sizeof (ss_voldata_t);
KM_NOSLEEP)) == NULL) {
return (-1);
}
/* setup the key to get a directory stream of all volumes */
/*
* if coming up after a crash, "refresh" the host
* memory copy from safestore.
*/
if (_sdbc_warm_start()) {
"cannot read safestore");
return (-1);
}
/*
* cycle through the vdir getting volume data
* and volume tokens
*/
== SS_OK) {
++fileinfo;
}
/*
* fail to configure since
* recovery is not possible.
*/
return (-1);
}
} else { /* normal initialization, not a warm start */
/*
* if this fails, continue: cache will start
* in writethru mode
*/
"cannot read safestore");
return (-1);
}
/*
* cycle through the vdir getting just the volume tokens
* and initializing volume entries
*/
&tempfinfo)) == 0) {
/*
* initialize the host memory copy of the
* global file region. this means setting the
* _pinned and _attached fields to _SD_NO_HOST
* because the default of zero conflicts with
* the min nodeid of zero.
*/
/* initialize the directory entry */
== SS_ERR) {
"sdbc(_sdbc_gl_file_configure): "
"volume entry write failure %p",
break;
}
++fileinfo;
}
/* coming up clean, continue in w-t mode */
"unable to init safe store volinfo");
}
return (0);
}
static void
{
if (_sdbc_gl_centry_info)
}
static int
{
int wblocks;
int err = 0;
KM_NOSLEEP)) == NULL) {
"sdbc(_sdbc_gl_centry_configure) "
"alloc failed for gl_centry_info region");
return (-1);
}
/*
* synchronize the centry info area with safe store
*/
/* setup the key to get a directory stream of all centrys */
if (_sdbc_warm_start()) {
"cannot read safestore");
return (-1);
}
/*
* cycle through the cdir getting resource
* tokens and reading centrys
*/
== 0) {
++cinfo;
}
/*
* fail to configure since
* recovery is not possible.
*/
return (-1);
}
} else {
"cannot read safestore");
return (-1);
}
/*
* cycle through the cdir getting resource
* tokens and initializing centrys
*/
== 0) {
== SS_ERR) {
"sdbc(_sdbc_gl_centry_configure): "
"cache entry write failure %p",
break;
}
++cinfo;
}
/* coming up clean, continue in w-t mode */
"_sdbc_gl_centry_info initialization failed");
}
}
return (0);
}
static void
{
if (_sdbc_gl_file_info)
}
/*
* _sdbc_mem_deconfigure - deconfigure the cache memory.
*
* ARGUMENTS:
* cblocks - Number of cache blocks.
*
*/
/* ARGSUSED */
static void
{
int i;
if (_sd_ccent_sync) {
for (i = 0; i < _sd_ccsync_cnt; i++) {
}
_sd_ccsync_cnt * sizeof (_sd_cctl_sync_t));
}
for (i = 0; i < _SD_CCTL_GROUPS; i++) {
_sd_cctl_groupsz * sizeof (_sd_cctl_t));
}
}
_sd_cctl_groupsz = 0;
_sd_htable = NULL;
}
static int
{
int i, valid;
valid = 0;
for (i = 0; i < _SD_CCTL_GROUPS; i++) {
valid = 1;
break;
}
}
return (valid);
}
#endif
/*
* _sd_ins_queue - insert centry into LRU queue
* (during initialization, locking not required)
*/
static void
{
q->sq_inq++;
ASSERT(GOOD_LRUSIZE(q));
}
void
{
_sd_queue_t *q = _SD_LRU_Q;
/* was FAST */
mutex_enter(&q->sq_qlock);
#if defined(_SD_DEBUG)
if (1) {
if (!_sd_cctl_valid(centry) ||
!_sd_cctl_valid(qp))
"_sd_requeue %x prev %x next %x qp %x",
}
#endif
/* was FAST */
mutex_exit(&q->sq_qlock);
(q->sq_req_stat)++;
}
void
{
_sd_queue_t *q = _SD_LRU_Q;
/* was FAST */
mutex_enter(&q->sq_qlock);
#if defined(_SD_DEBUG)
if (1) {
if (!_sd_cctl_valid(centry) ||
!_sd_cctl_valid(qn))
"_sd_requeue_head %x prev %x next %x qn %x",
}
#endif
/* was FAST */
mutex_exit(&q->sq_qlock);
}
/*
* _sd_open - Open a file.
*
* ARGUMENTS:
* filename - Name of the file to be opened.
* flag - Flag associated with open.
* (currently used to determine a ckd device)
* RETURNS:
* cd - the cache descriptor.
*/
int
{
int cd;
if (!_sd_cache_initialized) {
return (-EINVAL);
}
return (cd);
}
static int
{
int cd;
int rc = 0;
} else
return (rc);
}
int
{
int preexists = 0;
int failover_open, open_failed;
return (-EIO);
return (-ENAMETOOLONG);
/*
* If the cd is >= 0, then this is a open for a specific cd.
* This happens when the mirror node crashes, and we attempt to
* reopen the files with the same cache descriptors as existed on
* the other node
*/
failover_open = 0;
open_failed = 0;
if (cd >= 0) {
cd);
return (-EEXIST);
}
preexists = 1;
else {
filename);
}
}
goto known_cd;
}
new_cd = 0;
continue;
preexists = 1;
else {
if (cd == -2) {
return (-1);
}
filename);
/* update safestore */
}
}
break;
}
if (alloc_cd == -1)
return (-ENOSPC);
/*
* If preexists: someone else is attempting to open this file as
* well. Do only one open, but block everyone else here till the
* open is completed.
*/
if (preexists) {
}
goto retry_open;
return (alloc_cd);
}
}
/*
* take into account that there may be pinned data on a
* device that can no longer be opened
*/
open_failed++;
if (!rc)
return (-rc);
}
}
#ifdef DEBUG
/* put the dev_t in the ioerr_inject_table */
#endif
if (open_failed) {
else
#ifndef _SD_NOTRACE
(void) _sdbc_tr_configure(alloc_cd);
#endif
if (cd_kstat_add(alloc_cd) < 0) {
"%d", alloc_cd);
}
}
/*
* _sd_close - Close a cache descriptor.
*
* ARGUMENTS:
* cd - the cache descriptor to be closed.
* RETURNS:
* 0 on success.
* Error otherwise.
*
* Note: Under Construction.
*/
int
{
int rc;
if (!FILE_OPENED(cd)) {
goto out;
}
goto out;
}
/*
* _sd_flush_cd() will return -1 for the case where pinned
* data is present, but has been transfered to the mirror
* node. In this case it is safe to close the device as
* though _sd_flush_cd() had returned 0.
*/
if (rc == -1)
rc = 0;
if (rc != 0) {
}
goto out;
}
if (rc) {
goto out;
}
if (cd_kstat_remove(cd) < 0) {
"%d", cd);
}
/* cdi->cd_info = NULL; */
goto out;
out:
return (rc);
}
static int
{
int rc = 0;
}
return (rc);
}
/*
* to our remote mirror. Returns count of blocks reflected or -1 on error.
*
*/
int
{
int cnt = 0;
}
while (cc_ent) {
cnt++;
/* is this always necessary? jgk */
CACHE_BLOCK_SIZE, 0)) {
return (-1);
}
/* update the cache block metadata */
if (!cc_ent)
}
}
return (cnt);
}
/*
* _sd_flush_cd()
* reflect pinned blocks to mirrored node
* wait for dirty blocks to be flushed
* returns:
* EIO I/O failure, or pinned blocks and no mirror
* EAGAIN Hang: count of outstanding writes isn't decreasing
* -1 pinned blocks, reflected to mirror
* 0 success
*/
static int
_sd_flush_cd(int cd)
{
int rc;
return (0);
/*
* if we timed out simply return otherwise
* it must be an i/o type of error
*/
return (rc);
if (_sd_is_mirror_down())
return (EIO); /* already failed, no mirror */
if (_sdbc_remote_store_pinned(cd) >= 0)
/*
* At this point it looks like we have blocks on the
* failed list and taking up space on this node but
* no longer have responsibility for the blocks.
* These blocks will in fact be freed from the cache
* and the failed list when the mirror picks them up
* from safe storage and then calls _sd_cd_discard_mirror
* which will issue an rpc telling us to finish up.
*
* Should the other node die before sending the rpc then
* we are safe with these blocks simply waiting on the
* failed list.
*/
return (-1);
else
return (rc);
}
/*
* _sdbc_io_attach_cd -- set up for client access to device, reserve raw device
*
* ARGUMENTS:
* cd - the cache descriptor to attach.
*
* RETURNS:
* 0 on success.
* Error otherwise.
*/
int
{
int rc = 0;
if (!_sd_cache_initialized ||
!FILE_OPENED(cd)) {
return (EINVAL);
}
/*
* check if disk is failed without raw device open. If it is,
* it has to be recovered using _sd_disk_online
*/
_sd_print(3,
"_sdbc_io_attach_cd: pinned data. returning EINVAL");
return (EINVAL);
}
return (EINVAL);
}
#if defined(_SD_FAULT_RES)
/* wait for node recovery to finish */
if (_sd_node_recovery) (void)
#endif
/* this will provoke a sdbc_fd_attach_cd call .. */
return (rc);
}
/*
* sdbc_fd_attach_cd -- setup cache for access to raw device underlying cd.
* This is provoked by some piece of sdbc doing a reserve on the raw device.
*
* ARGUMENTS:
* cd - the cache descriptor to attach.
*
* RETURNS:
* 0 on success.
* Error otherwise.
*/
static int
{
int rc = 0;
return (EINVAL);
}
#if defined(_SD_FAULT_RES)
if (!_sd_node_recovery) {
(void) _sd_repin_cd(cd);
}
#endif
if (rc != 0) {
return (rc);
}
return (0);
}
/*
* _sdbc_io_detach_cd -- release raw device
* Called when a cache client is being detached from this cd.
*
* ARGUMENTS:
* cd - the cache descriptor to detach.
* RETURNS:
* 0 on success.
* Error otherwise.
*/
int
{
return (EINVAL);
}
#if defined(_SD_FAULT_RES)
if (_sd_node_recovery) (void)
#endif
/* relinquish responsibility for device */
return (EPROTO);
}
return (0);
}
/*
* _sdbc_detach_cd -- flush dirty writes to disk, release raw device
* Called when raw device is being detached from this cd.
*
* ARGUMENTS:
* cd - the cache descriptor to detach.
* rd_only - non-zero if detach is for read access.
* RETURNS:
* 0 on success.
* Error otherwise.
*/
static int
{
int rc;
return (EINVAL);
}
if (rc > 0) {
return (rc);
}
if (!rd_only) {
} else {
"sdbc(_sdbc_detach_cd) (%d) attached by node %d",
return (EPROTO);
}
}
return (0);
}
/*
* _sdbc_fd_detach_cd -- flush dirty writes to disk, release raw device
* Called when raw device is being detached from this cd.
*
* ARGUMENTS:
* xcd - the cache descriptor to detach.
* RETURNS:
* 0 on success.
* Error otherwise.
*/
static int
{
return (sdbc_detach_cd(xcd, 0));
}
/*
* sdbc_fd_flush_cd - raw device "xcd" is being detached and needs
* flushing. We only need to flush we don't need to hash invalidate
* this file.
*/
static int
{
}
/*
* _sd_get_pinned - re-issue PINNED callbacks for cache device
*
* ARGUMENTS:
* cd - the cache descriptor to reissue pinned calbacks from.
* RETURNS:
* 0 on success.
* Error otherwise.
*/
int
{
return (EINVAL);
}
if (!FILE_OPENED(cd)) {
return (0);
}
return (0);
}
while (cc_ent) {
if (CENTRY_PINNED(cc_ent))
BLK_FBAS);
if (!cc_ent)
}
return (0);
}
/*
* _sd_allocate_buf - allocate a vector of buffers for io.
* *This call has been replaced by _sd_alloc_buf*
*/
int *sts)
{
return (handle);
}
/*
* no 'bufvec' (data is not read by caller)
* skip leading valid or busy entries (data available sooner)
* truncate on busy block (to avoid deadlock)
* release trailing valid entries, adjust length before starting I/O.
*/
static int
{
sdbc_allocbuf_t alloc_tok = {0};
int this_entry_type = 0;
int pageio;
/* prefetch: truncate if req'd */
if (fba_len > sdbc_max_fbas)
goto done;
}
}
/*
* count number of blocks on chain that is required
*/
end_cblk_len = 0;
} else {
}
/* middle piece */
if (end_cblk_len)
stall = 0;
do {
cget:
if (centry = (_sd_cctl_t *)
try:
/* prefetch: skip leading valid blocks */
skip:
cblk++;
fba_len -= st_cblk_len;
st_cblk_off = 0;
continue;
}
if (SET_CENTRY_INUSE(centry)) {
/*
* prefetch: skip leading busy
* or truncate at busy block
*/
goto skip;
fba_orig_len -= fba_len;
fba_len = 0;
break;
}
/*
* bug 4529671
* now that we own the centry make sure that
* it is still good. it could have been processed
* by _sd_dealloc_dm() in the window between
* _sd_hash_search() and SET_CENTRY_INUSE().
*/
if ((_sd_cctl_t *)
#ifdef DEBUG
"prefetch centry %p cd %d cblk %" NSC_SZFMT
"cc_data %p",
#endif
continue;
}
/*
* Do pagelist io mutual exclusion
* before messing with the centry.
*/
/* flusher not done with pageio */
/*
* prefetch: skip leading busy
* or truncate at busy block
*/
goto skip;
fba_orig_len -= fba_len;
fba_len = 0;
break;
}
pageio = 0;
centry->cc_toflush = 0;
/* this will reset the age flag */
_sd_cctl_t *, centry);
} else {
/* block mismatch */
continue;
}
} else {
/*
* prefetch: cache is very busy. just do
* the i/o for the blocks already acquired,
* if any.
*/
fba_orig_len -= fba_len;
fba_len = 0;
/*
* if we have a chain of centry's
* then back up (set centry to lentry).
* if there is no chain (ioent == NULL)
* then centry remains NULL. this can occur
* if all previous centrys were hash hits
* on valid blocks that were processed in
* the skip logic above.
*/
if (ioent)
break;
}
/*
* dmchaining adjustment.
* if centry was obtained from the dmchain
* then clear local pageio variable because the
* centry already has cc_pageio set.
*/
if (CENTRY_PAGEIO(centry))
pageio = 0;
_sd_cctl_t *, centry);
else {
}
}
/*
* Do pagelist io mutual exclusion now if we did not do
* it above.
*/
/* flusher not done with pageio */
/*
* prefetch: skip leading busy
* or truncate at busy block
*/
goto skip;
fba_orig_len -= fba_len;
fba_len = 0;
break;
}
pageio = 0;
fba_len -= st_cblk_len;
centry)) {
} else {
int, st_cblk_len,
- 8));
}
st_cblk_off = 0;
} else {
last_ioent = centry;
else {
int, st_cblk_len,
- 8));
}
}
cblk++;
/* if this block has a new identity clear prefetch history */
if (this_entry_type != HASH_ENTRY_DM)
if (flag & NSC_METADATA)
} while (fba_len > 0);
if (locked) {
locked = 0;
}
if (centry) {
(void) _sd_free_buf(handle);
goto done;
}
}
if (ioent) {
/* prefetch: trailing valid can be released, adjust len */
if ((centry != last_ioent)) {
while (centry) {
}
}
if (sts > 0)
(void) _sd_free_buf(handle);
} else {
}
done:
if (locked)
return (sts);
}
/*
* _sd_cc_wait - wait for inuse cache block to become available
* Usage:
* if (SET_CENTRY_INUSE(centry)) {
* _sd_cc_wait(cd, blk, centry, CC_INUSE);
* goto try_again;
* }
* -or-
* if (SET_CENTRY_PAGEIO(centry)) {
* _sd_cc_wait(cd, blk, centry, CC_PAGEIO);
* goto try_again;
* }
*/
void
{
} else {
/* Oops! */
#ifdef DEBUG
#endif
return;
}
(*waiters)++;
sd_serialize();
if ((*uflag) != 0) {
(*waiters)--;
} else {
(*waiters)--;
}
} else
}
/*
* _sd_alloc_buf - Allocate a vector of buffers for io.
*
* ARGUMENTS:
* cd - Cache descriptor (from a previous open)
* fba_pos - disk position (512-byte FBAs)
* fba_len - length in disk FBAs.
* flag - allocation type. Flag is one or more of
* NSC_RDBUF, NSC_WRBUF, NSC_NOBLOCK and hints.
* NSC_RDAHEAD - prefetch for future read.
* handle_p - pointer to a handle pointer.
* If the handle pointer is non-null, its used as a
* pre-allocated handle. Else a new handle will be allocated
* and stored in *handle_p
*
* RETURNS:
* errno if return > 0.
* else NSC_HIT or NSC_DONE on success
* or NSC_PENDING on io in progress and NSC_NOBLOCK
* specified in the flag.
* USAGE:
* This routine allocates the cache blocks requested and creates a list
* of entries for this request.
* If NSC_NOBLOCK was not specified, this call could block on read io.
* If flag specified NSC_RDBUF and the request is not an entire
* hit, an io is initiated.
*/
int
{
int sts;
unsigned char cc_flag;
int this_entry_type;
int locked = 0;
sdbc_allocbuf_t alloc_tok = {0};
int min_frag = 0; /* frag statistics */
int max_frag = 0; /* frag statistics */
int nfrags = 0; /* frag statistics */
#ifdef DEBUG
int err = 0;
#endif
return (EIO);
if (xcd == NSC_ANON_CD)
cd = _CD_NOHASH;
/*
* Force large writes on nvram systems to be write-through to
* avoid the (slow) bcopy into nvram.
*/
flag |= NSC_WRTHRU;
}
}
#ifdef DEBUG
if (sdbc_pageio_debug != SDBC_PAGEIO_OFF) {
switch (sdbc_pageio_debug) {
case SDBC_PAGEIO_RDEV:
if (cd != _CD_NOHASH &&
flag |= NSC_PAGEIO;
break;
case SDBC_PAGEIO_RAND:
if ((nsc_lbolt() % 3) == 0)
flag |= NSC_PAGEIO;
break;
case SDBC_PAGEIO_ALL:
flag |= NSC_PAGEIO;
break;
}
}
#endif /* DEBUG */
locked = 1;
}
/*
* _CD_NOHASH: client wants temporary (not hashed) cache memory
* not associated with a local disk. Skip local disk checks.
*/
if (cd == _CD_NOHASH) {
goto setup;
}
locked);
goto done;
}
#if !defined(_SD_NOCHECKS)
/* prefetch: truncate if req'd */
if (fba_len > sdbc_max_fbas)
#ifdef NSC_MULTI_TERABYTE
#else
#endif
goto done;
}
}
} else
goto done;
}
#endif
if (fba_len == 0) {
goto done;
}
if (cdi->cd_recovering) {
/*
* If recovering this device, then block all allocates
* for reading or writing. If we allow reads then
* this path could see old data before we recover.
* If we allow writes then new data could be overwritten
* by old data.
* This is clearly still not a complete solution as
* the thread doing this allocate could conceivably be
* by this point (and in _sd_write/_sd_read for that matter
* which don't even have this protection). But this type
* of path seems to only exist in a failover situation
* where a device has failed on the other node and works
* on this node so the problem is not a huge one but exists
* never the less.
*/
goto done;
}
}
/* write & disk failed, return error immediately */
goto done;
}
/* CKD prefetch: bufvec not req'd, use placeholder */
}
end_cblk_len = 0;
} else
/*
* count number of blocks on chain that is required
*/
/* middle piece */
/* start piece */
/* end piece */
if (end_cblk_len)
cc_flag = 0;
cc_flag |= CC_PINNABLE;
stall = 0;
do {
cget:
if ((centry = (_sd_cctl_t *)
if (SET_CENTRY_INUSE(centry)) {
/* already inuse: wait for block, retry */
if (locked)
if (locked)
goto cget;
}
/*
* bug 4529671
* now that we own the centry make sure that
* it is still good. it could have been processed
* by _sd_dealloc_dm() in the window between
* _sd_hash_search() and SET_CENTRY_INUSE().
*/
if ((_sd_cctl_t *)
#ifdef DEBUG
"centry %p cd %d cblk %" NSC_SZFMT
#endif
goto cget;
}
/*
* Do pagelist io mutual exclusion
* before messing with the centry.
*/
/* wait for flusher to finish pageio */
if (locked)
if (locked)
goto cget;
}
pageio = 0;
centry->cc_toflush = 0;
/* this will reset the age flag */
_sd_cctl_t *, centry);
} else {
/* block mismatch: release, alloc new block */
goto cget;
}
} else {
/*
* dmchaining adjustment.
* if centry was obtained from the dmchain
* then clear local pageio variable because the
* centry already has cc_pageio set.
*/
if (CENTRY_PAGEIO(centry))
pageio = 0;
_sd_cctl_t *, centry);
else {
}
}
/*
* Do pagelist io mutual exclusion now if we did not do
* it above.
*/
/* wait for flusher to finish pageio */
if (locked)
if (locked)
goto cget;
}
pageio = 0;
if (CENTRY_DIRTY(centry)) {
/*
* end action might set PEND_DIRTY flag
* must lock if need to change flag bits
*/
/* was FAST */
/* was FAST */
}
} else
/*
* step 0:check valid bits in each cache ele as
* instance of invalid data
*/
fba_len -= st_cblk_len;
centry)) {
} else {
+ st_cblk_off),
int, st_cblk_len,
char *, *(int64_t *)
char *, *(int64_t *)
st_cblk_len) - 8));
}
}
cblk++;
fba_len -= end_cblk_len;
if (!SDBC_VALID_BITS(0, end_cblk_len,
centry)) {
} else {
int, end_cblk_len,
char *,
*(int64_t *)
char *,
*(int64_t *)
- 8));
}
}
}
} else {
if (!FULLY_VALID(centry)) {
} else {
BLK_FBAS);
(uint64_t)
int, BLK_FBAS,
char *,
*(int64_t *)
char *,
*(int64_t *)
8));
}
}
}
cblk++;
}
/* if this block has a new identity clear prefetch history */
if (this_entry_type != HASH_ENTRY_DM)
if (flag & NSC_METADATA)
} while (fba_len);
if (locked) {
locked = 0;
}
ASSERT(dmchain_request_blocks == 0);
/*
* do any necessary cleanup now that all the blocks are allocated.
*/
/* be sure you nul term. the chain */
/*
* between the centry ele in the list and calc the alloc size
* (fill in CATAGORY based on TYPE and immediate neighbors)
*/
#ifdef DEBUG
if (err) {
}
#else
(void) _sd_free_buf(handle);
#endif
goto done;
}
/*
* step two: alloc the needed mem and fill in the data and chaining
* fields (leave bufvec for step three)
*/
/*
* step three: do the bufvec
*/
while (centry) {
_sd_cctl_t *, centry,
int, cd,
if (fba_len == fba_orig_len) {
bufvec++;
fba_len -= st_cblk_len;
/* contiguous */
} else {
bufvec++;
}
fba_len -= end_cblk_len;
} else {
/* contiguous */
} else {
bufvec++;
}
}
}
/* be sure you nul term. the chain */
bufvec->bufvmeaddr = 0;
/* frag statistics */
{
++tbufvec) {
}
}
/* buffer memory frag stats */
int, max_frag);
if (_SD_IS_WRTHRU(handle))
goto alloc_done;
} else {
}
}
}
if (locked) {
locked = 0;
}
if (ioent) {
if (sts > 0)
(void) _sd_free_buf(handle);
} else
} else
done:
if (locked)
return (sts);
}
/*
* consistency checking for ccents
*/
#define ELIGIBLE(p) (p & ELIGIBLE_ENTRY_DM)
#define HOLD(p) (p & HOLD_ENTRY_DM)
#define HASHE(p) (p & HASH_ENTRY_DM)
#define HOST(p) (p & HOST_ENTRY_DM)
#define PARA(p) (p & PARASITIC_ENTRY_DM)
#define OTHER(p) \
#define AVAIL(p) (p & AVAIL_ENTRY_DM)
/*
* sdbc_check_cctl_cot -- consistency check for _sd_setup_category_on_type()
* may only be called on entry to state machine (when ccent is either
* ELIGIBLE_ENTRY_DM, HOLD_ENTRY_DM or HASH_ENTRY_DM).
*
* print message or panic (DEBUG) if inconsistency detected.
*/
static int
{
int size;
int host_or_other;
int para;
int ccent_ok = 1;
/*
* on entry to _sd_setup_category_on_type(),
* one of three mutually exclusive entry field bits must be set
*/
case ELIGIBLE_ENTRY_DM:
case HOLD_ENTRY_DM:
case HASH_ENTRY_DM:
/* ok */
break;
default:
/* zero or multiple flag bits */
ccent_ok = 0;
break;
}
/* categories are mutually exclusive */
ccent_ok = 0;
/* these bits should be cleared out (STICKY_METADATA_DM not used) */
ccent_ok = 0;
/* eligible has no data and no size */
ccent_ok = 0;
/* parasite has zero size and non-zero data */
ccent_ok = 0;
/* host has non-zero size and non-zero data */
ccent_ok = 0;
/* "other" is just like a host */
ccent_ok = 0;
/* a HOLD or a HASH must have a size */
ccent_ok = 0;
if (!ccent_ok)
"sdbc(sdbc_check_cctl_cot): inconsistent ccent %p "
(void *)data);
return (ccent_ok);
}
/*
* sdbc_mark_cctl_cot -- mark cctls bad and invalidate when
* inconsistency found in _sd_setup_category_on_type()
* returns nothing
*
* Note: this is an error recovery path that is triggered when an
* inconsistency in a cctl is detected. _sd_centry_release() will take
* these cache entries out of circulation and place them on a separate list
* for debugging purposes.
*/
void
{
/* the entire chain is guilty by association */
while (cur_ent) {
}
}
/*
* _sd_setup_category_on_type(_sd_cctl_t *) - Setup the centry CATEGORY based on
* centry TYPE and immediate neighbors. Identify each eligible (ie not HASH)
* them and parasites are chained to the host and point to page offsets within
* the host's memory.
*
* RETURNS:
* 0 on success, EINTR if inconsistency detected in centry
*
* Note:
* none
*/
static int
{
int cl;
int ret = 0;
if (sdbc_use_dmchain)
else {
/* pickup a fresh copy - has the world changed */
}
prev_ent = 0;
current_pest_count = 0;
cl = 2;
/* try to recover from bad cctl */
switch (cl) {
next_ent = 0;
cl = 0;
if (centry) {
if (sdbc_check_cot &&
break;
}
cl = 2;
}
break;
case (2): /* vector to appropriate routine */
cl = 5;
cl = 15;
else
cl = 10;
break;
case (5): /* process NON-ELIGIBLE entries */
if (!(centry->cc_aging_dm &
(HASH_ENTRY_DM|HOLD_ENTRY_DM))) {
/* no catagory */
/* consistency check */
if (centry->cc_alloc_size_dm ||
"sdbc(setup_cot): "
(void *)centry);
break;
}
centry->cc_aging_dm &=
_sd_cctl_t *, centry);
}
cl = 1;
break;
/*
* no prev entry (ie top of list) or no prev
* ELIGIBLE entry
*/
case (10):
/*
* this is an eligible entry, does it start
* a list or is it a loner
*/
/* consistency check */
if (centry->cc_alloc_size_dm ||
(void *)centry);
break;
}
/* it starts a list */
/* host catagory */
/* start out with one page */
_sd_cctl_t *, anchor);
cl = 1;
} else {
/*
* it's a loner
* drop status to no category and
* restart
*/
cl = 2;
centry->cc_aging_dm &=
}
break;
case (15): /* default to parasite catagory */
/* consistency check */
if (centry->cc_alloc_size_dm ||
(void *)centry);
break;
}
/* continue to grow the pest list */
centry->cc_aging_dm |=
/*
* offset of host ent mem this will pt
* to
*/
/*
* up the host mem req by one for
* this parasite
*/
_sd_cctl_t *, centry);
cl = 1;
} else {
/*
* term this pest list - restart fresh
* on this entry
*/
current_pest_count = 0;
prev_ent->cc_aging_dm &=
cl = 2;
}
break;
} /* switch(cl) */
} /* while (cl) */
if (ret != 0)
return (ret);
}
/*
* _sd_setup_mem_chaining(_sd_cctl_t *) - Allocate memory, setup
* sd_setup_category_on_type().
*
* RETURNS:
* 0 on success
* non-zero on error
*
* Note:
* if called with ALLOC_NOWAIT, caller must check for non-zero return
*/
static int
{
if (!header)
return (0);
prev_ent = 0;
cl = 2;
while (cl) {
switch (cl) {
next_ent = 0;
cl = 0;
if (centry) {
cl = 2;
}
break;
case (2): /* vector to appropriate routine */
cl = 10;
else if (centry->cc_aging_dm &
cl = 15;
else
cl = 5;
break;
case (5): /* OTHER processing - alloc mem */
/* The allocation failed */
cl = 0;
else
cl = 1;
break;
/*
* HOST entry processing - save the anchor pt,
* alloc the memory,
*/
case (10): /* setup head and nxt ptrs */
/* The allocation failed */
cl = 0;
else
cl = 1;
break;
/*
*/
case (15):
/*
* fudge the data mem ptr to an offset from
* the anchor alloc
*/
if (!(centry->cc_aging_dm &
(HASH_ENTRY_DM| HOLD_ENTRY_DM))) {
/* chain prev to this */
/*
* generate the actual data ptr into
* host entry memory
*/
centry->cc_alloc_size_dm = 0;
}
cl = 1;
break;
} /* switch(cl) */
} /* while (cl) */
return (rc);
}
/*
* _sd_check_buffer_alloc - Check if buffer allocation is invalid.
*
* RETURNS:
* 0 if its ok to continue with allocation.
* Else errno to be returned to the user.
*
* Note:
* This routine could block if the device is not local and
* recovery is in progress.
*/
/* ARGSUSED */
static int
{
/*
* This check exists to ensure that someone will not pass in an
* arbitrary pointer and try to pass it off as a handle.
*/
"cd %d invalid handle %p flags %x",
return (EINVAL);
}
"cd %d not open. Cache init %d",
return (EINVAL);
}
"sdbc(_sd_check_buffer_alloc) cd %d is not attached", cd);
return (EINVAL);
}
return (0);
}
/*
* sdbc_check_handle -- check that handle is valid
* return 1 if ok, 0 otherwise (if debug then panic).
*/
static int
{
int ret = 1;
if (!_SD_HANDLE_ACTIVE(handle)) {
ret = 0;
}
return (ret);
}
/*
* _sd_free_buf - Free the buffers allocated in _sd_alloc_buf.
*
* ARGUMENTS:
* handle - The handle allocated in _sd_alloc_buf.
*
* RETURNS:
* 0 on success.
* Else errno.
*
* NOTE:
* If handle was allocated through _sd_alloc_buf, the handle allocated
* flag (NSC_HALLOCATED) will be reset by _sd_alloc_buf. This indicates
* that _sd_free_buf should free up the handle as well.
* All other handles directly allocated from _sd_alloc_handle will have
* that flag set. Any handle with valid blocks will have the handle
* active flag. It is an error if the active flag is not set.
* (if free_buf were called without going through alloc_buf)
*/
int
{
if (sdbc_check_handle(handle) == 0)
return (EINVAL);
/*
* Data in this handle will be a mix of data from the
* source device and data from another device, so
* invalidate all the blocks.
*/
while (centry) {
}
}
}
while (centry) {
}
/*
* help prevent dup call to _sd_centry_release if this handle
* is erroneously _sd_free_buf'd twice. (should not happen).
*/
(void) _sd_free_handle(handle);
} else {
}
return (0);
}
static int _sd_lruq_srch = 0x2000;
/*
* sdbc_get_dmchain -- get a candidate centry chain pointing to
* contiguous memory
* ARGUMENTS:
* cblocks - number of cache blocks requested
* stall - pointer to stall count (no blocks avail)
* flag - ALLOC_NOWAIT flag
*
* RETURNS:
* a cache entry or possible NULL if ALLOC_NOWAIT set
* USAGE:
* attempt to satisfy entire request from queue
* that has no memory allocated.
* if this fails then attempt a partial allocation
* with a preallocated block of requested size up to
* max_dyn_list.
* then look for largest chain less than max_dyn_list.
*/
static _sd_cctl_t *
{
_sd_queue_t *q;
int num_tries;
int cblocks_orig = cblocks;
int i;
while (!cc_dmchain) {
/* get it from the os if possible */
q = &sdbc_dm_queues[0];
mutex_enter(&q->sq_qlock);
/*
* set the inuse and pageio bits
* Note: this code expects the cc_ent to
* be available. no other thread may set the
* inuse or pageio bit for an entry on the
* 0 queue.
*/
for (i = 0; i < cblocks; ++i) {
if (SET_CENTRY_INUSE(cc_ent)) {
"centry inuse on 0 q! %p",
(void *)cc_ent);
}
if (SET_CENTRY_PAGEIO(cc_ent)) {
"centry pageio on 0 q! %p",
(void *)cc_ent);
}
}
/* got a dmchain */
/* remove this chain from the 0 queue */
ASSERT(GOOD_LRUSIZE(q));
}
mutex_exit(&q->sq_qlock);
if (cc_dmchain)
continue;
}
/* look for a pre-allocated block of the requested size */
q = &sdbc_dm_queues[cblocks];
if (q->sq_inq != 0) {
mutex_enter(&q->sq_qlock);
/*
* get a dmchain
* set the inuse and pageio bits
*/
if (sdbc_dmchain_avail(tmp_dmchain)) {
/* put on MRU end of queue */
1, 0);
break;
}
}
mutex_exit(&q->sq_qlock);
if (cc_dmchain)
continue;
}
/*
* spin block
* nudge the deallocator, accelerate ageing
*/
if (nowait)
break;
if (!(--num_tries)) {
(void) (*stall)++;
} else { /* see if smaller request size is available */
if (!(--cblocks))
}
} /* while (!cc_dmchain) */
return (cc_dmchain);
}
static int
{
int chain_avail = 1;
while (cc_ent) {
chain_avail = 0;
break;
}
if (CENTRY_DIRTY(cc_ent)) {
chain_avail = 0;
break;
}
if (SET_CENTRY_INUSE(cc_ent)) {
chain_avail = 0;
break;
}
if ((SET_CENTRY_PAGEIO(cc_ent))) {
chain_avail = 0;
break;
}
if (CENTRY_DIRTY(cc_ent)) {
chain_avail = 0;
break;
}
cc_ent->cc_toflush = 0;
}
if (!chain_avail)
else {
/*
* prevent possible deadlocks in _sd_cc_wait():
* remove from hash and wakeup any waiters now that we
* have acquired the chain.
*/
while (cc_ent) {
if (cc_ent->cc_await_use) {
}
}
}
return (chain_avail);
}
static void
{
while (cc_ent != cc_ent_end) {
}
}
/*
* put a dmchain on the LRU end of a queue
*/
void
{
mutex_enter(&q->sq_qlock);
q->sq_inq++;
ASSERT(GOOD_LRUSIZE(q));
mutex_exit(&q->sq_qlock);
}
/*
* put a dmchain on the MRU end of a queue
*/
static void
{
mutex_enter(&q->sq_qlock);
q->sq_inq++;
ASSERT(GOOD_LRUSIZE(q));
mutex_exit(&q->sq_qlock);
}
/*
* remove dmchain from a queue
*/
void
{
mutex_enter(&q->sq_qlock);
q->sq_inq--;
ASSERT(GOOD_LRUSIZE(q));
mutex_exit(&q->sq_qlock);
}
/*
* requeue a dmchain to the MRU end of its queue.
* if getlock is 0 on entry the queue lock (sq_qlock) must be held
*/
void
int getlock)
{
if (getlock)
mutex_enter(&q->sq_qlock);
/* inline of sdbc_remq_dmchain() */
if (mru) { /* put on MRU end of queue */
/* inline of sdbc_ins_dmqueue_back */
(q->sq_req_stat)++;
} else { /* put on LRU end of queue i.e. requeue to head */
/* inline of sdbc_ins_dmqueue_front */
/*
* clear the CC_QHEAD bit on all members of the chain
*/
{
}
}
if (getlock)
mutex_exit(&q->sq_qlock);
}
/*
* sdbc_dmchain_dirty(cc_ent)
* return first dirty cc_ent in dmchain, NULL if chain is not dirty
*/
static _sd_cctl_t *
{
if (CENTRY_DIRTY(cc_ent))
break;
return (cc_ent);
}
/*
* sdbc_requeue_head_dm_try()
* attempt to requeue a dmchain to the head of the queue
*/
void
{
int qidx;
_sd_queue_t *q;
if (!sdbc_dmchain_dirty(cc_ent)) {
q = &sdbc_dm_queues[qidx];
}
}
/*
* sdbc_centry_alloc_blks -- allocate cache entries with memory
*
* ARGUMENTS:
* cd - Cache descriptor (from a previous open)
* cblk - cache block number.
* reqblks - number of cache blocks to be allocated
* flag - can be ALLOC_NOWAIT
* RETURNS:
* A cache block chain or NULL if ALLOC_NOWAIT and request fails
*
* Note: caller must check for null return if called with
* ALLOC_NOWAIT set.
*/
{
int stall = 0;
while (reqblks) {
if (!centry)
break;
else
else
else
--reqblks;
}
while (centry) {
}
} else
/* This is where the memory is actually allocated */
return (anchor);
}
/*
* sdbc_centry_alloc - sdbc internal function to allocate a new cache block.
*
* ARGUMENTS:
* cd - Cache descriptor (from a previous open)
* cblk - cache block number.
* stall - pointer to stall count (no blocks avail)
* req_blocks - number of cache blocks remaining in caller's i/o request
* alloc_tok - pointer to token initialized to 0 on first call to function
* flag - lock status of sdbc_queue_lock or ALLOC_NOWAIT flag
* RETURNS:
* A cache block, or possibly NULL if ALLOC_NOWAIT set .
*
* USAGE:
* switch to the appropriate allocation function.
* this function is used when callers need more than one cache block.
* it is called repeatedly until the entire request is satisfied,
* at which time the caller will then do the memory allocation.
* if only one cache block is needed callers may use
* sdbc_centry_alloc_blks() which also allocates memory.
*
* Note: caller must check for null return if called with
* ALLOC_NOWAIT set.
*/
{
if (sdbc_use_dmchain)
flag);
else
return (centry);
}
/*
* sdbc_alloc_dmc -- allocate a centry from a dmchain
*
* ARGUMENTS:
* cd - Cache descriptor (from a previous open)
* cblk - cache block number.
* stall - pointer to stall count (no blocks avail)
* req_blocks - number of cache blocks in clients i/o request
* alloc_tok - pointer to token initialized to 0 on first call to function
* flag - lock status of sdbc_queue_lock, or ALLOC_NOWAIT flag
* RETURNS:
* A cache block or possibly NULL if ALLOC_NOWAIT set
*
* USAGE:
* if dmchain is empty, allocate one.
*/
static _sd_cctl_t *
{
if (!dmc->sab_dmchain) {
/*
* Note - sdbc_get_dmchain() returns
* with cc_inuse and cc_pageio set
* for all members of dmchain.
*/
if (dmc->sab_dmchain =
/* remember q it came from */
}
}
/*
* Note: dmchain pointer is advanced in sdbc_alloc_from_dmchain()
*/
return (centry);
}
/*
* sdbc_alloc_from_dmchain -- allocate centry from a dmchain of centrys
*
* ARGUMENTS:
* cd - Cache descriptor (from a previous open)
* cblk - cache block number.
* alloc_tok - pointer to token
* flag - lock status of sdbc_queue_lock or ALLOC_NOWAIT
*
* RETURNS:
* A cache block or possibly NULL if ALLOC_NOWAIT set.
*
* USAGE:
* This routine allocates a new cache block from the supplied dmchain.
* Assumes that dmchain is non-NULL and that all cache entries in
* the dmchain have been removed from hash and have their cc_inuse and
* cc_pageio bits set.
*/
static _sd_cctl_t *
int flag)
{
int categorize_centry;
categorize_centry = 0;
if (cd == _CD_NOHASH)
else if ((old_ent = (_sd_cctl_t *)
_sd_htable)) != cc_ent) {
if (SET_CENTRY_INUSE(old_ent)) {
if (nowait) {
goto out;
}
if (locked)
if (locked)
goto alloc_try;
}
/*
* bug 4529671
* now that we own the centry make sure that
* it is still good. it could have been processed
* by _sd_dealloc_dm() in the window between
* _sd_hash_insert() and SET_CENTRY_INUSE().
*/
!= old_ent) {
#ifdef DEBUG
" lost to dealloc?! cc_data %p", (void *)old_ent,
#endif
if (nowait) {
goto out;
}
goto alloc_try;
}
old_ent->cc_toflush = 0;
/* _sd_centry_release(cc_ent); */
} else {
if (nowait) {
goto out;
}
goto alloc_try;
}
}
/*
* advance the dmchain pointer, but only if we got the
* cc_ent from the dmchain
*/
if (categorize_centry != FOUND_IN_HASH_DM) {
else
}
BLK_TO_FBA_NUM(cblk), 0, 0);
if (cc_ent->cc_await_use) {
}
out:
return (cc_ent);
}
/*
* sdbc_centry_alloc_end -- tidy up after all cache blocks have been
* allocated for a request
* ARGUMENTS:
* alloc_tok - pointer to allocation token
* RETURNS
* nothing
* USAGE:
* at this time only useful when sdbc_use_dmchain is true.
* if there are cache blocks remaining on the chain then the inuse and
* pageio bits must be cleared (they were set in sdbc_get_dmchain().
*
*/
static void
{
_sd_queue_t *q;
#ifdef DEBUG
int chainpull = 0;
#endif
if (!sdbc_use_dmchain)
return;
while (next_centry != NULL) {
if (next_centry->cc_data) {
#ifdef DEBUG
++chainpull;
#endif
/* clear bit after final reference */
} else {
/*
* a floater from the 0 queue, insert on q.
*
* since this centry is not on any queue
* the inuse bit can be cleared before
* inserting on the q. this is also required
* since sdbc_get_dmchain() does not expect
* inuse bits to be set on 0 queue entry's.
*/
q = &sdbc_dm_queues[0];
}
}
#ifdef DEBUG
/* compute wastage stats */
if (chainpull)
max_dm_queues + chainpull)))++;
#endif
}
/*
* sdbc_alloc_lru - allocate a new cache block from the lru queue
*
* ARGUMENTS:
* cd - Cache descriptor (from a previous open)
* cblk - cache block number.
* stall - pointer to stall count (no blocks avail)
* flag - lock status of sdbc_queue_lock or ALLOC_NOWAIT
*
* RETURNS:
* A cache block or NULL if ALLOC_NOWAIT specified
*
* USAGE:
* This routine allocates a new cache block from the lru.
* If an allocation cannot be done, we block, unless ALLOC_NOWAIT is set.
*/
static _sd_cctl_t *
{
_sd_queue_t *q = _SD_LRU_Q;
int categorize_centry;
if (nowait) {
if (num_tries <= 0) /* ensure num_tries is non-zero */
} else
if (--num_tries <= 0)
if (nowait) {
goto out;
} else
break;
continue;
if (CENTRY_DIRTY(cc_ent))
continue;
if (SET_CENTRY_INUSE(cc_ent))
continue;
if (CENTRY_DIRTY(cc_ent)) {
continue;
}
cc_ent->cc_toflush = 0;
/*
* Inlined requeue of the LRU. (should match _sd_requeue)
*/
/* was FAST */
mutex_enter(&q->sq_qlock);
#if defined(_SD_DEBUG)
if (1) {
if (!_sd_cctl_valid(cc_ent) ||
!_sd_cctl_valid(qp))
"_sd_centry_alloc %x prev %x next %x qp %x",
}
#endif
/* was FAST */
mutex_exit(&q->sq_qlock);
/*
* End inlined requeue.
*/
#if defined(_SD_STATS)
#else
#if defined(_SD_DEBUG)
if (cc_ent->cc_await_use ||
BLK_FBAS);
}
#else
#endif
#endif
categorize_centry = 0;
if (cd == _CD_NOHASH)
else if ((old_ent = (_sd_cctl_t *)
_sd_htable)) != cc_ent) {
if (SET_CENTRY_INUSE(old_ent)) {
if (nowait) {
goto out;
}
if (locked)
if (locked)
goto alloc_try;
}
/*
* bug 4529671
* now that we own the centry make sure that
* it is still good. it could have been processed
* by _sd_dealloc_dm() in the window between
* _sd_hash_insert() and SET_CENTRY_INUSE().
*/
if ((_sd_cctl_t *)
#ifdef DEBUG
NSC_SZFMT " lost to dealloc?! cc_data %p",
#endif
if (nowait) {
goto out;
}
goto alloc_try;
}
old_ent->cc_toflush = 0;
} else {
if (nowait) {
goto out;
}
goto alloc_try;
}
}
BLK_TO_FBA_NUM(cblk), 0, 0);
if (cc_ent->cc_await_use) {
}
out:
return (cc_ent);
}
(void) (*stall)++;
goto retry_alloc_centry;
}
/*
* sdbc_centry_init_dm - setup the cache block for dynamic memory allocation
*
* ARGUMENTS:
* centry - Cache block.
*
* RETURNS:
* NONE
*
* USAGE:
* This routine is the central point in which cache entry blocks are setup
*/
static void
{
/* an entry already setup - don't touch simply refresh age */
return;
}
"non-zero mem chain in ccent %p", (void *)centry);
centry->cc_head_dm = 0;
if (!sdbc_use_dmchain)
centry->cc_next_dm = 0;
}
/*
* sdbc_centry_memalloc_dm
*
* Actually allocate the cache memory, storing it in the cc_data field for
* the cctl
*
* ARGS:
* centry: cache control block for which to allocate the memory
* alloc_request: number of bytes to allocate
* flag: if called with ALLOC_NOWAIT, caller must check for non-zero return
*
* RETURNS:
* 0 on success
* non-zero on error
*/
static int
{
int cblocks;
int sleep;
/* host or other */
if (sdbc_use_dmchain) {
/* set the dmqueue index */
/* put on appropriate queue */
}
/*
* for KM_NOSLEEP (should never happen with KM_SLEEP)
*/
return (LOW_RESOURCES_DM);
centry->cc_alloc_ct_dm++;
}
return (0);
}
/*
* _sd_centry_release - release a cache block
*
* ARGUMENTS:
* centry - Cache block.
*
* RETURNS:
* NONE
*
* USAGE:
* This routine frees up a cache block. It also frees up a write
* block if allocated and its valid to release it.
*/
void
{
/* was FAST */
if (CENTRY_DIRTY(centry))
else {
}
/* was FAST */
if (wctl) {
}
}
if (sdbc_use_dmchain) {
if (centry->cc_alloc_size_dm) {
/* see if this can be queued to head */
if (CENTRY_QHEAD(centry)) {
} else {
int qidx;
_sd_queue_t *q;
q = &sdbc_dm_queues[qidx];
if (_sd_lru_reinsert(q, centry)) {
}
}
} else {
/*
* Fix for bug 4949134:
* If an internal block is marked with CC_QHEAD
* but the HOST block is not, the chain will
* never age properly, and will never be made
* available. Only the HOST of the dmchain is
* checked for CC_QHEAD, so clearing an internal
* block indiscriminately (as is being done
* here) does no damage.
*
* The same result could instead be achieved by
* not setting the CC_QHEAD flag in the first
* place, if the block is an internal dmchain
* block, and if it is found in the hash table.
* The current solution was chosen since it is
* the least intrusive.
*/
}
} else {
if (CENTRY_QHEAD(centry)) {
if (!CENTRY_DIRTY(centry))
}
}
/* only clear inuse after final reference to centry */
}
/*
* lookup to centry info associated with safestore resource
* return pointer to the centry info structure
*/
{
int found = 0;
return (NULL);
sizeof (ss_centry_info_t)) - 1;
++found;
break;
}
if (!found)
return (cinfo);
}
/*
* _sd_alloc_write - Allocate a write block (for remote mirroring)
* and set centry->cc_write
*
* ARGUMENTS:
* centry - Head of Cache chain
* stall - pointer to stall count (no blocks avail)
*
* RETURNS:
* 0 - and sets cc_write for all entries when write contl block obtained.
* -1 - if a write control block could not be obtained.
*/
int
{
_sd_cctl_t *ce;
int err;
int need;
need = 0;
need++;
}
if (!need)
return (0);
== SS_OK) {
continue;
/*
* this is bad and should not happen.
* we use the saved reslist to cleanup
* and return.
*/
"bad resource list 0x%p"
"changing to forced write thru mode",
(void *)savereslist);
(void) _sd_set_node_hint(NSC_FORCED_WRTHRU);
while (SSOP_GETRESOURCE(sdbc_safestore,
res);
}
return (-1);
}
}
return (0);
}
/* no safestore resources available. do sync write */
return (-1);
}
/*
* _sd_read - Interface call to do read.
*
* ARGUMENTS:
* handle - handle allocated earlier on.
* fba_pos - disk block number to read from.
* fba_len - length in fbas.
* flag - flag: (NSC_NOBLOCK for async io)
*
* RETURNS:
* errno if return > 0
* NSC_DONE or NSC_PENDING otherwise.
*
* USAGE:
* This routine checks if the request is valid and calls the underlying
* doread routine (also called by alloc_buf)
*/
int
int flag)
{
int ret;
goto out;
}
#if !defined(_SD_NOCHECKS)
if (!_SD_HANDLE_ACTIVE(handle)) {
(void *)handle);
goto out;
}
#endif
if (fba_len == 0) {
goto out;
}
end_cblk_len = 0;
} else {
}
goto need_io;
char *,
char *,
fba_pos += st_cblk_len;
fba_len -= st_cblk_len;
if (!FULLY_VALID(cc_ent))
goto need_io;
}
if (fba_len) {
goto need_io;
}
goto stats_exit;
out:
return (ret);
}
/*
* sdbc_doread_prefetch - read ahead one cache block
*
* ARGUMENTS:
* cc_ent - cache entry
* fba_pos - disk block number to read from
* fba_len - length in fbas.
*
* RETURNS:
* number of fbas, if any, that are to be read beyond (fba_pos + fba_len)
*
* USAGE:
* if readahead is to be done allocate a cache block and place
* on the cc_chain of cc_ent
*/
static int
{
int fba_count = 0; /* number of fbas to prefetch */
/* readahead only for small reads */
(vol_fill > 0)) {
/*
* if prev block is in cache and next block is not,
* then read ahead one block
*/
if (cc_ra) {
/* if in cache don't readahead */
++sdbc_ra_hash;
} else {
/*
* indicate implicit prefetch and
* mark for release in
* _sd_read_complete()
*/
}
} else
++sdbc_ra_none;
}
}
}
return (fba_count);
}
/*
* _sd_doread - Check if blocks in cache. If not completely true, do io.
*
* ARGUMENTS:
* handle - handle allocated earlier on.
* fba_pos - disk block number to read from.
* fba_len - length in fbas.
* flag - flag: (NSC_NOBLOCK for async io)
*
* RETURNS:
* errno if return > 0
* NSC_DONE(from disk), or NSC_PENDING otherwise.
*
* Comments:
* It initiates an io and either blocks waiting for the completion
* or return NSC_PENDING, depending on whether the flag bit
* NSC_NOBLOCK is reset or set.
*
*/
static int
{
int num_bdl;
unsigned int want_bits;
return (EIO);
}
/*
* adjust the position and length so that the entire cache
* block is read in
*/
/* first, adjust to beginning of cache block */
/* compute fill to end of cache block */
/* fill to lesser of cache block or end of volume */
/* for small reads do 1-block readahead if previous block is in cache */
if (sdbc_prefetch1)
end_cblk_len = 0;
} else {
}
num_bdl = 0;
while (cc_temp) {
}
return (E2BIG);
}
else {
}
fba_len -= st_cblk_len;
if (CENTRY_DIRTY(cc_ent))
else {
}
}
if (fba_len) {
else {
}
}
if (err != NSC_PENDING) {
}
return (err);
}
/*
* _sd_read_complete - Do whatever is necessary after a read io is done.
*
* ARGUMENTS:
* handle - handle allocated earlier on.
* fba_pos - disk block number to read from.
* fba_len - length in fbas.
* error - error from io if any.
*
* RETURNS:
* NONE.
*
* Comments:
* This routine marks the cache blocks valid if the io completed
* sucessfully. Called from the async end action as well as after
* a synchrnous read completes.
*/
void
{
end_cblk_len = 0;
} else {
}
int, st_cblk_len,
char *, *(int64_t *)
char *, *(int64_t *)
int, BLK_FBAS,
char *,
char *,
/*
* 4755485 release implicit prefetch buffers
*
* the cc_chain of the first buffer must NULL'd
* else _sd_free_buf() will do a double free when
* it traverses the chain.
*
* if a buffer has been marked PREFETCH_BUF_IR then
* it is guaranteed that
* 1. it is the second in a chain of two.
* 2. cur_fba_len is BLK_FBAS.
* 3. end_cblk_len is zero.
*
* because of 1 (and 2) above, we can safely exit the
* while loop via the break statement without
* executing the last two statements. the break
* statement is necessary because it would be unsafe
* to access cc_iocent which could be reallocated
* immediately after the _sd_centry_release().
*/
break;
}
cur_fba_len -= BLK_FBAS;
}
if (end_cblk_len) {
int, end_cblk_len,
char *,
char *,
}
}
}
/*
* _sd_async_read_ea - End action for async reads.
*
* ARGUMENTS:
* xhandle - handle allocated earlier on (cast to blind_t).
* fba_pos - disk block number read from.
* fba_len - length in fbas.
* error - error from io if any.
*
* RETURNS:
* NONE.
*
* Comments:
* This routine is called at interrupt level when the io is done.
* This is called only when read is asynchronous (NSC_NOBLOCK)
*/
static void
int error)
{
int cd;
if (error) {
}
#if defined(_SD_DEBUG_PATTERN)
#endif
}
/*
* _sd_async_write_ea - End action for async writes.
*
* ARGUMENTS:
* xhandle - handle allocated earlier on. (cast to blind_t)
* fba_pos - disk block number written to.
* fba_len - length in fbas.
* error - error from io if any.
*
* RETURNS:
* NONE.
*
* Comments:
* This routine is called at interrupt level when the write io is done.
* This is called only when we are in write-through mode and the write
* call indicated asynchronous callback. (NSC_NOBLOCK)
*/
/* ARGSUSED */
static void
int error)
{
if (error)
}
/*
* update_dirty - set dirty bits in cache block which is already dirty
* cc_inuse is held, need cc_lock to avoid race with _sd_process_pending
* must check for I/O in-progress and set PEND_DIRTY.
* return previous dirty bits
* [if set _sd_process_pending will re-issue]
*/
static _sd_bitmap_t
{
/* was FAST */
if (old) {
/*
* If we are writing to an FBA that is still marked dirty,
* record a write cancellation.
*/
}
/* This is a write to a block that was already dirty */
sd_serialize();
if (CENTRY_IO_INPROGRESS(cc_ent))
}
/* was FAST */
return (old);
}
/*
* _sd_write - Interface call to commit part of handle.
*
* ARGUMENTS:
* handle - handle allocated earlier o.
* fba_pos - disk block number to write to.
* fba_len - length in fbas.
* flag - (NSC_NOBLOCK | NSC_WRTHRU)
*
* RETURNS:
* errno if return > 0
* NSC_HIT (in cache), NSC_DONE (to disk) or NSC_PENDING otherwise.
*
* Comments:
* This routine checks validity of the handle and then calls the
* sync-write function if this write is determined to be write-through.
* Else, it reflects the data to the write blocks on the mirror node,
* (allocated in alloc_buf). If the cache block is not dirty, it is
* marked dirty and queued up for io processing later on.
* If parts are already dirty but io is not in progress yet, it is
* marked dirty and left alone (it is already in the queue)
* If parts are already dirty but io is in progress, it is marked
* dirty and also a flag is set indicating that this buffer should
* be reprocessed after the io-end-action.
* Attempt is made to coalesce multiple writes into a single list
* for io processing later on.
*
* Issuing of writes may be delayed until the handle is released;
* _sd_queue_write() sets NSC_QUEUE, indicating that dirty bits
* and reflection to mirror have already been done, just queue I/O.
*/
int
int flag)
{
if (_sdbc_shutdown_in_progress) {
goto out;
}
if (!_SD_HANDLE_ACTIVE(handle)) {
goto out;
}
#if !defined(_SD_NOCHECKS)
goto out;
}
#endif
if (fba_len == 0) {
goto out;
}
/*
* store_only: don't queue this I/O yet
* queue_only: queue I/O to disk, don't store in mirror node
*/
else
else
queue_only = store_only = 0;
goto out;
}
#if defined(_SD_DEBUG_PATTERN)
#endif
flag |= NSC_WRTHRU;
goto stats_exit;
}
if (store_only) /* enqueue in _sd_free_buf() */
end_cblk_len = 0;
} else {
}
goto loop1;
if (store_only) {
goto loop1;
}
num_queued = 1;
int, st_cblk_len,
char *,
if (cur_chain) {
}
goto loop2;
}
if (store_only) {
goto loop2;
}
if (dirty_next) {
dirty_next = cc_ent;
num_queued++;
} else {
num_queued = 1;
}
int, BLK_FBAS,
cur_fba_len -= BLK_FBAS;
}
#if defined(_SD_DEBUG)
if (cur_fba_len != end_cblk_len)
#endif
if (cur_fba_len) {
end_cblk_len)) {
if (cur_chain) {
}
goto loop3;
}
if (store_only) {
goto loop3;
}
if (dirty_next) {
dirty_next = cc_ent;
num_queued++;
} else {
num_queued = 1;
}
}
if (cur_fba_len) {
int, end_cblk_len,
}
if (!store_only && cur_chain) {
}
if (!queue_only) {
}
out:
return (ret);
}
/*
* _sd_queue_write(handle, fba_pos, fba_len): Queues delayed writes for
* flushing
*
* ARGUMENTS: handle - handle allocated with NSC_WRBUF
* fba_pos - starting fba pos from _sd_alloc_buf()
* fba_len - fba len from _sd_alloc_buf()
*
* USAGE : Called if _SD_DELAY_QUEUE is set. Finds all blocks in the
* handle marked for flushing and queues them to be written in
* optimized (i.e. sequential) order
*/
static void
{
int flush_pos_valid = 0;
nsc_size_t flush_len = 0;
cc_ent->cc_toflush = 0;
/*
* Full block
*/
if (_SD_BMAP_ISFULL(dirty)) {
if (flush_pos_valid == 0) {
flush_pos_valid = 1;
}
}
/*
* Partial block
*/
else while (dirty) {
if (sblk && flush_pos_valid) {
flush_pos_valid = 0;
flush_len = 0;
}
if (flush_pos_valid == 0) {
flush_pos_valid = 1;
}
}
/*
* If we find a gap, write out what we've got
*/
flush_pos_valid = 0;
flush_len = 0;
}
}
if (flush_pos_valid)
}
static int
{
return (0);
end_cblk_len = 0;
} else {
}
fba_len -= st_cblk_len;
FBA_SIZE(st_cblk_off))) {
"sdbc(_sd_write) safe store failed. Going synchronous");
fba_pos, 0, -1);
return (-1);
}
CACHE_BLOCK_SIZE, 0)) {
"sdbc(_sd_write) safe store failed. "
"Going synchronous");
fba_pos, 0, -1);
return (-1);
}
} /* end while */
if (fba_len) {
FBA_SIZE(end_cblk_len), 0)) {
"sdbc(_sd_write) nvmem dma failed. "
"Going synchronous");
fba_pos, 0, -1);
return (-1);
}
}
return (0);
}
/*
* _sd_sync_write2 - Write-through function.
*
* ARGUMENTS:
* wr_handle - handle into which to write the data.
* wr_st_pos - starting FBA position in wr_handle.
* fba_len - length in fbas.
* flag - NSC_NOBLOCK for async io.
* rd_handle - handle from which to read the data, or NULL.
* rd_st_pos - starting FBA position in rd_handle.
*
* RETURNS:
* errno if return > 0
* NSC_DONE or NSC_PENDING otherwise.
*
* Comments:
* This routine initiates io of the indicated portion. It returns
* synchronously after io is completed if NSC_NOBLOCK is not set.
* Else NSC_PENDING is returned with a subsequent write callback on
* io completion.
*
* See _sd_copy_direct() for usage when
* (wr_handle != rd_handle && rd_handle != NULL)
*/
static int
{
int err;
}
return (E2BIG);
log_bytes = 0;
do {
/*
* clear dirty bits in the write handle.
*/
if (CENTRY_DIRTY(wr_ent)) {
if (CENTRY_DIRTY(wr_ent)) {
/*
* optimization for when we have a
* full cache block, or are doing
* copy_direct (see below).
*/
} else {
dirty &= ~(SDBC_GET_BITS(
}
}
}
/*
* update valid bits in the write handle.
*/
} else {
}
} else {
/*
* doing copy_direct, so mark the write handle
* as invalid since the data is on disk, but not
* in cache.
*/
}
char *,
char *,
wr_pos = 0;
}
rd_pos = 0;
}
} while (fba_len > 0);
if (err != NSC_PENDING) {
}
return (err);
}
static int
int flag)
{
}
/*
* _sd_zero - Interface call to zero out a portion of cache blocks.
*
* ARGUMENTS:
* handle - handle allocated earlier on.
* fba_pos - disk block number to zero from.
* fba_len - length in fbas.
* flag - NSC_NOBLOCK for async io.
*
* RETURNS:
* errno if return > 0
* NSC_DONE or NSC_PENDING otherwise.
*
* Comments:
* This routine zeroes out the indicated portion of the cache blocks
* and commits the data to disk.
* (See write for more details on the commit)
*/
int
int flag)
{
int cd;
int ret;
if (_sdbc_shutdown_in_progress) {
return (EIO);
}
if (!_SD_HANDLE_ACTIVE(handle)) {
(void *)handle);
return (EINVAL);
}
return (EINVAL);
}
if (fba_len == 0) {
return (NSC_DONE);
}
if (_SD_FORCE_DISCONNECT(fba_len))
end_cblk_len = 0;
} else {
}
cur_fba_len -= BLK_FBAS;
}
if (cur_fba_len) {
}
return (ret);
}
/*
* _sd_copy - Copies portions of 2 handles.
*
* ARGUMENTS:
* handle1 - handle allocated earlier on.
* handle2 - handle allocated earlier on.
* fba_pos1 - disk block number to read from.
* fba_pos2 - disk block number to write to.
* fba_len - length in fbas.
*
* RETURNS:
* errno if return > 0
* NSC_DONE otherwise.
*
* Comments:
* This routine copies the 2 handles.
* WARNING: this could put the cache blocks in the destination handle
* in an inconsistent state. (the blocks could be valid in cache,
* but the copy makes the cache different from disk)
*
*/
int
{
if (_sdbc_shutdown_in_progress) {
return (EIO);
}
return (EINVAL);
}
/* Different offsets, do it slowly (per fba) */
while (fba_len) {
FBA_SIZE(1));
fba_pos1++;
fba_pos2++;
fba_len--;
}
return (NSC_DONE);
}
end_cblk_len = 0;
} else {
}
cur_fba_len -= BLK_FBAS;
}
if (cur_fba_len) {
}
return (NSC_DONE);
}
/*
* _sd_copy_direct - Copies data from one handle direct to another disk.
*
* ARGUMENTS:
* handle1 - handle to read from
* handle2 - handle to write to
* fba_pos1 - disk block number to read from.
* fba_pos2 - disk block number to write to.
* fba_len - length in fbas.
*
* RETURNS:
* errno if return > 0
* NSC_DONE otherwise.
*
* Comments:
* This routine copies data from handle1 directly (sync write)
* onto the disk pointed to by handle2. The handle2 is then
* invalidated since the data it contains is now stale compared to
* the disk.
*/
static int
{
int rc;
if (_sdbc_shutdown_in_progress) {
return (EIO);
}
"sdbc(_sd_copy_direct) handle %p or %p not active",
return (EINVAL);
}
"sdbc(_sd_copy_direct) handle2 %p is not writeable",
(void *)handle2);
return (EINVAL);
}
return (rc);
}
/*
* _sd_enqueue_dirty - Enqueue a list of dirty buffers.
*
* ARGUMENTS:
* cd - cache descriptor.
* chain - pointer to list.
* cc_last - last entry in the chain.
* numq - number of entries in the list.
*
* RETURNS:
* NONE.
*
* Comments:
* This routine queues up the dirty blocks for io processing.
* It uses the cc_last to try to coalesce multiple lists into a
* single list, if consecutive writes are sequential in nature.
*/
void
{
#if defined(_SD_DEBUG)
if (chain->cc_dirty_link)
#endif
/* was FAST */
numq = 0;
} else {
} else {
start_write = 1;
}
}
/* was FAST */
if (start_write)
(void) _SD_CD_WRITER(cd);
}
/*
* _sd_enqueue_dirty_chain - Enqueue a chain of a list of dirty buffers.
*
* ARGUMENTS:
* cd - cache descriptor.
* chain_first - first list in this chain.
* chain_last - last list in this chain.
* numq - number of entries being queue (total of all lists)
*
* RETURNS:
* NONE.
*
* Comments:
* This routine is called from the processing after io completions.
* If the buffers are still dirty, they are queued up in one shot.
*/
void
int numq)
{
if (chain_last->cc_dirty_link)
"_sd_enqueue_dirty_chain: chain_last %p dirty_link %p",
/* was FAST */
cdi->cd_lastchain = 0;
} else {
}
/* was FAST */
}
#ifndef _MULTI_DATAMODEL
/* ARGSUSED */
#endif
static int
/*
* Convert the 64 bit statistic structure to 32bit version.
* Possibly losing information when cache is > 4gb. Ha!
*
* NOTE: this code isn't really MT ready since the copied to struct
* is static. However the race is pretty benign and isn't a whole
* lot worse than the vanilla version which copies data to user
* space from kernel structures that can be changing under it too.
* We can't use a local stack structure since the data size is
* 70k or so and kernel stacks are tiny (8k).
*/
{
#ifndef _MULTI_DATAMODEL
return (SDBC_EMODELCONVERT);
#else
int rc = 0;
/*
* This could be done in less code with bcopy type operations
* but this is simpler to follow and easier to change if
* the structures change.
*/
/*
* bcopy the shared stats which has nothing that needs conversion
* in them
*/
sizeof (_sd_shared_t) * sdbc_max_devs);
return (rc);
#endif /* _MULTI_DATAMODEL */
}
int
{
int rc = 0;
if (_sd_cache_stats == NULL) {
static _sd_stats_t dummy;
#ifdef _MULTI_DATAMODEL
static _sd_stats32_t dummy32;
#endif
if (convert_32) {
#ifdef _MULTI_DATAMODEL
#else
#endif
return (rc);
}
if (sdbc_safestore) {
else
_sd_cache_stats->st_wlru_inq = 0;
}
if (convert_32)
return (rc);
}
int
{
int ret = 0;
if (FILE_OPENED(cd)) {
} else
return (ret);
}
int
{
int ret = 0;
if (FILE_OPENED(cd)) {
} else
return (ret);
}
int
{
*hint = 0;
if (FILE_OPENED(cd)) {
return (0);
} else
return (EINVAL);
}
static int
{
int rc;
switch (hint_action) {
case NSC_GET_NODE_HINT:
break;
case NSC_SET_NODE_HINT:
break;
case NSC_CLEAR_NODE_HINT:
break;
default:
break;
}
return (rc);
}
int
{
if ((_sd_node_hint & NSC_NO_FORCED_WRTHRU) &&
(hint & NSC_FORCED_WRTHRU))
return (EINVAL);
return (0);
}
int
{
return (0);
}
int
{
*hint = _sd_node_hint;
return (0);
}
int
{
if (FILE_OPENED(cd)) {
return (0);
} else
return (EINVAL);
}
int
{
if (!FILE_OPENED(cd))
return (EINVAL);
if (flag & NSC_CACHEBLK)
else
*ptr = sdbc_max_fbas;
return (0);
}
int
{
}
int
{
int found = 0;
int rc;
return (EINVAL);
}
if (cc_ent =
if (!CENTRY_PINNED(cc_ent))
continue;
/*
* remove cc_ent from failed links
* cc_lst - pointer to "cc_dirty_link" pointer
* starts at &cd_failed_head.
* cc_tmp - pointer to "cc_dirty_next"
* except when equal to cc_lst.
*/
if (!*cc_tmp)
}
if (*cc_tmp) {
found++;
if (nxt) {
nxt->cc_dirty_link =
(*cc_lst)->cc_dirty_link;
} else {
}
BLK_FBAS);
}
/* clear dirty bits */
/* was FAST */
/* was FAST */
/* release cache block to head of LRU */
if (wctl) {
}
if (!sdbc_use_dmchain)
}
}
return (rc);
}
/*
* Handle allocation
*/
/*
* _sdbc_handles_unload - cache is being unloaded.
*/
void
_sdbc_handles_unload(void)
{
}
/*
* _sdbc_handles_load - cache is being unloaded.
*/
int
_sdbc_handles_load(void)
{
return (0);
}
int
{
_sd_handle_list.hl_count = 0;
return (0);
}
/*
* _sdbc_handles_deconfigure - cache is being deconfigured
*/
void
{
_sd_handle_list.hl_count = 0;
}
{
KM_SLEEP);
/* maintain list and count for debugging */
#if !defined(_SD_NOCHECKS)
#endif
return (handle);
}
int
{
(void *)handle);
return (EINVAL);
}
if (_SD_HANDLE_ACTIVE(handle)) {
"sdbc(_sd_free_handle) attempt to free active handle %p",
(void *)handle);
return (EINVAL);
}
/* remove from queue before free */
return (0);
}
#if !defined (_SD_8K_BLKSIZE)
#define _SD_MAX_MAP 0x100
#else /* !(_SD_8K_BLKSIZE) */
#define _SD_MAX_MAP 0x10000
#endif /* !(_SD_8K_BLKSIZE) */
char _sd_contig_bmap[_SD_MAX_MAP];
void
_sd_init_contig_bmap(void)
{
int i, j;
for (j = i; j < _SD_MAX_MAP; j <<= 1)
_sd_contig_bmap[j] = 1;
}
void
_sd_init_lookup_map(void)
{
unsigned int i, j, k;
for (i = 0; i < _SD_MAX_MAP; i++) {
for (j = i, k = 0; j && ((j & 1) == 0); j >>= 1, k++)
;
stpos = k;
_sd_lookup_map[i].mi_stpos = (unsigned char)k;
for (k = 0; j & 1; j >>= 1, k++)
;
len = k;
_sd_lookup_map[i].mi_len = (unsigned char)k;
}
for (i = 0; i < _SD_MAX_MAP; i++) {
mask = (_sd_bitmap_t)i;
for (j = 0; mask; j++)
_sd_lookup_map[i].mi_dirty_count = (unsigned char)j;
}
for (i = 0; i < _SD_MAX_MAP; i++) {
mask = ~i;
}
}
nsc_def_t _sd_sdbc_def[] = {
"Provide", NSC_CACHE, 0,
0, 0, 0
};
/*
* do the SD_GET_CD_CLUSTER_DATA ioctl (get the global filename data)
*/
/* ARGSUSED */
int
sd_get_file_info_data(char *uaddrp)
{
return (ENOTTY);
}
/*
* do the SD_GET_CD_CLUSTER_SIZE ioctl (get size of global filename area)
*/
int
sd_get_file_info_size(void *uaddrp)
{
sizeof (_sdbc_gl_file_info_size))) {
return (EFAULT);
}
return (0);
}
/*
* SD_GET_GLMUL_SIZES ioctl
* get sizes of the global info regions (for this node only)
*/
/* ARGSUSED */
int
sd_get_glmul_sizes(int *uaddrp)
{
return (ENOTTY);
}
/*
* SD_GET_GLMUL_INFO ioctl
* get the global metadata for write blocks (for this node only)
*/
/* ARGSUSED */
int
sd_get_glmul_info(char *uaddrp)
{
return (ENOTTY);
}
int
{
if (rw == KSTAT_WRITE) {
return (EACCES);
}
/* default to READ */
#ifdef DEBUG
#endif
(void) _sd_get_node_hint(&hint);
return (0);
}
int
{
int name_len;
if (rw == KSTAT_WRITE) {
return (EACCES);
}
/* copy tail of filename to kstat. leave 1 byte for null char */
if (name_len < 0) {
name_len = 0;
}
} else {
"with cache descriptor");
}
#ifdef NSC_MULTI_TERABYTE
#else
#endif
return (0);
}
/*
* cd_kstat_add
*
* Installs all kstats and associated infrastructure (mutex, buffer),
* associated with a particular cache descriptor. This function is called
* when the cache descriptor is opened in _sd_open().
* "cd" -- cache descriptor number whose kstats we wish to add
* returns: 0 on success, -1 on failure
*/
static int
cd_kstat_add(int cd)
{
char name[KSTAT_STRLEN];
return (-1);
}
/* create a regular kstat for this cache descriptor */
if (!sdbc_cd_kstats) {
return (-1);
}
sizeof (sdbc_cd_stats)/sizeof (kstat_named_t),
} else {
}
/* create an I/O kstat for this cache descriptor */
if (!sdbc_cd_io_kstats) {
return (-1);
}
if (sdbc_cd_io_kstats[cd]) {
if (!sdbc_cd_io_kstats_mutexes) {
"allocated");
return (-1);
}
MUTEX_DRIVER, NULL);
} else {
}
return (0);
}
/*
* cd_kstat_remove
*
* Uninstalls all kstats and associated infrastructure (mutex, buffer),
* associated with a particular cache descriptor. This function is called
* when the cache descriptor is closed in _sd_close().
* "cd" -- cache descriptor number whose kstats we wish to remove
* returns: 0 on success, -1 on failure
*/
static int
cd_kstat_remove(int cd)
{
return (-1);
}
/* delete the regular kstat corresponding to this cache descriptor */
}
/* delete the I/O kstat corresponding to this cache descriptor */
if (sdbc_cd_io_kstats_mutexes) {
/* destroy the mutex associated with this I/O kstat */
}
}
return (0);
}
#ifdef DEBUG
/*
* kstat update
*/
int
{
simplect_dm++;
/* global dynmem_processing_dm */
if (rw == KSTAT_WRITE) {
}
return (0);
}
/* default to READ */
return (0);
}
#endif