fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * CDDL HEADER START
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * The contents of this file are subject to the terms of the
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Common Development and Distribution License (the "License").
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * You may not use this file except in compliance with the License.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * See the License for the specific language governing permissions
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * and limitations under the License.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * When distributing Covered Code, include this CDDL HEADER in each
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * If applicable, add the following below this CDDL HEADER, with the
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * fields enclosed by brackets "[]" replaced with your own identifying
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * information: Portions Copyright [yyyy] [name of copyright owner]
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * CDDL HEADER END
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Use is subject to license terms.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Normally we unregister memory at deconfig time. By setting this non-zero
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * it will be delayed until unload time.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Fortensc_mem_t *sdbc_local_mem, *sdbc_stats_mem, *sdbc_cache_mem;
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Forward declare all statics that are used before defined to enforce
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * parameter checking
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Some (if not all) of these could be removed if the code were reordered
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forteint _sd_fill_pattern(caddr_t addr, uint_t pat, uint_t size);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Fortestatic void _sdbc_nodeid_deconfigure(void);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Fortestatic void _sdbc_nodeid_configure(void);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Fortestatic void _sdbc_thread_deconfigure(void);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Fortestatic int _sdbc_thread_configure(void);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * _sdbc_memtype_configure - register with the sd layer the types of memory
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * we want to use. If any of the critical memory types can't be registered
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * we return non-zero otherwise 0.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if ((sdbc_info_mem = nsc_register_mem("sdbc:info",
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sdbc_local_mem = nsc_register_mem("sdbc:local", NSC_MEM_LOCAL, 0);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sdbc_stats_mem = nsc_register_mem("sdbc:stats", NSC_MEM_LOCAL, 0);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sdbc_iobuf_mem = nsc_register_mem("sdbc:iobuf", NSC_MEM_LOCAL, 0);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sdbc_cache_mem = nsc_register_mem("sdbc:cache", NSC_MEM_LOCAL, 0);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sdbc_hash_mem = nsc_register_mem("sdbc:hash", NSC_MEM_LOCAL, 0);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * _sdbc_memtype_deconfigure - undo the effects of _sdbc_memtype_configure.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sdbc_local_mem = sdbc_stats_mem = sdbc_cache_mem = NULL;
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * figure out what kind of safe storage we need
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana (_sd_cache_config.cache_mem[_SD_NO_NET] * MEGABYTE)/2;
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte safestore_config.ssc_pattern = _sd_cache_config.fill_pattern;
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte safestore_config.ssc_flag = _sd_cache_config.gen_pattern ?
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * _sdbc_configure - process the ioctl that describes the configuration
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * for the cache. This is the main driver routine for cache configuration
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Return 0 on success, otherwise nonzero.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte _sdbc_config_t *mgmt, spcs_s_info_t spcs_kstatus)
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte _sd_print(1, "sdbc(_sdbc_configure) _SD_MAGIC 0x%x\n", _SD_MAGIC);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte bzero(&_sd_cache_config, sizeof (_sd_cache_config));
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* copy in mgmt config info */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte for (i = 0; i < CACHE_MEM_PAD; i++) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte _sd_cache_config.cache_mem[i] = mgmt->cache_mem[i];
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* fake the rest as a single node config */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Check that the requested cache size doesn't break the code.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * This test can be refined once the cache size is stored in variables
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * larger than an int.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte for (i = 0; i < MAX_CACHE_NET; i++) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte "negative cache size (%d) for net %d",
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if (_sd_cache_config.cache_mem[i] > MAX_CACHE_SIZE) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte "cache size limited to %d megabytes for net %d",
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte (void) spcs_s_inttostring(_sd_cache_config.blk_size, itmp,
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sizeof (itmp), 0);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte get_high_bit(_sd_cache_config.blk_size)) == -1) ||
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte (_sd_cache_config.blk_size != (1 << _sd_cblock_shift))) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte (void) spcs_s_inttostring(_sd_cache_config.blk_size, itmp,
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sizeof (itmp), 0);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sdbc_use_dmchain = (_sd_cache_config.reserved1 & CFG_USE_DMCHAIN);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sdbc_static_cache = (_sd_cache_config.reserved1 & CFG_STATIC_CACHE);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte (void) spcs_s_inttostring((_SD_SELF_HOST > nsc_max_nodeid ?
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte _SD_SELF_HOST : _SD_MIRROR_HOST), itmp, sizeof (itmp), 0);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte spcs_s_add(spcs_kstatus, SDBC_EINVHOSTID, itmp, itmp2);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte spcs_s_add(spcs_kstatus, SDBC_ENOTSAME, itmp, itmp2);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* initialize the safestore modules */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* figure out which kind of safestore we need to use */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* open and configure the safestore module */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if ((sdbc_safestore = sst_open(ss_type, 0)) == NULL) {
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana cmn_err(CE_WARN, "!cannot open safestore module for type %x",
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if (SSOP_CONFIGURE(sdbc_safestore, &safestore_config,
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana "!cannot configure safestore module for type %x",
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* try ram if possible, otherwise return */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte ASSERT(safestore_config.ssc_ss_psize <= UINT16_MAX); /* LINTED */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte _sd_net_config.sn_psize = safestore_config.ssc_ss_psize;
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana _sd_cache_config.cache_mem[_SD_NO_NET] * MEGABYTE;
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte cache_bytes = _sd_net_config.sn_cpages * BLK_SIZE(1);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if ((rc = _sdbc_iobuf_configure(_sd_cache_config.iobuf))) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * nvmem support:
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * if the cache did not shutdown properly we mark it as dirty.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * this must be done before _sdbc_cache_configure() so it can
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * refresh sd_info_mem and sd_file_mem from nvmem if necsssary,
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * and before _sdbc_ft_configure() so the ft thread will do a recovery.
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana "!sdbc(_sdbc_configure) cache marked dirty after"
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte " incomplete shutdown");
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if ((rc = _sdbc_cache_configure(cache_bytes / BLK_SIZE(1),
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* ST_ALERT trace buffer */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if (_sdbc_tr_configure(-1 /* SDT_INV_CD */) != 0) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if (_sdbc_tdaemon_configure(_sd_cache_config.test_demons)) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte sdbc_power = nsc_register_power("sdbc", _sdbc_power_def);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * try to control the race between the ft thread
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * and threads that will open the devices that the ft thread
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * may be recovering. this synchronizing with the ft thread
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * prevents sd_cadmin from returning until ft has opened
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * the recovery devices, so if other apps wait for sd_cadmin
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * to complete the race is prevented.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte cv_wait(&_sdbc_ft_hold_io_cv, &_sdbc_ft_hold_io_lk);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte io = nsc_register_io("sdbc", NSC_SDBC_ID|NSC_FILTER,
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana cmn_err(CE_NOTE, "!sd_config: Cache has been configured");
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * _sdbc_deconfigure - Put the cache back to the unconfigured state. Release
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * any memory we allocated as part of the configuration process (but not the
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * load/init process). Put globals back to unconfigured state and shut down
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * any processes/threads we have running.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Since the cache has loaded we know that global lock/sv's are present and
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * we can use them to produce an orderly deconfiguration.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * NOTE: this routine and its callee should always be capable of reversing
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * the effects of _sdbc_configure no matter what partially configured
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * state might be present.
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana cmn_err(CE_NOTE, "!SD cache being deconfigured.");
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* check if there is pinned data and our mirror is down */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte for (i = 0; i < sdbc_max_devs; i++) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * if (!(cdi->cd_info->sh_failed))
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* remember hint setting for restoration in case shutdown fails */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* TODO - there is a possible race between deconfig and power hits... */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* Re-register-power if it was register before. */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* Remove NSC_FORCED_WRTHRU if we set it */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte _sd_remote_disable(0); /* notify mirror to forced_wrthru */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * close devices, deconfigure processes, wait for exits
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte for (i = 0; i < sdbc_max_devs; i++) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if (FILE_OPENED(i) && ((rc = _sd_close(i)) > 0)) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * look for pinned data
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * TODO sort this out for multinode systems.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * cannot shutdown with pinned data on multinode.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * the state of pinned data should be determined in
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * the close operation.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte for (i = 0; i < sdbc_max_devs; i++) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * if (!(cdi->cd_info->sh_failed))
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana "!sdbc(_sd_deconfigure) Pinned Data on cd %d(%s)",
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * remove all dynamically allocated cache data memory
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * there should be no i/o at this point
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * At this point no thread of control should be active in the cache
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * but us (unless they are blocked on the config lock).
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte _sd_remote_disable(1); /* notify mirror I/O shutdown complete */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte#define KEEP_TRACES 0 /* set to 1 keep traces after deconfig */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * This needs to happen before we unregister the memory.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* delete/free hash table, cache blocks, etc */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Call ss deconfig(),
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * check for valid pointer in case _sdbc_configure()
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * failed before safestrore system was initialized.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* tear down safestore system */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte bzero(&_sd_cache_config, sizeof (_sd_cache_param_t));
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana cmn_err(CE_NOTE, "!SD cache deconfigured.");
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte while ((lowbit = find_low_bit(newblk, 0)) != 32) {
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana "!sdbc(get_high_bit) invalid block size %x\n", size);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte return (-1);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte_sd_fill_pattern(caddr_t addr, uint_t pat, uint_t size)
3270659f55e0928d6edec3d26217cc29398a8149Srikanth, Ramana cmn_err(CE_WARN, "!sdbc(_sd_fill pattern) no more memory");
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte return (-1);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * _sdbc_nodeid_deconfigure - merely a place holder until
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * such time as there is something to be undone w.r.t.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * _sdbc_nodeid_configure.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* My but we're quick */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * _sdbc_nodeid_configure - configure the nodeid's we need to connect
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * to any other nodes in the network.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte _sd_nodes_configured = _sd_cache_config.num_nodes;
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * _sdbc_thread_deconfigure - cache is being deconfigure, stop any
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * thread activity.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * _sdbc_thread_configure - cache is being configured, initialize the
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * threads we need for flushing dirty cds.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte _sd_ioset = nst_init("sd_thr", _sd_cache_config.threads);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte for (i = 0; i < CACHE_MEM_PAD; i++) {