io/vioblk/vioblk.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved.
 * Copyright (c) 2012, Alexey Zaytsev <alexey.zaytsev@gmail.com>
 */


#include <sys/modctl.h>
#include <sys/blkdev.h>
#include <sys/types.h>
#include <sys/errno.h>
#include <sys/param.h>
#include <sys/stropts.h>
#include <sys/stream.h>
#include <sys/strsubr.h>
#include <sys/kmem.h>
#include <sys/conf.h>
#include <sys/devops.h>
#include <sys/ksynch.h>
#include <sys/stat.h>
#include <sys/modctl.h>
#include <sys/debug.h>
#include <sys/pci.h>
#include <sys/sysmacros.h>
#include "virtiovar.h"
#include "virtioreg.h"

/* Feature bits */
#define VIRTIO_BLK_F_BARRIER    (1<<0)
#define VIRTIO_BLK_F_SIZE_MAX   (1<<1)
#define VIRTIO_BLK_F_SEG_MAX    (1<<2)
#define VIRTIO_BLK_F_GEOMETRY   (1<<4)
#define VIRTIO_BLK_F_RO     (1<<5)
#define VIRTIO_BLK_F_BLK_SIZE   (1<<6)
#define VIRTIO_BLK_F_SCSI   (1<<7)
#define VIRTIO_BLK_F_FLUSH  (1<<9)
#define VIRTIO_BLK_F_TOPOLOGY   (1<<10)

/* Configuration registers */
#define VIRTIO_BLK_CONFIG_CAPACITY  0 /* 64bit */
#define VIRTIO_BLK_CONFIG_SIZE_MAX  8 /* 32bit */
#define VIRTIO_BLK_CONFIG_SEG_MAX   12 /* 32bit */
#define VIRTIO_BLK_CONFIG_GEOMETRY_C    16 /* 16bit */
#define VIRTIO_BLK_CONFIG_GEOMETRY_H    18 /* 8bit */
#define VIRTIO_BLK_CONFIG_GEOMETRY_S    19 /* 8bit */
#define VIRTIO_BLK_CONFIG_BLK_SIZE  20 /* 32bit */
#define VIRTIO_BLK_CONFIG_TOPO_PBEXP    24 /* 8bit */
#define VIRTIO_BLK_CONFIG_TOPO_ALIGN    25 /* 8bit */
#define VIRTIO_BLK_CONFIG_TOPO_MIN_SZ   26 /* 16bit */
#define VIRTIO_BLK_CONFIG_TOPO_OPT_SZ   28 /* 32bit */

/* Command */
#define VIRTIO_BLK_T_IN         0
#define VIRTIO_BLK_T_OUT        1
#define VIRTIO_BLK_T_SCSI_CMD       2
#define VIRTIO_BLK_T_SCSI_CMD_OUT   3
#define VIRTIO_BLK_T_FLUSH      4
#define VIRTIO_BLK_T_FLUSH_OUT      5
#define VIRTIO_BLK_T_GET_ID     8
#define VIRTIO_BLK_T_BARRIER        0x80000000

#define VIRTIO_BLK_ID_BYTES 20 /* devid */

/* Statuses */
#define VIRTIO_BLK_S_OK     0
#define VIRTIO_BLK_S_IOERR  1
#define VIRTIO_BLK_S_UNSUPP 2

#define DEF_MAXINDIRECT     (128)
#define DEF_MAXSECTOR       (4096)

#define VIOBLK_POISON       0xdead0001dead0001

/*
 * Static Variables.
 */
static char vioblk_ident[] = "VirtIO block driver";

/* Request header structure */
struct vioblk_req_hdr {
    uint32_t        type;   /* VIRTIO_BLK_T_* */
    uint32_t        ioprio;
    uint64_t        sector;
};

struct vioblk_req {
    struct vioblk_req_hdr   hdr;
    uint8_t         status;
    uint8_t         unused[3];
    unsigned int        ndmac;
    ddi_dma_handle_t    dmah;
    ddi_dma_handle_t    bd_dmah;
    ddi_dma_cookie_t    dmac;
    bd_xfer_t       *xfer;
};

struct vioblk_stats {
    struct kstat_named  sts_rw_outofmemory;
    struct kstat_named  sts_rw_badoffset;
    struct kstat_named  sts_rw_queuemax;
    struct kstat_named  sts_rw_cookiesmax;
    struct kstat_named  sts_rw_cacheflush;
    struct kstat_named  sts_intr_queuemax;
    struct kstat_named  sts_intr_total;
    struct kstat_named  sts_io_errors;
    struct kstat_named  sts_unsupp_errors;
    struct kstat_named  sts_nxio_errors;
};

struct vioblk_lstats {
    uint64_t        rw_cacheflush;
    uint64_t        intr_total;
    unsigned int        rw_cookiesmax;
    unsigned int        intr_queuemax;
    unsigned int        io_errors;
    unsigned int        unsupp_errors;
    unsigned int        nxio_errors;
};

struct vioblk_softc {
    dev_info_t      *sc_dev; /* mirrors virtio_softc->sc_dev */
    struct virtio_softc sc_virtio;
    struct virtqueue    *sc_vq;
    bd_handle_t     bd_h;
    struct vioblk_req   *sc_reqs;
    struct vioblk_stats *ks_data;
    kstat_t         *sc_intrstat;
    uint64_t        sc_capacity;
    uint64_t        sc_nblks;
    struct vioblk_lstats    sc_stats;
    short           sc_blkflags;
    boolean_t       sc_in_poll_mode;
    boolean_t       sc_readonly;
    int         sc_blk_size;
    int         sc_pblk_size;
    int         sc_seg_max;
    int         sc_seg_size_max;
    kmutex_t        lock_devid;
    kcondvar_t      cv_devid;
    char            devid[VIRTIO_BLK_ID_BYTES + 1];
};

static int vioblk_get_id(struct vioblk_softc *sc);

static int vioblk_read(void *arg, bd_xfer_t *xfer);
static int vioblk_write(void *arg, bd_xfer_t *xfer);
static int vioblk_flush(void *arg, bd_xfer_t *xfer);
static void vioblk_driveinfo(void *arg, bd_drive_t *drive);
static int vioblk_mediainfo(void *arg, bd_media_t *media);
static int vioblk_devid_init(void *, dev_info_t *, ddi_devid_t *);
uint_t vioblk_int_handler(caddr_t arg1, caddr_t arg2);

static bd_ops_t vioblk_ops = {
    BD_OPS_VERSION_0,
    vioblk_driveinfo,
    vioblk_mediainfo,
    vioblk_devid_init,
    vioblk_flush,
    vioblk_read,
    vioblk_write,
};

static int vioblk_quiesce(dev_info_t *);
static int vioblk_attach(dev_info_t *, ddi_attach_cmd_t);
static int vioblk_detach(dev_info_t *, ddi_detach_cmd_t);

static struct dev_ops vioblk_dev_ops = {
    DEVO_REV,
    0,
    ddi_no_info,
    nulldev,    /* identify */
    nulldev,    /* probe */
    vioblk_attach,  /* attach */
    vioblk_detach,  /* detach */
    nodev,      /* reset */
    NULL,       /* cb_ops */
    NULL,       /* bus_ops */
    NULL,       /* power */
    vioblk_quiesce  /* quiesce */
};


/* Standard Module linkage initialization for a Streams driver */
extern struct mod_ops mod_driverops;

static struct modldrv modldrv = {
    &mod_driverops,     /* Type of module.  This one is a driver */
    vioblk_ident,    /* short description */
    &vioblk_dev_ops /* driver specific ops */
};

static struct modlinkage modlinkage = {
    MODREV_1,
    {
        (void *)&modldrv,
        NULL,
    },
};

ddi_device_acc_attr_t vioblk_attr = {
    DDI_DEVICE_ATTR_V0,
    DDI_NEVERSWAP_ACC,  /* virtio is always native byte order */
    DDI_STORECACHING_OK_ACC,
    DDI_DEFAULT_ACC
};

/* DMA attr for the header/status blocks. */
static ddi_dma_attr_t vioblk_req_dma_attr = {
    DMA_ATTR_V0,            /* dma_attr version */
    0,              /* dma_attr_addr_lo */
    0xFFFFFFFFFFFFFFFFull,      /* dma_attr_addr_hi */
    0x00000000FFFFFFFFull,      /* dma_attr_count_max   */
    1,              /* dma_attr_align   */
    1,              /* dma_attr_burstsizes  */
    1,              /* dma_attr_minxfer */
    0xFFFFFFFFull,          /* dma_attr_maxxfer */
    0xFFFFFFFFFFFFFFFFull,      /* dma_attr_seg     */
    1,              /* dma_attr_sgllen  */
    1,              /* dma_attr_granular    */
    0,              /* dma_attr_flags   */
};

/* DMA attr for the data blocks. */
static ddi_dma_attr_t vioblk_bd_dma_attr = {
    DMA_ATTR_V0,            /* dma_attr version */
    0,              /* dma_attr_addr_lo */
    0xFFFFFFFFFFFFFFFFull,      /* dma_attr_addr_hi */
    0x00000000FFFFFFFFull,      /* dma_attr_count_max   */
    1,              /* dma_attr_align   */
    1,              /* dma_attr_burstsizes  */
    1,              /* dma_attr_minxfer */
    0,              /* dma_attr_maxxfer, set in attach */
    0xFFFFFFFFFFFFFFFFull,      /* dma_attr_seg     */
    0,              /* dma_attr_sgllen, set in attach */
    1,              /* dma_attr_granular    */
    0,              /* dma_attr_flags   */
};

static int
vioblk_rw(struct vioblk_softc *sc, bd_xfer_t *xfer, int type,
    uint32_t len)
{
    struct vioblk_req *req;
    struct vq_entry *ve_hdr;
    int total_cookies, write;

    write = (type == VIRTIO_BLK_T_OUT ||
        type == VIRTIO_BLK_T_FLUSH_OUT) ? 1 : 0;
    total_cookies = 2;

    if ((xfer->x_blkno + xfer->x_nblks) > sc->sc_nblks) {
        sc->ks_data->sts_rw_badoffset.value.ui64++;
        return (EINVAL);
    }

    /* allocate top entry */
    ve_hdr = vq_alloc_entry(sc->sc_vq);
    if (!ve_hdr) {
        sc->ks_data->sts_rw_outofmemory.value.ui64++;
        return (ENOMEM);
    }

    /* getting request */
    req = &sc->sc_reqs[ve_hdr->qe_index];
    req->hdr.type = type;
    req->hdr.ioprio = 0;
    req->hdr.sector = xfer->x_blkno;
    req->xfer = xfer;

    /* Header */
    virtio_ve_add_indirect_buf(ve_hdr, req->dmac.dmac_laddress,
        sizeof (struct vioblk_req_hdr), B_TRUE);

    /* Payload */
    if (len > 0) {
        virtio_ve_add_cookie(ve_hdr, xfer->x_dmah, xfer->x_dmac,
            xfer->x_ndmac, write ? B_TRUE : B_FALSE);
        total_cookies += xfer->x_ndmac;
    }

    /* Status */
    virtio_ve_add_indirect_buf(ve_hdr,
        req->dmac.dmac_laddress + sizeof (struct vioblk_req_hdr),
        sizeof (uint8_t), B_FALSE);

    /* sending the whole chain to the device */
    virtio_push_chain(ve_hdr, B_TRUE);

    if (sc->sc_stats.rw_cookiesmax < total_cookies)
        sc->sc_stats.rw_cookiesmax = total_cookies;

    return (DDI_SUCCESS);
}

/*
 * Now in polling mode. Interrupts are off, so we
 * 1) poll for the already queued requests to complete.
 * 2) push our request.
 * 3) wait for our request to complete.
 */
static int
vioblk_rw_poll(struct vioblk_softc *sc, bd_xfer_t *xfer,
    int type, uint32_t len)
{
    clock_t tmout;
    int ret;

    ASSERT(xfer->x_flags & BD_XFER_POLL);

    /* Prevent a hard hang. */
    tmout = drv_usectohz(30000000);

    /* Poll for an empty queue */
    while (vq_num_used(sc->sc_vq)) {
        /* Check if any pending requests completed. */
        ret = vioblk_int_handler((caddr_t)&sc->sc_virtio, NULL);
        if (ret != DDI_INTR_CLAIMED) {
            drv_usecwait(10);
            tmout -= 10;
            return (ETIMEDOUT);
        }
    }

    ret = vioblk_rw(sc, xfer, type, len);
    if (ret)
        return (ret);

    tmout = drv_usectohz(30000000);
    /* Poll for an empty queue again. */
    while (vq_num_used(sc->sc_vq)) {
        /* Check if any pending requests completed. */
        ret = vioblk_int_handler((caddr_t)&sc->sc_virtio, NULL);
        if (ret != DDI_INTR_CLAIMED) {
            drv_usecwait(10);
            tmout -= 10;
            return (ETIMEDOUT);
        }
    }

    return (DDI_SUCCESS);
}

static int
vioblk_read(void *arg, bd_xfer_t *xfer)
{
    int ret;
    struct vioblk_softc *sc = (void *)arg;

    if (xfer->x_flags & BD_XFER_POLL) {
        if (!sc->sc_in_poll_mode) {
            virtio_stop_vq_intr(sc->sc_vq);
            sc->sc_in_poll_mode = 1;
        }

        ret = vioblk_rw_poll(sc, xfer, VIRTIO_BLK_T_IN,
            xfer->x_nblks * DEV_BSIZE);
    } else {
        if (sc->sc_in_poll_mode) {
            virtio_start_vq_intr(sc->sc_vq);
            sc->sc_in_poll_mode = 0;
        }

        ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_IN,
            xfer->x_nblks * DEV_BSIZE);
    }

    return (ret);
}

static int
vioblk_write(void *arg, bd_xfer_t *xfer)
{
    int ret;
    struct vioblk_softc *sc = (void *)arg;

    if (xfer->x_flags & BD_XFER_POLL) {
        if (!sc->sc_in_poll_mode) {
            virtio_stop_vq_intr(sc->sc_vq);
            sc->sc_in_poll_mode = 1;
        }

        ret = vioblk_rw_poll(sc, xfer, VIRTIO_BLK_T_OUT,
            xfer->x_nblks * DEV_BSIZE);
    } else {
        if (sc->sc_in_poll_mode) {
            virtio_start_vq_intr(sc->sc_vq);
            sc->sc_in_poll_mode = 0;
        }

        ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_OUT,
            xfer->x_nblks * DEV_BSIZE);
    }
    return (ret);
}

static int
vioblk_flush(void *arg, bd_xfer_t *xfer)
{
    int ret;
    struct vioblk_softc *sc = (void *)arg;

    ASSERT((xfer->x_flags & BD_XFER_POLL) == 0);

    ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_FLUSH_OUT,
        xfer->x_nblks * DEV_BSIZE);

    if (!ret)
        sc->sc_stats.rw_cacheflush++;

    return (ret);
}


static void
vioblk_driveinfo(void *arg, bd_drive_t *drive)
{
    struct vioblk_softc *sc = (void *)arg;

    drive->d_qsize = sc->sc_vq->vq_num;
    drive->d_removable = B_FALSE;
    drive->d_hotpluggable = B_TRUE;
    drive->d_target = 0;
    drive->d_lun = 0;

    drive->d_vendor = "Virtio";
    drive->d_vendor_len = strlen(drive->d_vendor);

    drive->d_product = "Block Device";
    drive->d_product_len = strlen(drive->d_product);

    (void) vioblk_get_id(sc);
    drive->d_serial = sc->devid;
    drive->d_serial_len = strlen(drive->d_serial);

    drive->d_revision = "0000";
    drive->d_revision_len = strlen(drive->d_revision);
}

static int
vioblk_mediainfo(void *arg, bd_media_t *media)
{
    struct vioblk_softc *sc = (void *)arg;

    media->m_nblks = sc->sc_nblks;
    media->m_blksize = sc->sc_blk_size;
    media->m_readonly = sc->sc_readonly;
    media->m_pblksize = sc->sc_pblk_size;
    return (0);
}

static int
vioblk_get_id(struct vioblk_softc *sc)
{
    clock_t deadline;
    int ret;
    bd_xfer_t xfer;

    deadline = ddi_get_lbolt() + (clock_t)drv_usectohz(3 * 1000000);
    (void) memset(&xfer, 0, sizeof (bd_xfer_t));
    xfer.x_nblks = 1;

    ret = ddi_dma_alloc_handle(sc->sc_dev, &vioblk_bd_dma_attr,
        DDI_DMA_SLEEP, NULL, &xfer.x_dmah);
    if (ret != DDI_SUCCESS)
        goto out_alloc;

    ret = ddi_dma_addr_bind_handle(xfer.x_dmah, NULL, (caddr_t)&sc->devid,
        VIRTIO_BLK_ID_BYTES, DDI_DMA_READ | DDI_DMA_CONSISTENT,
        DDI_DMA_SLEEP, NULL, &xfer.x_dmac, &xfer.x_ndmac);
    if (ret != DDI_DMA_MAPPED) {
        ret = DDI_FAILURE;
        goto out_map;
    }

    mutex_enter(&sc->lock_devid);

    ret = vioblk_rw(sc, &xfer, VIRTIO_BLK_T_GET_ID,
        VIRTIO_BLK_ID_BYTES);
    if (ret) {
        mutex_exit(&sc->lock_devid);
        goto out_rw;
    }

    /* wait for reply */
    ret = cv_timedwait(&sc->cv_devid, &sc->lock_devid, deadline);
    mutex_exit(&sc->lock_devid);

    (void) ddi_dma_unbind_handle(xfer.x_dmah);
    ddi_dma_free_handle(&xfer.x_dmah);

    /* timeout */
    if (ret < 0) {
        dev_err(sc->sc_dev, CE_WARN,
            "Cannot get devid from the device");
        return (DDI_FAILURE);
    }

    return (0);

out_rw:
    (void) ddi_dma_unbind_handle(xfer.x_dmah);
out_map:
    ddi_dma_free_handle(&xfer.x_dmah);
out_alloc:
    return (ret);
}

static int
vioblk_devid_init(void *arg, dev_info_t *devinfo, ddi_devid_t *devid)
{
    struct vioblk_softc *sc = (void *)arg;
    int ret;

    ret = vioblk_get_id(sc);
    if (ret != DDI_SUCCESS)
        return (ret);

    ret = ddi_devid_init(devinfo, DEVID_ATA_SERIAL,
        VIRTIO_BLK_ID_BYTES, sc->devid, devid);
    if (ret != DDI_SUCCESS) {
        dev_err(devinfo, CE_WARN, "Cannot build devid from the device");
        return (ret);
    }

    dev_debug(sc->sc_dev, CE_NOTE,
        "devid %x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x",
        sc->devid[0], sc->devid[1], sc->devid[2], sc->devid[3],
        sc->devid[4], sc->devid[5], sc->devid[6], sc->devid[7],
        sc->devid[8], sc->devid[9], sc->devid[10], sc->devid[11],
        sc->devid[12], sc->devid[13], sc->devid[14], sc->devid[15],
        sc->devid[16], sc->devid[17], sc->devid[18], sc->devid[19]);

    return (0);
}

static void
vioblk_show_features(struct vioblk_softc *sc, const char *prefix,
    uint32_t features)
{
    char buf[512];
    char *bufp = buf;
    char *bufend = buf + sizeof (buf);

    /* LINTED E_PTRDIFF_OVERFLOW */
    bufp += snprintf(bufp, bufend - bufp, prefix);

    /* LINTED E_PTRDIFF_OVERFLOW */
    bufp += virtio_show_features(features, bufp, bufend - bufp);


    /* LINTED E_PTRDIFF_OVERFLOW */
    bufp += snprintf(bufp, bufend - bufp, "Vioblk ( ");

    if (features & VIRTIO_BLK_F_BARRIER)
        /* LINTED E_PTRDIFF_OVERFLOW */
        bufp += snprintf(bufp, bufend - bufp, "BARRIER ");
    if (features & VIRTIO_BLK_F_SIZE_MAX)
        /* LINTED E_PTRDIFF_OVERFLOW */
        bufp += snprintf(bufp, bufend - bufp, "SIZE_MAX ");
    if (features & VIRTIO_BLK_F_SEG_MAX)
        /* LINTED E_PTRDIFF_OVERFLOW */
        bufp += snprintf(bufp, bufend - bufp, "SEG_MAX ");
    if (features & VIRTIO_BLK_F_GEOMETRY)
        /* LINTED E_PTRDIFF_OVERFLOW */
        bufp += snprintf(bufp, bufend - bufp, "GEOMETRY ");
    if (features & VIRTIO_BLK_F_RO)
        /* LINTED E_PTRDIFF_OVERFLOW */
        bufp += snprintf(bufp, bufend - bufp, "RO ");
    if (features & VIRTIO_BLK_F_BLK_SIZE)
        /* LINTED E_PTRDIFF_OVERFLOW */
        bufp += snprintf(bufp, bufend - bufp, "BLK_SIZE ");
    if (features & VIRTIO_BLK_F_SCSI)
        /* LINTED E_PTRDIFF_OVERFLOW */
        bufp += snprintf(bufp, bufend - bufp, "SCSI ");
    if (features & VIRTIO_BLK_F_FLUSH)
        /* LINTED E_PTRDIFF_OVERFLOW */
        bufp += snprintf(bufp, bufend - bufp, "FLUSH ");
    if (features & VIRTIO_BLK_F_TOPOLOGY)
        /* LINTED E_PTRDIFF_OVERFLOW */
        bufp += snprintf(bufp, bufend - bufp, "TOPOLOGY ");

    /* LINTED E_PTRDIFF_OVERFLOW */
    bufp += snprintf(bufp, bufend - bufp, ")");
    *bufp = '\0';

    dev_debug(sc->sc_dev, CE_NOTE, "%s", buf);
}

static int
vioblk_dev_features(struct vioblk_softc *sc)
{
    uint32_t host_features;

    host_features = virtio_negotiate_features(&sc->sc_virtio,
        VIRTIO_BLK_F_RO |
        VIRTIO_BLK_F_GEOMETRY |
        VIRTIO_BLK_F_BLK_SIZE |
        VIRTIO_BLK_F_FLUSH |
        VIRTIO_BLK_F_TOPOLOGY |
        VIRTIO_BLK_F_SEG_MAX |
        VIRTIO_BLK_F_SIZE_MAX |
        VIRTIO_F_RING_INDIRECT_DESC);

    vioblk_show_features(sc, "Host features: ", host_features);
    vioblk_show_features(sc, "Negotiated features: ",
        sc->sc_virtio.sc_features);

    if (!(sc->sc_virtio.sc_features & VIRTIO_F_RING_INDIRECT_DESC)) {
        dev_err(sc->sc_dev, CE_NOTE,
            "Host does not support RING_INDIRECT_DESC, bye.");
        return (DDI_FAILURE);
    }

    return (DDI_SUCCESS);
}

/* ARGSUSED */
uint_t
vioblk_int_handler(caddr_t arg1, caddr_t arg2)
{
    struct virtio_softc *vsc = (void *)arg1;
    struct vioblk_softc *sc = container_of(vsc,
        struct vioblk_softc, sc_virtio);
    struct vq_entry *ve;
    uint32_t len;
    int i = 0, error;

    while ((ve = virtio_pull_chain(sc->sc_vq, &len))) {
        struct vioblk_req *req = &sc->sc_reqs[ve->qe_index];
        bd_xfer_t *xfer = req->xfer;
        uint8_t status = req->status;
        uint32_t type = req->hdr.type;

        if (req->xfer == (void *)VIOBLK_POISON) {
            dev_err(sc->sc_dev, CE_WARN, "Poisoned descriptor!");
            virtio_free_chain(ve);
            return (DDI_INTR_CLAIMED);
        }

        req->xfer = (void *) VIOBLK_POISON;

        /* Note: blkdev tears down the payload mapping for us. */
        virtio_free_chain(ve);

        /* returning payload back to blkdev */
        switch (status) {
            case VIRTIO_BLK_S_OK:
                error = 0;
                break;
            case VIRTIO_BLK_S_IOERR:
                error = EIO;
                sc->sc_stats.io_errors++;
                break;
            case VIRTIO_BLK_S_UNSUPP:
                sc->sc_stats.unsupp_errors++;
                error = ENOTTY;
                break;
            default:
                sc->sc_stats.nxio_errors++;
                error = ENXIO;
                break;
        }

        if (type == VIRTIO_BLK_T_GET_ID) {
            /* notify devid_init */
            mutex_enter(&sc->lock_devid);
            cv_broadcast(&sc->cv_devid);
            mutex_exit(&sc->lock_devid);
        } else
            bd_xfer_done(xfer, error);

        i++;
    }

    /* update stats */
    if (sc->sc_stats.intr_queuemax < i)
        sc->sc_stats.intr_queuemax = i;
    sc->sc_stats.intr_total++;

    return (DDI_INTR_CLAIMED);
}

/* ARGSUSED */
uint_t
vioblk_config_handler(caddr_t arg1, caddr_t arg2)
{
    return (DDI_INTR_CLAIMED);
}

static int
vioblk_register_ints(struct vioblk_softc *sc)
{
    int ret;

    struct virtio_int_handler vioblk_conf_h = {
        vioblk_config_handler
    };

    struct virtio_int_handler vioblk_vq_h[] = {
        { vioblk_int_handler },
        { NULL },
    };

    ret = virtio_register_ints(&sc->sc_virtio,
        &vioblk_conf_h, vioblk_vq_h);

    return (ret);
}

static void
vioblk_free_reqs(struct vioblk_softc *sc)
{
    int i, qsize;

    qsize = sc->sc_vq->vq_num;

    for (i = 0; i < qsize; i++) {
        struct vioblk_req *req = &sc->sc_reqs[i];

        if (req->ndmac)
            (void) ddi_dma_unbind_handle(req->dmah);

        if (req->dmah)
            ddi_dma_free_handle(&req->dmah);
    }

    kmem_free(sc->sc_reqs, sizeof (struct vioblk_req) * qsize);
}

static int
vioblk_alloc_reqs(struct vioblk_softc *sc)
{
    int i, qsize;
    int ret;

    qsize = sc->sc_vq->vq_num;

    sc->sc_reqs = kmem_zalloc(sizeof (struct vioblk_req) * qsize, KM_SLEEP);

    for (i = 0; i < qsize; i++) {
        struct vioblk_req *req = &sc->sc_reqs[i];

        ret = ddi_dma_alloc_handle(sc->sc_dev, &vioblk_req_dma_attr,
            DDI_DMA_SLEEP, NULL, &req->dmah);
        if (ret != DDI_SUCCESS) {

            dev_err(sc->sc_dev, CE_WARN,
                "Can't allocate dma handle for req "
                "buffer %d", i);
            goto exit;
        }

        ret = ddi_dma_addr_bind_handle(req->dmah, NULL,
            (caddr_t)&req->hdr,
            sizeof (struct vioblk_req_hdr) + sizeof (uint8_t),
            DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP,
            NULL, &req->dmac, &req->ndmac);
        if (ret != DDI_DMA_MAPPED) {
            dev_err(sc->sc_dev, CE_WARN,
                "Can't bind req buffer %d", i);
            goto exit;
        }
    }

    return (0);

exit:
    vioblk_free_reqs(sc);
    return (ENOMEM);
}


static int
vioblk_ksupdate(kstat_t *ksp, int rw)
{
    struct vioblk_softc *sc = ksp->ks_private;

    if (rw == KSTAT_WRITE)
        return (EACCES);

    sc->ks_data->sts_rw_cookiesmax.value.ui32 = sc->sc_stats.rw_cookiesmax;
    sc->ks_data->sts_intr_queuemax.value.ui32 = sc->sc_stats.intr_queuemax;
    sc->ks_data->sts_unsupp_errors.value.ui32 = sc->sc_stats.unsupp_errors;
    sc->ks_data->sts_nxio_errors.value.ui32 = sc->sc_stats.nxio_errors;
    sc->ks_data->sts_io_errors.value.ui32 = sc->sc_stats.io_errors;
    sc->ks_data->sts_rw_cacheflush.value.ui64 = sc->sc_stats.rw_cacheflush;
    sc->ks_data->sts_intr_total.value.ui64 = sc->sc_stats.intr_total;


    return (0);
}

static int
vioblk_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
{
    int ret = DDI_SUCCESS;
    int instance;
    struct vioblk_softc *sc;
    struct virtio_softc *vsc;
    struct vioblk_stats *ks_data;

    instance = ddi_get_instance(devinfo);

    switch (cmd) {
    case DDI_ATTACH:
        break;

    case DDI_RESUME:
    case DDI_PM_RESUME:
        dev_err(devinfo, CE_WARN, "resume not supported yet");
        ret = DDI_FAILURE;
        goto exit;

    default:
        dev_err(devinfo, CE_WARN, "cmd 0x%x not recognized", cmd);
        ret = DDI_FAILURE;
        goto exit;
    }

    sc = kmem_zalloc(sizeof (struct vioblk_softc), KM_SLEEP);
    ddi_set_driver_private(devinfo, sc);

    vsc = &sc->sc_virtio;

    /* Duplicate for faster access / less typing */
    sc->sc_dev = devinfo;
    vsc->sc_dev = devinfo;

    cv_init(&sc->cv_devid, NULL, CV_DRIVER, NULL);
    mutex_init(&sc->lock_devid, NULL, MUTEX_DRIVER, NULL);

    /*
     * Initialize interrupt kstat.  This should not normally fail, since
     * we don't use a persistent stat.  We do it this way to avoid having
     * to test for it at run time on the hot path.
     */
    sc->sc_intrstat = kstat_create("vioblk", instance,
        "intrs", "controller", KSTAT_TYPE_NAMED,
        sizeof (struct vioblk_stats) / sizeof (kstat_named_t),
        KSTAT_FLAG_PERSISTENT);
    if (sc->sc_intrstat == NULL) {
        dev_err(devinfo, CE_WARN, "kstat_create failed");
        goto exit_intrstat;
    }
    ks_data = (struct vioblk_stats *)sc->sc_intrstat->ks_data;
    kstat_named_init(&ks_data->sts_rw_outofmemory,
        "total_rw_outofmemory", KSTAT_DATA_UINT64);
    kstat_named_init(&ks_data->sts_rw_badoffset,
        "total_rw_badoffset", KSTAT_DATA_UINT64);
    kstat_named_init(&ks_data->sts_intr_total,
        "total_intr", KSTAT_DATA_UINT64);
    kstat_named_init(&ks_data->sts_io_errors,
        "total_io_errors", KSTAT_DATA_UINT32);
    kstat_named_init(&ks_data->sts_unsupp_errors,
        "total_unsupp_errors", KSTAT_DATA_UINT32);
    kstat_named_init(&ks_data->sts_nxio_errors,
        "total_nxio_errors", KSTAT_DATA_UINT32);
    kstat_named_init(&ks_data->sts_rw_cacheflush,
        "total_rw_cacheflush", KSTAT_DATA_UINT64);
    kstat_named_init(&ks_data->sts_rw_cookiesmax,
        "max_rw_cookies", KSTAT_DATA_UINT32);
    kstat_named_init(&ks_data->sts_intr_queuemax,
        "max_intr_queue", KSTAT_DATA_UINT32);
    sc->ks_data = ks_data;
    sc->sc_intrstat->ks_private = sc;
    sc->sc_intrstat->ks_update = vioblk_ksupdate;
    kstat_install(sc->sc_intrstat);

    /* map BAR0 */
    ret = ddi_regs_map_setup(devinfo, 1,
        (caddr_t *)&sc->sc_virtio.sc_io_addr,
        0, 0, &vioblk_attr, &sc->sc_virtio.sc_ioh);
    if (ret != DDI_SUCCESS) {
        dev_err(devinfo, CE_WARN, "unable to map bar0: [%d]", ret);
        goto exit_map;
    }

    virtio_device_reset(&sc->sc_virtio);
    virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
    virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);

    if (vioblk_register_ints(sc)) {
        dev_err(devinfo, CE_WARN, "Unable to add interrupt");
        goto exit_int;
    }

    ret = vioblk_dev_features(sc);
    if (ret)
        goto exit_features;

    if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_RO)
        sc->sc_readonly = B_TRUE;
    else
        sc->sc_readonly = B_FALSE;

    sc->sc_capacity = virtio_read_device_config_8(&sc->sc_virtio,
        VIRTIO_BLK_CONFIG_CAPACITY);
    sc->sc_nblks = sc->sc_capacity;

    sc->sc_blk_size = DEV_BSIZE;
    if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_BLK_SIZE) {
        sc->sc_blk_size = virtio_read_device_config_4(&sc->sc_virtio,
            VIRTIO_BLK_CONFIG_BLK_SIZE);
    }

    sc->sc_pblk_size = sc->sc_blk_size;
    if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_TOPOLOGY) {
        sc->sc_pblk_size <<= virtio_read_device_config_1(&sc->sc_virtio,
            VIRTIO_BLK_CONFIG_TOPO_PBEXP);
    }

    /* Flushing is not supported. */
    if (!(sc->sc_virtio.sc_features & VIRTIO_BLK_F_FLUSH)) {
        vioblk_ops.o_sync_cache = NULL;
    }

    sc->sc_seg_max = DEF_MAXINDIRECT;
    /* The max number of segments (cookies) in a request */
    if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_SEG_MAX) {
        sc->sc_seg_max = virtio_read_device_config_4(&sc->sc_virtio,
            VIRTIO_BLK_CONFIG_SEG_MAX);

        /* That's what Linux does. */
        if (!sc->sc_seg_max)
            sc->sc_seg_max = 1;

        /*
         * SEG_MAX corresponds to the number of _data_
         * blocks in a request
         */
        sc->sc_seg_max += 2;
    }
    /* 2 descriptors taken for header/status */
    vioblk_bd_dma_attr.dma_attr_sgllen = sc->sc_seg_max - 2;


    /* The maximum size for a cookie in a request. */
    sc->sc_seg_size_max = DEF_MAXSECTOR;
    if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_SIZE_MAX) {
        sc->sc_seg_size_max = virtio_read_device_config_4(
            &sc->sc_virtio, VIRTIO_BLK_CONFIG_SIZE_MAX);
    }

    /* The maximum request size */
    vioblk_bd_dma_attr.dma_attr_maxxfer =
        vioblk_bd_dma_attr.dma_attr_sgllen * sc->sc_seg_size_max;

    dev_debug(devinfo, CE_NOTE,
        "nblks=%" PRIu64 " blksize=%d (%d) num_seg=%d, "
        "seg_size=%d, maxxfer=%" PRIu64,
        sc->sc_nblks, sc->sc_blk_size, sc->sc_pblk_size,
        vioblk_bd_dma_attr.dma_attr_sgllen,
        sc->sc_seg_size_max,
        vioblk_bd_dma_attr.dma_attr_maxxfer);


    sc->sc_vq = virtio_alloc_vq(&sc->sc_virtio, 0, 0,
        sc->sc_seg_max, "I/O request");
    if (sc->sc_vq == NULL) {
        goto exit_alloc1;
    }

    ret = vioblk_alloc_reqs(sc);
    if (ret) {
        goto exit_alloc2;
    }

    sc->bd_h = bd_alloc_handle(sc, &vioblk_ops, &vioblk_bd_dma_attr,
        KM_SLEEP);


    virtio_set_status(&sc->sc_virtio,
        VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
    virtio_start_vq_intr(sc->sc_vq);

    ret = virtio_enable_ints(&sc->sc_virtio);
    if (ret)
        goto exit_enable_ints;

    ret = bd_attach_handle(devinfo, sc->bd_h);
    if (ret != DDI_SUCCESS) {
        dev_err(devinfo, CE_WARN, "Failed to attach blkdev");
        goto exit_attach_bd;
    }

    return (DDI_SUCCESS);

exit_attach_bd:
    /*
     * There is no virtio_disable_ints(), it's done in virtio_release_ints.
     * If they ever get split, don't forget to add a call here.
     */
exit_enable_ints:
    virtio_stop_vq_intr(sc->sc_vq);
    bd_free_handle(sc->bd_h);
    vioblk_free_reqs(sc);
exit_alloc2:
    virtio_free_vq(sc->sc_vq);
exit_alloc1:
exit_features:
    virtio_release_ints(&sc->sc_virtio);
exit_int:
    virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
    ddi_regs_map_free(&sc->sc_virtio.sc_ioh);
exit_map:
    kstat_delete(sc->sc_intrstat);
exit_intrstat:
    mutex_destroy(&sc->lock_devid);
    cv_destroy(&sc->cv_devid);
    kmem_free(sc, sizeof (struct vioblk_softc));
exit:
    return (ret);
}

static int
vioblk_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
{
    struct vioblk_softc *sc = ddi_get_driver_private(devinfo);

    switch (cmd) {
    case DDI_DETACH:
        break;

    case DDI_PM_SUSPEND:
        cmn_err(CE_WARN, "suspend not supported yet");
        return (DDI_FAILURE);

    default:
        cmn_err(CE_WARN, "cmd 0x%x unrecognized", cmd);
        return (DDI_FAILURE);
    }

    (void) bd_detach_handle(sc->bd_h);
    virtio_stop_vq_intr(sc->sc_vq);
    virtio_release_ints(&sc->sc_virtio);
    vioblk_free_reqs(sc);
    virtio_free_vq(sc->sc_vq);
    virtio_device_reset(&sc->sc_virtio);
    ddi_regs_map_free(&sc->sc_virtio.sc_ioh);
    kstat_delete(sc->sc_intrstat);
    kmem_free(sc, sizeof (struct vioblk_softc));

    return (DDI_SUCCESS);
}

static int
vioblk_quiesce(dev_info_t *devinfo)
{
    struct vioblk_softc *sc = ddi_get_driver_private(devinfo);

    virtio_stop_vq_intr(sc->sc_vq);
    virtio_device_reset(&sc->sc_virtio);

    return (DDI_SUCCESS);
}

int
_init(void)
{
    int rv;

    bd_mod_init(&vioblk_dev_ops);

    if ((rv = mod_install(&modlinkage)) != 0) {
        bd_mod_fini(&vioblk_dev_ops);
    }

    return (rv);
}

int
_fini(void)
{
    int rv;

    if ((rv = mod_remove(&modlinkage)) == 0) {
        bd_mod_fini(&vioblk_dev_ops);
    }

    return (rv);
}

int
_info(struct modinfo *modinfop)
{
    return (mod_info(&modlinkage, modinfop));
}