io/virtio/virtio.c

e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * CDDL HEADER START
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev *
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * The contents of this file are subject to the terms of the
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Common Development and Distribution License (the "License").
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * You may not use this file except in compliance with the License.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev *
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * or http://www.opensolaris.org/os/licensing.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * See the License for the specific language governing permissions
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * and limitations under the License.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev *
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * When distributing Covered Code, include this CDDL HEADER in each
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * If applicable, add the following below this CDDL HEADER, with the
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * fields enclosed by brackets "[]" replaced with your own identifying
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * information: Portions Copyright [yyyy] [name of copyright owner]
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev *
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * CDDL HEADER END
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Copyright 2012 Alexey Zaytsev <alexey.zaytsev@gmail.com>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/* Based on the NetBSD virtio driver by Minoura Makoto. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Copyright (c) 2010 Minoura Makoto.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * All rights reserved.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev *
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Redistribution and use in source and binary forms, with or without
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * modification, are permitted provided that the following conditions
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * are met:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * 1. Redistributions of source code must retain the above copyright
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev *    notice, this list of conditions and the following disclaimer.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * 2. Redistributions in binary form must reproduce the above copyright
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev *    notice, this list of conditions and the following disclaimer in the
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev *    documentation and/or other materials provided with the distribution.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev *
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev *
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/conf.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/kmem.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/debug.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/modctl.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/autoconf.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/ddi_impldefs.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/ddi.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/sunddi.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/sunndi.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/avintr.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/spl.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/promif.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/list.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/bootconf.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/bootsvcs.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/sysmacros.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include <sys/pci.h>
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include "virtiovar.h"
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#include "virtioreg.h"
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#define NDEVNAMES   (sizeof (virtio_device_name) / sizeof (char *))
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#define MINSEG_INDIRECT 2   /* use indirect if nsegs >= this value */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev#define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1)) & \
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ~(VIRTIO_PAGE_SIZE-1))
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_set_status(struct virtio_softc *sc, unsigned int status)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int old = 0;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    if (status != 0) {
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        old = ddi_get8(sc->sc_ioh, (uint8_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_CONFIG_DEVICE_STATUS));
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    ddi_put8(sc->sc_ioh, (uint8_t *)(sc->sc_io_addr +
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        VIRTIO_CONFIG_DEVICE_STATUS), status | old);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Negotiate features, save the result in sc->sc_features
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevuint32_t
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    uint32_t host_features;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    uint32_t features;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    host_features = ddi_get32(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_DEVICE_FEATURES));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    dev_debug(sc->sc_dev, CE_NOTE, "host features: %x, guest features: %x",
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        host_features, guest_features);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    features = host_features & guest_features;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put32(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_GUEST_FEATURES),
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        features);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    sc->sc_features = features;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (host_features);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevsize_t
8a324c92216752a3ac2de7c31f554588932de707Dan McDonaldvirtio_show_features(uint32_t features, char *buf, size_t len)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    char *orig_buf = buf;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    char *bufend = buf + len;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* LINTED E_PTRDIFF_OVERFLOW */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    buf += snprintf(buf, bufend - buf, "Generic ( ");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (features & VIRTIO_F_RING_INDIRECT_DESC)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_PTRDIFF_OVERFLOW */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        buf += snprintf(buf, bufend - buf, "INDIRECT_DESC ");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* LINTED E_PTRDIFF_OVERFLOW */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    buf += snprintf(buf, bufend - buf, ") ");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* LINTED E_PTRDIFF_OVERFLOW */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (buf - orig_buf);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevboolean_t
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_has_feature(struct virtio_softc *sc, uint32_t feature)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (sc->sc_features & feature);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Device configuration registers.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevuint8_t
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_read_device_config_1(struct virtio_softc *sc, unsigned int index)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(sc->sc_config_offset);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return ddi_get8(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint8_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevuint16_t
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_read_device_config_2(struct virtio_softc *sc, unsigned int index)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(sc->sc_config_offset);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return ddi_get16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint16_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevuint32_t
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_read_device_config_4(struct virtio_softc *sc, unsigned int index)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(sc->sc_config_offset);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return ddi_get32(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevuint64_t
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_read_device_config_8(struct virtio_softc *sc, unsigned int index)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    uint64_t r;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(sc->sc_config_offset);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    r = ddi_get32(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        index + sizeof (uint32_t)));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    r <<= 32;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    r += ddi_get32(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (r);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
8a324c92216752a3ac2de7c31f554588932de707Dan McDonaldvirtio_write_device_config_1(struct virtio_softc *sc, unsigned int index,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    uint8_t value)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(sc->sc_config_offset);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put8(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint8_t *)(sc->sc_io_addr + sc->sc_config_offset + index), value);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
8a324c92216752a3ac2de7c31f554588932de707Dan McDonaldvirtio_write_device_config_2(struct virtio_softc *sc, unsigned int index,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    uint16_t value)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(sc->sc_config_offset);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint16_t *)(sc->sc_io_addr + sc->sc_config_offset + index), value);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
8a324c92216752a3ac2de7c31f554588932de707Dan McDonaldvirtio_write_device_config_4(struct virtio_softc *sc, unsigned int index,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    uint32_t value)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(sc->sc_config_offset);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put32(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index), value);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
8a324c92216752a3ac2de7c31f554588932de707Dan McDonaldvirtio_write_device_config_8(struct virtio_softc *sc, unsigned int index,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    uint64_t value)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(sc->sc_config_offset);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put32(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index),
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        value & 0xFFFFFFFF);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put32(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        index + sizeof (uint32_t)), value >> 32);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Start/stop vq interrupt.  No guarantee.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_stop_vq_intr(struct virtqueue *vq)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_start_vq_intr(struct virtqueue *vq)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic ddi_dma_attr_t virtio_vq_dma_attr = {
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    DMA_ATTR_V0,        /* Version number */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0,          /* low address */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0x00000FFFFFFFFFFF, /* high address. Has to fit into 32 bits */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald                /* after page-shifting */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0xFFFFFFFF,     /* counter register max */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    VIRTIO_PAGE_SIZE,   /* page alignment required */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0x3F,           /* burst sizes: 1 - 32 */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0x1,            /* minimum transfer size */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0xFFFFFFFF,     /* max transfer size */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0xFFFFFFFF,     /* address register max */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    1,          /* no scatter-gather */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    1,          /* device operates on bytes */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0,          /* attr flag: set to 0 */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev};
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic ddi_dma_attr_t virtio_vq_indirect_dma_attr = {
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    DMA_ATTR_V0,        /* Version number */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0,          /* low address */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0xFFFFFFFFFFFFFFFF, /* high address */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0xFFFFFFFF,     /* counter register max */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    1,          /* No specific alignment */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0x3F,           /* burst sizes: 1 - 32 */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0x1,            /* minimum transfer size */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0xFFFFFFFF,     /* max transfer size */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0xFFFFFFFF,     /* address register max */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    1,          /* no scatter-gather */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    1,          /* device operates on bytes */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    0,          /* attr flag: set to 0 */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev};
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/* Same for direct and indirect descriptors. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic ddi_device_acc_attr_t virtio_vq_devattr = {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    DDI_DEVICE_ATTR_V0,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    DDI_NEVERSWAP_ACC,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    DDI_STORECACHING_OK_ACC,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    DDI_DEFAULT_ACC
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev};
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic void
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_free_indirect(struct vq_entry *entry)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    (void) ddi_dma_unbind_handle(entry->qe_indirect_dma_handle);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_dma_mem_free(&entry->qe_indirect_dma_acch);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_dma_free_handle(&entry->qe_indirect_dma_handle);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    entry->qe_indirect_descs = NULL;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic int
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_alloc_indirect(struct virtio_softc *sc, struct vq_entry *entry)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int allocsize, num;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    size_t len;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    unsigned int ncookies;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int ret;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    num = entry->qe_queue->vq_indirect_num;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(num > 1);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    allocsize = sizeof (struct vring_desc) * num;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_dma_alloc_handle(sc->sc_dev, &virtio_vq_indirect_dma_attr,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        DDI_DMA_SLEEP, NULL, &entry->qe_indirect_dma_handle);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald            "Failed to allocate dma handle for indirect descriptors, "
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald            "entry %d, vq %d", entry->qe_index,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            entry->qe_queue->vq_index);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_alloc_handle;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    ret = ddi_dma_mem_alloc(entry->qe_indirect_dma_handle, allocsize,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        &virtio_vq_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (caddr_t *)&entry->qe_indirect_descs, &len,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        &entry->qe_indirect_dma_acch);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald            "Failed to allocate dma memory for indirect descriptors, "
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald            "entry %d, vq %d,", entry->qe_index,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            entry->qe_queue->vq_index);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_alloc;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    (void) memset(entry->qe_indirect_descs, 0xff, allocsize);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_dma_addr_bind_handle(entry->qe_indirect_dma_handle, NULL,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (caddr_t)entry->qe_indirect_descs, len,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        &entry->qe_indirect_dma_cookie, &ncookies);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_DMA_MAPPED) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald            "Failed to bind dma memory for indirect descriptors, "
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            "entry %d, vq %d", entry->qe_index,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            entry->qe_queue->vq_index);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_bind;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* We asked for a single segment */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(ncookies == 1);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (0);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_bind:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_dma_mem_free(&entry->qe_indirect_dma_acch);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_alloc:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_dma_free_handle(&entry->qe_indirect_dma_handle);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_alloc_handle:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (ret);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Initialize the vq structure.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic int
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int ret;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    uint16_t i;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int vq_size = vq->vq_num;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int indirect_num = vq->vq_indirect_num;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* free slot management */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    list_create(&vq->vq_freelist, sizeof (struct vq_entry),
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        offsetof(struct vq_entry, qe_list));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; i < vq_size; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        struct vq_entry *entry = &vq->vq_entries[i];
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        list_insert_tail(&vq->vq_freelist, entry);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        entry->qe_index = i;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        entry->qe_desc = &vq->vq_descs[i];
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        entry->qe_queue = vq;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (indirect_num) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            ret = virtio_alloc_indirect(sc, entry);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            if (ret)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                goto out_indirect;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    mutex_init(&vq->vq_freelist_lock, "virtio-freelist", MUTEX_DRIVER,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        DDI_INTR_PRI(sc->sc_intr_prio));
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    mutex_init(&vq->vq_avail_lock, "virtio-avail", MUTEX_DRIVER,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        DDI_INTR_PRI(sc->sc_intr_prio));
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    mutex_init(&vq->vq_used_lock, "virtio-used", MUTEX_DRIVER,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        DDI_INTR_PRI(sc->sc_intr_prio));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (0);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_indirect:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; i < vq_size; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        struct vq_entry *entry = &vq->vq_entries[i];
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (entry->qe_indirect_descs)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            virtio_free_indirect(entry);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (ret);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Allocate/free a vq.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstruct virtqueue *
8a324c92216752a3ac2de7c31f554588932de707Dan McDonaldvirtio_alloc_vq(struct virtio_softc *sc, unsigned int index, unsigned int size,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    unsigned int indirect_num, const char *name)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int vq_size, allocsize1, allocsize2, allocsize = 0;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int ret;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    unsigned int ncookies;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    size_t len;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtqueue *vq;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SELECT), index);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq_size = ddi_get16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SIZE));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (vq_size == 0) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            "virtqueue dest not exist, index %d for %s\n", index, name);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq = kmem_zalloc(sizeof (struct virtqueue), KM_SLEEP);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* size 0 => use native vq size, good for receive queues. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (size)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vq_size = MIN(vq_size, size);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* allocsize1: descriptor table + avail ring + pad */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    allocsize1 = VIRTQUEUE_ALIGN(sizeof (struct vring_desc) * vq_size +
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        sizeof (struct vring_avail) + sizeof (uint16_t) * vq_size);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* allocsize2: used ring + pad */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    allocsize2 = VIRTQUEUE_ALIGN(sizeof (struct vring_used) +
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        sizeof (struct vring_used_elem) * vq_size);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    allocsize = allocsize1 + allocsize2;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_dma_alloc_handle(sc->sc_dev, &virtio_vq_dma_attr,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        DDI_DMA_SLEEP, NULL, &vq->vq_dma_handle);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            "Failed to allocate dma handle for vq %d", index);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_alloc_handle;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_dma_mem_alloc(vq->vq_dma_handle, allocsize,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        &virtio_vq_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (caddr_t *)&vq->vq_vaddr, &len, &vq->vq_dma_acch);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald            "Failed to allocate dma memory for vq %d", index);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_alloc;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_dma_addr_bind_handle(vq->vq_dma_handle, NULL,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        (caddr_t)vq->vq_vaddr, len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        DDI_DMA_SLEEP, NULL, &vq->vq_dma_cookie, &ncookies);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_DMA_MAPPED) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            "Failed to bind dma memory for vq %d", index);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_bind;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* We asked for a single segment */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(ncookies == 1);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* and page-ligned buffers. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(vq->vq_dma_cookie.dmac_laddress % VIRTIO_PAGE_SIZE == 0);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    (void) memset(vq->vq_vaddr, 0, allocsize);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Make sure all zeros hit the buffer before we point the host to it */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    membar_producer();
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* set the vq address */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put32(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_ADDRESS),
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (vq->vq_dma_cookie.dmac_laddress / VIRTIO_PAGE_SIZE));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* remember addresses and offsets for later use */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_owner = sc;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_num = vq_size;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_index = index;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_descs = vq->vq_vaddr;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_availoffset = sizeof (struct vring_desc)*vq_size;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_avail = (void *)(((char *)vq->vq_descs) + vq->vq_availoffset);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_usedoffset = allocsize1;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_used = (void *)(((char *)vq->vq_descs) + vq->vq_usedoffset);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(indirect_num == 0 ||
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        virtio_has_feature(sc, VIRTIO_F_RING_INDIRECT_DESC));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_indirect_num = indirect_num;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* free slot management */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_entries = kmem_zalloc(sizeof (struct vq_entry) * vq_size,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        KM_SLEEP);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = virtio_init_vq(sc, vq);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_init;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    dev_debug(sc->sc_dev, CE_NOTE,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        "Allocated %d entries for vq %d:%s (%d indirect descs)",
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vq_size, index, name, indirect_num * vq_size);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (vq);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_init:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    kmem_free(vq->vq_entries, sizeof (struct vq_entry) * vq_size);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    (void) ddi_dma_unbind_handle(vq->vq_dma_handle);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_bind:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_dma_mem_free(&vq->vq_dma_acch);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_alloc:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_dma_free_handle(&vq->vq_dma_handle);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_alloc_handle:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    kmem_free(vq, sizeof (struct virtqueue));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (NULL);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_free_vq(struct virtqueue *vq)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_softc *sc = vq->vq_owner;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int i;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* tell device that there's no virtqueue any longer */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SELECT),
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vq->vq_index);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put32(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_ADDRESS), 0);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Free the indirect descriptors, if any. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; i < vq->vq_num; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        struct vq_entry *entry = &vq->vq_entries[i];
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (entry->qe_indirect_descs)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            virtio_free_indirect(entry);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    kmem_free(vq->vq_entries, sizeof (struct vq_entry) * vq->vq_num);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    (void) ddi_dma_unbind_handle(vq->vq_dma_handle);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_dma_mem_free(&vq->vq_dma_acch);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_dma_free_handle(&vq->vq_dma_handle);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_destroy(&vq->vq_used_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_destroy(&vq->vq_avail_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_destroy(&vq->vq_freelist_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    kmem_free(vq, sizeof (struct virtqueue));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Free descriptor management.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstruct vq_entry *
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvq_alloc_entry(struct virtqueue *vq)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct vq_entry *qe;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_enter(&vq->vq_freelist_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (list_is_empty(&vq->vq_freelist)) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        mutex_exit(&vq->vq_freelist_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        return (NULL);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    qe = list_remove_head(&vq->vq_freelist);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(vq->vq_used_entries >= 0);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_used_entries++;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_exit(&vq->vq_freelist_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    qe->qe_next = NULL;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    qe->qe_indirect_next = 0;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    (void) memset(qe->qe_desc, 0, sizeof (struct vring_desc));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (qe);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvq_free_entry(struct virtqueue *vq, struct vq_entry *qe)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_enter(&vq->vq_freelist_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    list_insert_head(&vq->vq_freelist, qe);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_used_entries--;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(vq->vq_used_entries >= 0);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_exit(&vq->vq_freelist_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * We (intentionally) don't have a global vq mutex, so you are
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * responsible for external locking to avoid allocting/freeing any
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * entries before using the returned value. Have fun.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevuint_t
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvq_num_used(struct virtqueue *vq)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* vq->vq_freelist_lock would not help here. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (vq->vq_used_entries);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic inline void
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_ve_set_desc(struct vring_desc *desc, uint64_t paddr, uint32_t len,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    boolean_t write)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    desc->addr = paddr;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    desc->len = len;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    desc->next = 0;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    desc->flags = 0;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* 'write' - from the driver's point of view */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (!write)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        desc->flags = VRING_DESC_F_WRITE;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_ve_set(struct vq_entry *qe, uint64_t paddr, uint32_t len,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    boolean_t write)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    virtio_ve_set_desc(qe->qe_desc, paddr, len, write);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonaldunsigned int
8a324c92216752a3ac2de7c31f554588932de707Dan McDonaldvirtio_ve_indirect_available(struct vq_entry *qe)
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald{
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    return (qe->qe_queue->vq_indirect_num - (qe->qe_indirect_next - 1));
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald}
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_ve_add_indirect_buf(struct vq_entry *qe, uint64_t paddr, uint32_t len,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    boolean_t write)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct vring_desc *indirect_desc;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(qe->qe_queue->vq_indirect_num);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(qe->qe_indirect_next < qe->qe_queue->vq_indirect_num);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    indirect_desc = &qe->qe_indirect_descs[qe->qe_indirect_next];
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    virtio_ve_set_desc(indirect_desc, paddr, len, write);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    qe->qe_indirect_next++;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_ve_add_cookie(struct vq_entry *qe, ddi_dma_handle_t dma_handle,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_dma_cookie_t dma_cookie, unsigned int ncookies, boolean_t write)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int i;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; i < ncookies; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        virtio_ve_add_indirect_buf(qe, dma_cookie.dmac_laddress,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            dma_cookie.dmac_size, write);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ddi_dma_nextcookie(dma_handle, &dma_cookie);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_sync_vq(struct virtqueue *vq)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_softc *vsc = vq->vq_owner;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Make sure the avail ring update hit the buffer */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    membar_producer();
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_avail->idx = vq->vq_avail_idx;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Make sure the avail idx update hits the buffer */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    membar_producer();
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Make sure we see the flags update */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    membar_consumer();
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ddi_put16(vsc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            (uint16_t *)(vsc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_CONFIG_QUEUE_NOTIFY),
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            vq->vq_index);
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_push_chain(struct vq_entry *qe, boolean_t sync)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtqueue *vq = qe->qe_queue;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct vq_entry *head = qe;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct vring_desc *desc;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int idx;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(qe);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev     * Bind the descs together, paddr and len should be already
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev     * set with virtio_ve_set
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev     */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    do {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* Bind the indirect descriptors */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (qe->qe_indirect_next > 1) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            uint16_t i = 0;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev             * Set the pointer/flags to the
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev             * first indirect descriptor
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev             */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            virtio_ve_set_desc(qe->qe_desc,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                qe->qe_indirect_dma_cookie.dmac_laddress,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                sizeof (struct vring_desc) * qe->qe_indirect_next,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                B_FALSE);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            qe->qe_desc->flags |= VRING_DESC_F_INDIRECT;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* For all but the last one, add the next index/flag */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            do {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                desc = &qe->qe_indirect_descs[i];
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                i++;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                desc->flags |= VRING_DESC_F_NEXT;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                desc->next = i;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            } while (i < qe->qe_indirect_next - 1);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (qe->qe_next) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            qe->qe_desc->flags |= VRING_DESC_F_NEXT;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            qe->qe_desc->next = qe->qe_next->qe_index;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        qe = qe->qe_next;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    } while (qe);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_enter(&vq->vq_avail_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    idx = vq->vq_avail_idx;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_avail_idx++;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Make sure the bits hit the descriptor(s) */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    membar_producer();
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_avail->ring[idx % vq->vq_num] = head->qe_index;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Notify the device, if needed. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (sync)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        virtio_sync_vq(vq);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_exit(&vq->vq_avail_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald/*
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald * Get a chain of descriptors from the used ring, if one is available.
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstruct vq_entry *
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_pull_chain(struct virtqueue *vq, uint32_t *len)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct vq_entry *head;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int slot;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int usedidx;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_enter(&vq->vq_used_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* No used entries? Bye. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (vq->vq_used_idx == vq->vq_used->idx) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        mutex_exit(&vq->vq_used_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        return (NULL);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    usedidx = vq->vq_used_idx;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vq->vq_used_idx++;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    mutex_exit(&vq->vq_used_lock);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    usedidx %= vq->vq_num;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Make sure we do the next step _after_ checking the idx. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    membar_consumer();
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    slot = vq->vq_used->ring[usedidx].id;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    *len = vq->vq_used->ring[usedidx].len;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    head = &vq->vq_entries[slot];
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (head);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_free_chain(struct vq_entry *qe)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct vq_entry *tmp;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtqueue *vq = qe->qe_queue;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(qe);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    do {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ASSERT(qe->qe_queue == vq);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        tmp = qe->qe_next;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vq_free_entry(vq, qe);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        qe = tmp;
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    } while (tmp != NULL);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_ventry_stick(struct vq_entry *first, struct vq_entry *second)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    first->qe_next = second;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic int
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_register_msi(struct virtio_softc *sc,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_int_handler *config_handler,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    struct virtio_int_handler vq_handlers[], int intr_types)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int count, actual;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int int_type;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int i;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int handler_count;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int ret;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* If both MSI and MSI-x are reported, prefer MSI-x. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int_type = DDI_INTR_TYPE_MSI;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (intr_types & DDI_INTR_TYPE_MSIX)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        int_type = DDI_INTR_TYPE_MSIX;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Walk the handler table to get the number of handlers. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (handler_count = 0;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vq_handlers && vq_handlers[handler_count].vh_func;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        handler_count++)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* +1 if there is a config change handler. */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    if (config_handler != NULL)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        handler_count++;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Number of MSIs supported by the device. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_intr_get_nintrs(sc->sc_dev, int_type, &count);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN, "ddi_intr_get_nintrs failed");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        return (ret);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev     * Those who try to register more handlers then the device
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev     * supports shall suffer.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev     */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(handler_count <= count);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    sc->sc_intr_htable = kmem_zalloc(sizeof (ddi_intr_handle_t) *
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        handler_count, KM_SLEEP);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_intr_alloc(sc->sc_dev, sc->sc_intr_htable, int_type, 0,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        handler_count, &actual, DDI_INTR_ALLOC_NORMAL);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN, "Failed to allocate MSI: %d", ret);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_msi_alloc;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (actual != handler_count) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            "Not enough MSI available: need %d, available %d",
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            handler_count, actual);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_msi_available;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    sc->sc_intr_num = handler_count;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    sc->sc_intr_config = B_FALSE;
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    if (config_handler != NULL) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        sc->sc_intr_config = B_TRUE;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Assume they are all same priority */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_intr_get_pri(sc->sc_intr_htable[0], &sc->sc_intr_prio);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN, "ddi_intr_get_pri failed");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_msi_prio;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Add the vq handlers */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; vq_handlers[i].vh_func; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ret = ddi_intr_add_handler(sc->sc_intr_htable[i],
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald            vq_handlers[i].vh_func, sc, vq_handlers[i].vh_priv);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                "ddi_intr_add_handler failed");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* Remove the handlers that succeeded. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            while (--i >= 0) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                (void) ddi_intr_remove_handler(
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                    sc->sc_intr_htable[i]);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            goto out_add_handlers;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Don't forget the config handler */
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    if (config_handler != NULL) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ret = ddi_intr_add_handler(sc->sc_intr_htable[i],
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald            config_handler->vh_func, sc, config_handler->vh_priv);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                "ddi_intr_add_handler failed");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* Remove the handlers that succeeded. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            while (--i >= 0) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                (void) ddi_intr_remove_handler(
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                    sc->sc_intr_htable[i]);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            goto out_add_handlers;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    ret = ddi_intr_get_cap(sc->sc_intr_htable[0], &sc->sc_intr_cap);
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    if (ret == DDI_SUCCESS) {
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        sc->sc_int_type = int_type;
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        return (DDI_SUCCESS);
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_add_handlers:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_msi_prio:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_msi_available:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; i < actual; i++)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (void) ddi_intr_free(sc->sc_intr_htable[i]);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_msi_alloc:
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    kmem_free(sc->sc_intr_htable,
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        sizeof (ddi_intr_handle_t) * handler_count);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (ret);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstruct virtio_handler_container {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int nhandlers;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_int_handler config_handler;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_int_handler vq_handlers[];
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev};
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevuint_t
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_intx_dispatch(caddr_t arg1, caddr_t arg2)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_softc *sc = (void *)arg1;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_handler_container *vhc = (void *)arg2;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    uint8_t isr_status;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int i;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    isr_status = ddi_get8(sc->sc_ioh, (uint8_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        VIRTIO_CONFIG_ISR_STATUS));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (!isr_status)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        return (DDI_INTR_UNCLAIMED);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if ((isr_status & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vhc->config_handler.vh_func) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vhc->config_handler.vh_func((void *)sc,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            vhc->config_handler.vh_priv);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Notify all handlers */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; i < vhc->nhandlers; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vhc->vq_handlers[i].vh_func((void *)sc,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            vhc->vq_handlers[i].vh_priv);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (DDI_INTR_CLAIMED);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * config_handler and vq_handlers may be allocated on stack.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Take precautions not to loose them.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic int
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_register_intx(struct virtio_softc *sc,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_int_handler *config_handler,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_int_handler vq_handlers[])
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int vq_handler_count;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int config_handler_count = 0;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int actual;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_handler_container *vhc;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int ret = DDI_FAILURE;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Walk the handler table to get the number of handlers. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (vq_handler_count = 0;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vq_handlers && vq_handlers[vq_handler_count].vh_func;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vq_handler_count++)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    if (config_handler != NULL)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        config_handler_count = 1;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vhc = kmem_zalloc(sizeof (struct virtio_handler_container) +
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        sizeof (struct virtio_int_handler) * vq_handler_count, KM_SLEEP);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    vhc->nhandlers = vq_handler_count;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    (void) memcpy(vhc->vq_handlers, vq_handlers,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        sizeof (struct virtio_int_handler) * vq_handler_count);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    if (config_handler != NULL) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (void) memcpy(&vhc->config_handler, config_handler,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            sizeof (struct virtio_int_handler));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Just a single entry for a single interrupt. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    sc->sc_intr_htable = kmem_zalloc(sizeof (ddi_intr_handle_t), KM_SLEEP);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_intr_alloc(sc->sc_dev, sc->sc_intr_htable,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        DDI_INTR_TYPE_FIXED, 0, 1, &actual, DDI_INTR_ALLOC_NORMAL);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            "Failed to allocate a fixed interrupt: %d", ret);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_int_alloc;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ASSERT(actual == 1);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    sc->sc_intr_num = 1;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_intr_get_pri(sc->sc_intr_htable[0], &sc->sc_intr_prio);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN, "ddi_intr_get_pri failed");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_prio;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_intr_add_handler(sc->sc_intr_htable[0],
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        virtio_intx_dispatch, sc, vhc);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN, "ddi_intr_add_handler failed");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_add_handlers;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    sc->sc_int_type = DDI_INTR_TYPE_FIXED;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (DDI_SUCCESS);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_add_handlers:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_prio:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    (void) ddi_intr_free(sc->sc_intr_htable[0]);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_int_alloc:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    kmem_free(sc->sc_intr_htable, sizeof (ddi_intr_handle_t));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    kmem_free(vhc, sizeof (struct virtio_int_handler) *
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (vq_handler_count + config_handler_count));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (ret);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * We find out if we support MSI during this, and the register layout
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * depends on the MSI (doh). Don't acces the device specific bits in
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * BAR 0 before calling it!
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevint
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_register_ints(struct virtio_softc *sc,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_int_handler *config_handler,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    struct virtio_int_handler vq_handlers[])
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int ret;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int intr_types;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    /* Default offset until MSI-X is enabled, if ever. */
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSIX;
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Determine which types of interrupts are supported */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_intr_get_supported_types(sc->sc_dev, &intr_types);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN, "Can't get supported int types");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        goto out_inttype;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* If we have msi, let's use them. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (intr_types & (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ret = virtio_register_msi(sc, config_handler,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            vq_handlers, intr_types);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (!ret)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            return (0);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Fall back to old-fashioned interrupts. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (intr_types & DDI_INTR_TYPE_FIXED) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_debug(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            "Using legacy interrupts");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        return (virtio_register_intx(sc, config_handler, vq_handlers));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        "MSI failed and fixed interrupts not supported. Giving up.");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = DDI_FAILURE;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_inttype:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (ret);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic int
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_enable_msi(struct virtio_softc *sc)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int ret, i;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int vq_handler_count = sc->sc_intr_num;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Number of handlers, not counting the counfig. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (sc->sc_intr_config)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        vq_handler_count--;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    /* Enable the interrupts. Either the whole block, or one by one. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (sc->sc_intr_cap & DDI_INTR_FLAG_BLOCK) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ret = ddi_intr_block_enable(sc->sc_intr_htable,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            sc->sc_intr_num);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                "Failed to enable MSI, falling back to INTx");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            goto out_enable;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    } else {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        for (i = 0; i < sc->sc_intr_num; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            ret = ddi_intr_enable(sc->sc_intr_htable[i]);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                    "Failed to enable MSI %d, "
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                    "falling back to INTx", i);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                while (--i >= 0) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                    (void) ddi_intr_disable(
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                        sc->sc_intr_htable[i]);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                goto out_enable;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Bind the allocated MSI to the queues and config */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; i < vq_handler_count; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        int check;
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ddi_put16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_CONFIG_QUEUE_SELECT), i);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ddi_put16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_CONFIG_QUEUE_VECTOR), i);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        check = ddi_get16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_CONFIG_QUEUE_VECTOR));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (check != i) {
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald            dev_err(sc->sc_dev, CE_WARN, "Failed to bind handler "
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                "for VQ %d, MSI %d. Check = %x", i, i, check);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            ret = ENODEV;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            goto out_bind;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (sc->sc_intr_config) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        int check;
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ddi_put16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_CONFIG_CONFIG_VECTOR), i);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        check = ddi_get16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_CONFIG_CONFIG_VECTOR));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (check != i) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            dev_err(sc->sc_dev, CE_WARN, "Failed to bind handler "
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                "for Config updates, MSI %d", i);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            ret = ENODEV;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            goto out_bind;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    /* Configuration offset depends on whether MSI-X is used. */
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    if (sc->sc_int_type == DDI_INTR_TYPE_MSIX)
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSIX;
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    else
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        ASSERT(sc->sc_int_type == DDI_INTR_TYPE_MSI);
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (DDI_SUCCESS);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_bind:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* Unbind the vqs */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; i < vq_handler_count - 1; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ddi_put16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_CONFIG_QUEUE_SELECT), i);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ddi_put16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_CONFIG_QUEUE_VECTOR),
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_MSI_NO_VECTOR);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* And the config */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ddi_put16(sc->sc_ioh, (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        VIRTIO_CONFIG_CONFIG_VECTOR), VIRTIO_MSI_NO_VECTOR);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    /* Disable the interrupts. Either the whole block, or one by one. */
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    if (sc->sc_intr_cap & DDI_INTR_FLAG_BLOCK) {
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        ret = ddi_intr_block_disable(sc->sc_intr_htable,
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon            sc->sc_intr_num);
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        if (ret != DDI_SUCCESS) {
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon            dev_err(sc->sc_dev, CE_WARN,
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon                "Failed to disable MSIs, won't be able to "
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon                "reuse next time");
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        }
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    } else {
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        for (i = 0; i < sc->sc_intr_num; i++) {
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon            ret = ddi_intr_disable(sc->sc_intr_htable[i]);
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon            if (ret != DDI_SUCCESS) {
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon                dev_err(sc->sc_dev, CE_WARN,
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon                    "Failed to disable interrupt %d, "
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon                    "won't be able to reuse", i);
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon            }
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        }
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    }
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = DDI_FAILURE;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevout_enable:
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (ret);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonaldstatic int
8a324c92216752a3ac2de7c31f554588932de707Dan McDonaldvirtio_enable_intx(struct virtio_softc *sc)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int ret;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    ret = ddi_intr_enable(sc->sc_intr_htable[0]);
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            "Failed to enable interrupt: %d", ret);
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    }
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (ret);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * We can't enable/disable individual handlers in the INTx case so do
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * the whole bunch even in the msi case.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevint
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_enable_ints(struct virtio_softc *sc)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    ASSERT(sc->sc_config_offset == VIRTIO_CONFIG_DEVICE_CONFIG_NOMSIX);
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* See if we are using MSI. */
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    if (sc->sc_int_type == DDI_INTR_TYPE_MSIX ||
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        sc->sc_int_type == DDI_INTR_TYPE_MSI)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        return (virtio_enable_msi(sc));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    ASSERT(sc->sc_int_type == DDI_INTR_TYPE_FIXED);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (virtio_enable_intx(sc));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvoid
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevvirtio_release_ints(struct virtio_softc *sc)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int i;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    int ret;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    /* We were running with MSI, unbind them. */
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    if (sc->sc_int_type == DDI_INTR_TYPE_MSIX ||
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon        sc->sc_int_type == DDI_INTR_TYPE_MSI) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* Unbind all vqs */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        for (i = 0; i < sc->sc_nvqs; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            ddi_put16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                VIRTIO_CONFIG_QUEUE_SELECT), i);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            ddi_put16(sc->sc_ioh,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                VIRTIO_CONFIG_QUEUE_VECTOR),
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                VIRTIO_MSI_NO_VECTOR);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* And the config */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        /* LINTED E_BAD_PTR_CAST_ALIGN */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ddi_put16(sc->sc_ioh, (uint16_t *)(sc->sc_io_addr +
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_CONFIG_CONFIG_VECTOR),
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            VIRTIO_MSI_NO_VECTOR);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    /* Disable the interrupts. Either the whole block, or one by one. */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    if (sc->sc_intr_cap & DDI_INTR_FLAG_BLOCK) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        ret = ddi_intr_block_disable(sc->sc_intr_htable,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            sc->sc_intr_num);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            dev_err(sc->sc_dev, CE_WARN,
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald                "Failed to disable MSIs, won't be able to "
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                "reuse next time");
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    } else {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        for (i = 0; i < sc->sc_intr_num; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            ret = ddi_intr_disable(sc->sc_intr_htable[i]);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            if (ret != DDI_SUCCESS) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                dev_err(sc->sc_dev, CE_WARN,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                    "Failed to disable interrupt %d, "
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev                    "won't be able to reuse", i);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev            }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; i < sc->sc_intr_num; i++) {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (void) ddi_intr_remove_handler(sc->sc_intr_htable[i]);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    for (i = 0; i < sc->sc_intr_num; i++)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (void) ddi_intr_free(sc->sc_intr_htable[i]);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    kmem_free(sc->sc_intr_htable, sizeof (ddi_intr_handle_t) *
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald        sc->sc_intr_num);
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    /* After disabling interrupts, the config offset is non-MSI-X. */
17ad7f9fd28ceea21aea94421cb8ada963285765Andriy Gapon    sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSIX;
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev/*
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev * Module linkage information for the kernel.
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic struct modlmisc modlmisc = {
8a324c92216752a3ac2de7c31f554588932de707Dan McDonald    &mod_miscops,   /* Type of module */
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    "VirtIO common library module",
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev};
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevstatic struct modlinkage modlinkage = {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    MODREV_1,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    {
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        (void *)&modlmisc,
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev        NULL
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    }
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev};
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevint
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev_init(void)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (mod_install(&modlinkage));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevint
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev_fini(void)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (mod_remove(&modlinkage));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsevint
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev_info(struct modinfo *modinfop)
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev{
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev    return (mod_info(&modlinkage, modinfop));
e0724c534a46ca4754330bc022bf1e2a68f5bb93Alexey Zaytsev}