i86pc/io/immu_intrmap.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * Copyright (c) 2009, Intel Corporation.
 * All rights reserved.
 */


#include <sys/apic.h>
#include <vm/hat_i86.h>
#include <sys/sysmacros.h>
#include <sys/smp_impldefs.h>
#include <sys/immu.h>


typedef struct intrmap_private {
    immu_t      *ir_immu;
    immu_inv_wait_t ir_inv_wait;
    uint16_t    ir_idx;
    uint32_t    ir_sid_svt_sq;
} intrmap_private_t;

#define INTRMAP_PRIVATE(intrmap) ((intrmap_private_t *)intrmap)

/* interrupt remapping table entry */
typedef struct intrmap_rte {
    uint64_t    lo;
    uint64_t    hi;
} intrmap_rte_t;

#define IRTE_HIGH(sid_svt_sq) (sid_svt_sq)
#define IRTE_LOW(dst, vector, dlm, tm, rh, dm, fpd, p)  \
        (((uint64_t)(dst) << 32) |  \
        ((uint64_t)(vector) << 16) | \
        ((uint64_t)(dlm) << 5) | \
        ((uint64_t)(tm) << 4) | \
        ((uint64_t)(rh) << 3) | \
        ((uint64_t)(dm) << 2) | \
        ((uint64_t)(fpd) << 1) | \
        (p))

typedef enum {
    SVT_NO_VERIFY = 0,  /* no verification */
    SVT_ALL_VERIFY,     /* using sid and sq to verify */
    SVT_BUS_VERIFY,     /* verify #startbus and #endbus */
    SVT_RSVD
} intrmap_svt_t;

typedef enum {
    SQ_VERIFY_ALL = 0,  /* verify all 16 bits */
    SQ_VERIFY_IGR_1,    /* ignore bit 3 */
    SQ_VERIFY_IGR_2,    /* ignore bit 2-3 */
    SQ_VERIFY_IGR_3     /* ignore bit 1-3 */
} intrmap_sq_t;

/*
 * S field of the Interrupt Remapping Table Address Register
 * the size of the interrupt remapping table is 1 << (immu_intrmap_irta_s + 1)
 */
static uint_t intrmap_irta_s = INTRMAP_MAX_IRTA_SIZE;

/*
 * If true, arrange to suppress broadcast EOI by setting edge-triggered mode
 * even for level-triggered interrupts in the interrupt-remapping engine.
 * If false, broadcast EOI can still be suppressed if the CPU supports the
 * APIC_SVR_SUPPRESS_BROADCAST_EOI bit.  In both cases, the IOAPIC is still
 * programmed with the correct trigger mode, and pcplusmp must send an EOI
 * to the IOAPIC by writing to the IOAPIC's EOI register to make up for the
 * missing broadcast EOI.
 */
static int intrmap_suppress_brdcst_eoi = 0;

/*
 * whether verify the source id of interrupt request
 */
static int intrmap_enable_sid_verify = 0;

/* fault types for DVMA remapping */
static char *immu_dvma_faults[] = {
    "Reserved",
    "The present field in root-entry is Clear",
    "The present field in context-entry is Clear",
    "Hardware detected invalid programming of a context-entry",
    "The DMA request attempted to access an address beyond max support",
    "The Write field in a page-table entry is Clear when DMA write",
    "The Read field in a page-table entry is Clear when DMA read",
    "Access the next level page table resulted in error",
    "Access the root-entry table resulted in error",
    "Access the context-entry table resulted in error",
    "Reserved field not initialized to zero in a present root-entry",
    "Reserved field not initialized to zero in a present context-entry",
    "Reserved field not initialized to zero in a present page-table entry",
    "DMA blocked due to the Translation Type field in context-entry",
    "Incorrect fault event reason number",
};
#define DVMA_MAX_FAULTS (sizeof (immu_dvma_faults)/(sizeof (char *))) - 1

/* fault types for interrupt remapping */
static char *immu_intrmap_faults[] = {
    "reserved field set in IRTE",
    "interrupt_index exceed the intr-remap table size",
    "present field in IRTE is clear",
    "hardware access intr-remap table address resulted in error",
    "reserved field set in IRTE, include various conditional",
    "hardware blocked an interrupt request in Compatibility format",
    "remappable interrupt request blocked due to verification failure"
};
#define INTRMAP_MAX_FAULTS \
    (sizeof (immu_intrmap_faults) / (sizeof (char *))) - 1

/* Function prototypes */
static int immu_intrmap_init(int apic_mode);
static void immu_intrmap_switchon(int suppress_brdcst_eoi);
static void immu_intrmap_alloc(void **intrmap_private_tbl, dev_info_t *dip,
    uint16_t type, int count, uchar_t ioapic_index);
static void immu_intrmap_map(void *intrmap_private, void *intrmap_data,
    uint16_t type, int count);
static void immu_intrmap_free(void **intrmap_privatep);
static void immu_intrmap_rdt(void *intrmap_private, ioapic_rdt_t *irdt);
static void immu_intrmap_msi(void *intrmap_private, msi_regs_t *mregs);

static struct apic_intrmap_ops intrmap_ops = {
    immu_intrmap_init,
    immu_intrmap_switchon,
    immu_intrmap_alloc,
    immu_intrmap_map,
    immu_intrmap_free,
    immu_intrmap_rdt,
    immu_intrmap_msi,
};

/* apic mode, APIC/X2APIC */
static int intrmap_apic_mode = LOCAL_APIC;


/*
 * helper functions
 */
static uint_t
bitset_find_free(bitset_t *b, uint_t post)
{
    uint_t  i;
    uint_t  cap = bitset_capacity(b);

    if (post == cap)
        post = 0;

    ASSERT(post < cap);

    for (i = post; i < cap; i++) {
        if (!bitset_in_set(b, i))
            return (i);
    }

    for (i = 0; i < post; i++) {
        if (!bitset_in_set(b, i))
            return (i);
    }

    return (INTRMAP_IDX_FULL);  /* no free index */
}

/*
 * helper function to find 'count' contigous free
 * interrupt remapping table entries
 */
static uint_t
bitset_find_multi_free(bitset_t *b, uint_t post, uint_t count)
{
    uint_t  i, j;
    uint_t  cap = bitset_capacity(b);

    if (post == INTRMAP_IDX_FULL) {
        return (INTRMAP_IDX_FULL);
    }

    if (count > cap)
        return (INTRMAP_IDX_FULL);

    ASSERT(post < cap);

    for (i = post; (i + count) <= cap; i++) {
        for (j = 0; j < count; j++) {
            if (bitset_in_set(b, (i + j))) {
                i = i + j;
                break;
            }
            if (j == count - 1)
                return (i);
        }
    }

    for (i = 0; (i < post) && ((i + count) <= cap); i++) {
        for (j = 0; j < count; j++) {
            if (bitset_in_set(b, (i + j))) {
                i = i + j;
                break;
            }
            if (j == count - 1)
                return (i);
        }
    }

    return (INTRMAP_IDX_FULL);          /* no free index */
}

/* alloc one interrupt remapping table entry */
static int
alloc_tbl_entry(intrmap_t *intrmap)
{
    uint32_t idx;

    for (;;) {
        mutex_enter(&intrmap->intrmap_lock);
        idx = intrmap->intrmap_free;
        if (idx != INTRMAP_IDX_FULL) {
            bitset_add(&intrmap->intrmap_map, idx);
            intrmap->intrmap_free =
                bitset_find_free(&intrmap->intrmap_map, idx + 1);
            mutex_exit(&intrmap->intrmap_lock);
            break;
        }

        /* no free intr entry, use compatible format intr */
        mutex_exit(&intrmap->intrmap_lock);

        if (intrmap_apic_mode != LOCAL_X2APIC) {
            break;
        }

        /*
         * x2apic mode not allowed compatible
         * interrupt
         */
        delay(IMMU_ALLOC_RESOURCE_DELAY);
    }

    return (idx);
}

/* alloc 'cnt' contigous interrupt remapping table entries */
static int
alloc_tbl_multi_entries(intrmap_t *intrmap, uint_t cnt)
{
    uint_t idx, pos, i;

    for (; ; ) {
        mutex_enter(&intrmap->intrmap_lock);
        pos = intrmap->intrmap_free;
        idx = bitset_find_multi_free(&intrmap->intrmap_map, pos, cnt);

        if (idx != INTRMAP_IDX_FULL) {
            if (idx <= pos && pos < (idx + cnt)) {
                intrmap->intrmap_free = bitset_find_free(
                    &intrmap->intrmap_map, idx + cnt);
            }
            for (i = 0; i < cnt; i++) {
                bitset_add(&intrmap->intrmap_map, idx + i);
            }
            mutex_exit(&intrmap->intrmap_lock);
            break;
        }

        mutex_exit(&intrmap->intrmap_lock);

        if (intrmap_apic_mode != LOCAL_X2APIC) {
            break;
        }

        /* x2apic mode not allowed comapitible interrupt */
        delay(IMMU_ALLOC_RESOURCE_DELAY);
    }

    return (idx);
}

/* init interrupt remapping table */
static int
init_unit(immu_t *immu)
{
    intrmap_t *intrmap;
    size_t size;

    ddi_dma_attr_t intrmap_dma_attr = {
        DMA_ATTR_V0,
        0U,
        0xffffffffffffffffULL,
        0xffffffffU,
        MMU_PAGESIZE,   /* page aligned */
        0x1,
        0x1,
        0xffffffffU,
        0xffffffffffffffffULL,
        1,
        4,
        0
    };

    ddi_device_acc_attr_t intrmap_acc_attr = {
        DDI_DEVICE_ATTR_V0,
        DDI_NEVERSWAP_ACC,
        DDI_STRICTORDER_ACC
    };

    /*
     * Using interrupt remapping implies using the queue
     * invalidation interface. According to Intel,
     * hardware that supports interrupt remapping should
     * also support QI.
     */
    ASSERT(IMMU_ECAP_GET_QI(immu->immu_regs_excap));

    if (intrmap_apic_mode == LOCAL_X2APIC) {
        if (!IMMU_ECAP_GET_EIM(immu->immu_regs_excap)) {
            return (DDI_FAILURE);
        }
    }

    if (intrmap_irta_s > INTRMAP_MAX_IRTA_SIZE) {
        intrmap_irta_s = INTRMAP_MAX_IRTA_SIZE;
    }

    intrmap =  kmem_zalloc(sizeof (intrmap_t), KM_SLEEP);

    if (ddi_dma_alloc_handle(immu->immu_dip,
        &intrmap_dma_attr,
        DDI_DMA_SLEEP,
        NULL,
        &(intrmap->intrmap_dma_hdl)) != DDI_SUCCESS) {
        kmem_free(intrmap, sizeof (intrmap_t));
        return (DDI_FAILURE);
    }

    intrmap->intrmap_size = 1 << (intrmap_irta_s + 1);
    size = intrmap->intrmap_size * INTRMAP_RTE_SIZE;
    if (ddi_dma_mem_alloc(intrmap->intrmap_dma_hdl,
        size,
        &intrmap_acc_attr,
        DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED,
        DDI_DMA_SLEEP,
        NULL,
        &(intrmap->intrmap_vaddr),
        &size,
        &(intrmap->intrmap_acc_hdl)) != DDI_SUCCESS) {
        ddi_dma_free_handle(&(intrmap->intrmap_dma_hdl));
        kmem_free(intrmap, sizeof (intrmap_t));
        return (DDI_FAILURE);
    }

    ASSERT(!((uintptr_t)intrmap->intrmap_vaddr & MMU_PAGEOFFSET));
    bzero(intrmap->intrmap_vaddr, size);
    intrmap->intrmap_paddr = pfn_to_pa(
        hat_getpfnum(kas.a_hat, intrmap->intrmap_vaddr));

    mutex_init(&(intrmap->intrmap_lock), NULL, MUTEX_DRIVER, NULL);
    bitset_init(&intrmap->intrmap_map);
    bitset_resize(&intrmap->intrmap_map, intrmap->intrmap_size);
    intrmap->intrmap_free = 0;

    immu->immu_intrmap = intrmap;

    return (DDI_SUCCESS);
}

static immu_t *
get_immu(dev_info_t *dip, uint16_t type, uchar_t ioapic_index)
{
    immu_t  *immu = NULL;

    if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
        immu = immu_dmar_ioapic_immu(ioapic_index);
    } else {
        if (dip != NULL)
            immu = immu_dmar_get_immu(dip);
    }

    return (immu);
}

static int
get_top_pcibridge(dev_info_t *dip, void *arg)
{
    dev_info_t **topdipp = arg;
    immu_devi_t *immu_devi;

    mutex_enter(&(DEVI(dip)->devi_lock));
    immu_devi = DEVI(dip)->devi_iommu;
    mutex_exit(&(DEVI(dip)->devi_lock));

    if (immu_devi == NULL || immu_devi->imd_pcib_type == IMMU_PCIB_BAD ||
        immu_devi->imd_pcib_type == IMMU_PCIB_ENDPOINT) {
        return (DDI_WALK_CONTINUE);
    }

    *topdipp = dip;

    return (DDI_WALK_CONTINUE);
}

static dev_info_t *
intrmap_top_pcibridge(dev_info_t *rdip)
{
    dev_info_t *top_pcibridge = NULL;

    if (immu_walk_ancestor(rdip, NULL, get_top_pcibridge,
        &top_pcibridge, NULL, 0) != DDI_SUCCESS) {
        return (NULL);
    }

    return (top_pcibridge);
}

/* function to get interrupt request source id */
static uint32_t
get_sid(dev_info_t *dip, uint16_t type, uchar_t ioapic_index)
{
    dev_info_t  *pdip;
    immu_devi_t *immu_devi;
    uint16_t    sid;
    uchar_t     svt, sq;

    if (!intrmap_enable_sid_verify) {
        return (0);
    }

    if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
        /* for interrupt through I/O APIC */
        sid = immu_dmar_ioapic_sid(ioapic_index);
        svt = SVT_ALL_VERIFY;
        sq = SQ_VERIFY_ALL;
    } else {
        /* MSI/MSI-X interrupt */
        ASSERT(dip);
        pdip = intrmap_top_pcibridge(dip);
        ASSERT(pdip);
        immu_devi = DEVI(pdip)->devi_iommu;
        ASSERT(immu_devi);
        if (immu_devi->imd_pcib_type == IMMU_PCIB_PCIE_PCI) {
            /* device behind pcie to pci bridge */
            sid = (immu_devi->imd_bus << 8) | immu_devi->imd_sec;
            svt = SVT_BUS_VERIFY;
            sq = SQ_VERIFY_ALL;
        } else {
            /* pcie device or device behind pci to pci bridge */
            sid = (immu_devi->imd_bus << 8) |
                immu_devi->imd_devfunc;
            svt = SVT_ALL_VERIFY;
            sq = SQ_VERIFY_ALL;
        }
    }

    return (sid | (svt << 18) | (sq << 16));
}

static void
intrmap_enable(immu_t *immu)
{
    intrmap_t *intrmap;
    uint64_t irta_reg;

    intrmap = immu->immu_intrmap;

    irta_reg = intrmap->intrmap_paddr | intrmap_irta_s;
    if (intrmap_apic_mode == LOCAL_X2APIC) {
        irta_reg |= (0x1 << 11);
    }

    immu_regs_intrmap_enable(immu, irta_reg);
}

/* ####################################################################### */

/*
 * immu_intr_handler()
 *  the fault event handler for a single immu unit
 */
int
immu_intr_handler(immu_t *immu)
{
    uint32_t status;
    int index, fault_reg_offset;
    int max_fault_index;
    boolean_t found_fault;
    dev_info_t *idip;

    mutex_enter(&(immu->immu_intr_lock));
    mutex_enter(&(immu->immu_regs_lock));

    /* read the fault status */
    status = immu_regs_get32(immu, IMMU_REG_FAULT_STS);

    idip = immu->immu_dip;
    ASSERT(idip);

    /* check if we have a pending fault for this immu unit */
    if ((status & IMMU_FAULT_STS_PPF) == 0) {
        mutex_exit(&(immu->immu_regs_lock));
        mutex_exit(&(immu->immu_intr_lock));
        return (DDI_INTR_UNCLAIMED);
    }

    /*
     * handle all primary pending faults
     */
    index = IMMU_FAULT_GET_INDEX(status);
    max_fault_index =  IMMU_CAP_GET_NFR(immu->immu_regs_cap) - 1;
    fault_reg_offset = IMMU_CAP_GET_FRO(immu->immu_regs_cap);

    found_fault = B_FALSE;
    _NOTE(CONSTCOND)
    while (1) {
        uint64_t val;
        uint8_t fault_reason;
        uint8_t fault_type;
        uint16_t sid;
        uint64_t pg_addr;
        uint64_t idx;

        /* read the higher 64bits */
        val = immu_regs_get64(immu, fault_reg_offset + index * 16 + 8);

        /* check if this fault register has pending fault */
        if (!IMMU_FRR_GET_F(val)) {
            break;
        }

        found_fault = B_TRUE;

        /* get the fault reason, fault type and sid */
        fault_reason = IMMU_FRR_GET_FR(val);
        fault_type = IMMU_FRR_GET_FT(val);
        sid = IMMU_FRR_GET_SID(val);

        /* read the first 64bits */
        val = immu_regs_get64(immu, fault_reg_offset + index * 16);
        pg_addr = val & IMMU_PAGEMASK;
        idx = val >> 48;

        /* clear the fault */
        immu_regs_put32(immu, fault_reg_offset + index * 16 + 12,
            (((uint32_t)1) << 31));

        /* report the fault info */
        if (fault_reason < 0x20) {
            /* immu-remapping fault */
            ddi_err(DER_WARN, idip,
                "generated a fault event when translating DMA %s\n"
                "\t on address 0x%" PRIx64 " for PCI(%d, %d, %d), "
                "the reason is:\n\t %s",
                fault_type ? "read" : "write", pg_addr,
                (sid >> 8) & 0xff, (sid >> 3) & 0x1f, sid & 0x7,
                immu_dvma_faults[MIN(fault_reason,
                DVMA_MAX_FAULTS)]);
            immu_print_fault_info(sid, pg_addr);
        } else if (fault_reason < 0x27) {
            /* intr-remapping fault */
            ddi_err(DER_WARN, idip,
                "generated a fault event when translating "
                "interrupt request\n"
                "\t on index 0x%" PRIx64 " for PCI(%d, %d, %d), "
                "the reason is:\n\t %s",
                idx,
                (sid >> 8) & 0xff, (sid >> 3) & 0x1f, sid & 0x7,
                immu_intrmap_faults[MIN((fault_reason - 0x20),
                INTRMAP_MAX_FAULTS)]);
        } else {
            ddi_err(DER_WARN, idip, "Unknown fault reason: 0x%x",
                fault_reason);
        }

        index++;
        if (index > max_fault_index)
            index = 0;
    }

    /* Clear the fault */
    if (!found_fault) {
        ddi_err(DER_MODE, idip,
            "Fault register set but no fault present");
    }
    immu_regs_put32(immu, IMMU_REG_FAULT_STS, 1);
    mutex_exit(&(immu->immu_regs_lock));
    mutex_exit(&(immu->immu_intr_lock));
    return (DDI_INTR_CLAIMED);
}
/* ######################################################################### */

/*
 * Interrupt remap entry points
 */

/* initialize interrupt remapping */
static int
immu_intrmap_init(int apic_mode)
{
    immu_t *immu;
    int error = DDI_FAILURE;

    if (immu_intrmap_enable == B_FALSE) {
        return (DDI_SUCCESS);
    }

    intrmap_apic_mode = apic_mode;

    immu = list_head(&immu_list);
    for (; immu; immu = list_next(&immu_list, immu)) {
        if ((immu->immu_intrmap_running == B_TRUE) &&
            IMMU_ECAP_GET_IR(immu->immu_regs_excap)) {
            if (init_unit(immu) == DDI_SUCCESS) {
                error = DDI_SUCCESS;
            }
        }
    }

    /*
     * if all IOMMU units disable intr remapping,
     * return FAILURE
     */
    return (error);
}


/* enable interrupt remapping */
static void
immu_intrmap_switchon(int suppress_brdcst_eoi)
{
    immu_t *immu;


    intrmap_suppress_brdcst_eoi = suppress_brdcst_eoi;

    immu = list_head(&immu_list);
    for (; immu; immu = list_next(&immu_list, immu)) {
        if (immu->immu_intrmap_setup == B_TRUE) {
            intrmap_enable(immu);
        }
    }
}

/* alloc remapping entry for the interrupt */
static void
immu_intrmap_alloc(void **intrmap_private_tbl, dev_info_t *dip,
    uint16_t type, int count, uchar_t ioapic_index)
{
    immu_t  *immu;
    intrmap_t *intrmap;
    immu_inv_wait_t *iwp;
    uint32_t        idx, i;
    uint32_t        sid_svt_sq;
    intrmap_private_t   *intrmap_private;

    if (intrmap_private_tbl[0] == INTRMAP_DISABLE ||
        intrmap_private_tbl[0] != NULL) {
        return;
    }

    intrmap_private_tbl[0] =
        kmem_zalloc(sizeof (intrmap_private_t), KM_SLEEP);
    intrmap_private = INTRMAP_PRIVATE(intrmap_private_tbl[0]);

    immu = get_immu(dip, type, ioapic_index);
    if ((immu != NULL) && (immu->immu_intrmap_running == B_TRUE)) {
        intrmap_private->ir_immu = immu;
    } else {
        goto intrmap_disable;
    }

    intrmap = immu->immu_intrmap;

    if (count == 1) {
        idx = alloc_tbl_entry(intrmap);
    } else {
        idx = alloc_tbl_multi_entries(intrmap, count);
    }

    if (idx == INTRMAP_IDX_FULL) {
        goto intrmap_disable;
    }

    intrmap_private->ir_idx = idx;

    sid_svt_sq = intrmap_private->ir_sid_svt_sq =
        get_sid(dip, type, ioapic_index);
    iwp = &intrmap_private->ir_inv_wait;
    immu_init_inv_wait(iwp, "intrmaplocal", B_TRUE);

    if (count == 1) {
        if (IMMU_CAP_GET_CM(immu->immu_regs_cap)) {
            immu_qinv_intr_one_cache(immu, idx, iwp);
        } else {
            immu_regs_wbf_flush(immu);
        }
        return;
    }

    for (i = 1; i < count; i++) {
        intrmap_private_tbl[i] =
            kmem_zalloc(sizeof (intrmap_private_t), KM_SLEEP);

        INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_immu = immu;
        INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_sid_svt_sq =
            sid_svt_sq;
        INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_idx = idx + i;
    }

    if (IMMU_CAP_GET_CM(immu->immu_regs_cap)) {
        immu_qinv_intr_caches(immu, idx, count, iwp);
    } else {
        immu_regs_wbf_flush(immu);
    }

    return;

intrmap_disable:
    kmem_free(intrmap_private_tbl[0], sizeof (intrmap_private_t));
    intrmap_private_tbl[0] = INTRMAP_DISABLE;
}


/* remapping the interrupt */
static void
immu_intrmap_map(void *intrmap_private, void *intrmap_data, uint16_t type,
    int count)
{
    immu_t  *immu;
    immu_inv_wait_t *iwp;
    intrmap_t   *intrmap;
    ioapic_rdt_t    *irdt = (ioapic_rdt_t *)intrmap_data;
    msi_regs_t  *mregs = (msi_regs_t *)intrmap_data;
    intrmap_rte_t   irte;
    uint_t      idx, i;
    uint32_t    dst, sid_svt_sq;
    uchar_t     vector, dlm, tm, rh, dm;

    if (intrmap_private == INTRMAP_DISABLE)
        return;

    idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
    immu = INTRMAP_PRIVATE(intrmap_private)->ir_immu;
    iwp = &INTRMAP_PRIVATE(intrmap_private)->ir_inv_wait;
    intrmap = immu->immu_intrmap;
    sid_svt_sq = INTRMAP_PRIVATE(intrmap_private)->ir_sid_svt_sq;

    if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
        dm = RDT_DM(irdt->ir_lo);
        rh = 0;
        tm = RDT_TM(irdt->ir_lo);
        dlm = RDT_DLM(irdt->ir_lo);
        dst = irdt->ir_hi;

        /*
         * Mark the IRTE's TM as Edge to suppress broadcast EOI.
         */
        if (intrmap_suppress_brdcst_eoi) {
            tm = TRIGGER_MODE_EDGE;
        }

        vector = RDT_VECTOR(irdt->ir_lo);
    } else {
        dm = MSI_ADDR_DM_PHYSICAL;
        rh = MSI_ADDR_RH_FIXED;
        tm = TRIGGER_MODE_EDGE;
        dlm = 0;
        dst = mregs->mr_addr;

        vector = mregs->mr_data & 0xff;
    }

    if (intrmap_apic_mode == LOCAL_APIC)
        dst = (dst & 0xFF) << 8;

    if (count == 1) {
        irte.lo = IRTE_LOW(dst, vector, dlm, tm, rh, dm, 0, 1);
        irte.hi = IRTE_HIGH(sid_svt_sq);

        /* set interrupt remapping table entry */
        bcopy(&irte, intrmap->intrmap_vaddr +
            idx * INTRMAP_RTE_SIZE,
            INTRMAP_RTE_SIZE);

        immu_qinv_intr_one_cache(immu, idx, iwp);

    } else {
        for (i = 0; i < count; i++) {
            irte.lo = IRTE_LOW(dst, vector, dlm, tm, rh, dm, 0, 1);
            irte.hi = IRTE_HIGH(sid_svt_sq);

            /* set interrupt remapping table entry */
            bcopy(&irte, intrmap->intrmap_vaddr +
                idx * INTRMAP_RTE_SIZE,
                INTRMAP_RTE_SIZE);
            vector++;
            idx++;
        }

        immu_qinv_intr_caches(immu, idx, count, iwp);
    }
}

/* free the remapping entry */
static void
immu_intrmap_free(void **intrmap_privatep)
{
    immu_t *immu;
    immu_inv_wait_t *iwp;
    intrmap_t *intrmap;
    uint32_t idx;

    if (*intrmap_privatep == INTRMAP_DISABLE || *intrmap_privatep == NULL) {
        *intrmap_privatep = NULL;
        return;
    }

    immu = INTRMAP_PRIVATE(*intrmap_privatep)->ir_immu;
    iwp = &INTRMAP_PRIVATE(*intrmap_privatep)->ir_inv_wait;
    intrmap = immu->immu_intrmap;
    idx = INTRMAP_PRIVATE(*intrmap_privatep)->ir_idx;

    bzero(intrmap->intrmap_vaddr + idx * INTRMAP_RTE_SIZE,
        INTRMAP_RTE_SIZE);

    immu_qinv_intr_one_cache(immu, idx, iwp);

    mutex_enter(&intrmap->intrmap_lock);
    bitset_del(&intrmap->intrmap_map, idx);
    if (intrmap->intrmap_free == INTRMAP_IDX_FULL) {
        intrmap->intrmap_free = idx;
    }
    mutex_exit(&intrmap->intrmap_lock);

    kmem_free(*intrmap_privatep, sizeof (intrmap_private_t));
    *intrmap_privatep = NULL;
}

/* record the ioapic rdt entry */
static void
immu_intrmap_rdt(void *intrmap_private, ioapic_rdt_t *irdt)
{
    uint32_t rdt_entry, tm, pol, idx, vector;

    rdt_entry = irdt->ir_lo;

    if (intrmap_private != INTRMAP_DISABLE && intrmap_private != NULL) {
        idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
        tm = RDT_TM(rdt_entry);
        pol = RDT_POL(rdt_entry);
        vector = RDT_VECTOR(rdt_entry);
        irdt->ir_lo = (tm << INTRMAP_IOAPIC_TM_SHIFT) |
            (pol << INTRMAP_IOAPIC_POL_SHIFT) |
            ((idx >> 15) << INTRMAP_IOAPIC_IDX15_SHIFT) |
            vector;
        irdt->ir_hi = (idx << INTRMAP_IOAPIC_IDX_SHIFT) |
            (1 << INTRMAP_IOAPIC_FORMAT_SHIFT);
    } else {
        irdt->ir_hi <<= APIC_ID_BIT_OFFSET;
    }
}

/* record the msi interrupt structure */
/*ARGSUSED*/
static void
immu_intrmap_msi(void *intrmap_private, msi_regs_t *mregs)
{
    uint_t  idx;

    if (intrmap_private != INTRMAP_DISABLE && intrmap_private != NULL) {
        idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;

        mregs->mr_data = 0;
        mregs->mr_addr = MSI_ADDR_HDR |
            ((idx & 0x7fff) << INTRMAP_MSI_IDX_SHIFT) |
            (1 << INTRMAP_MSI_FORMAT_SHIFT) |
            (1 << INTRMAP_MSI_SHV_SHIFT) |
            ((idx >> 15) << INTRMAP_MSI_IDX15_SHIFT);
    } else {
        mregs->mr_addr = MSI_ADDR_HDR |
            (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
            (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT) |
            (mregs->mr_addr << MSI_ADDR_DEST_SHIFT);
        mregs->mr_data = (MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) |
            mregs->mr_data;
    }
}

/* ######################################################################### */
/*
 * Functions exported by immu_intr.c
 */
void
immu_intrmap_setup(list_t *listp)
{
    immu_t *immu;

    /*
     * Check if ACPI DMAR tables say that
     * interrupt remapping is supported
     */
    if (immu_dmar_intrmap_supported() == B_FALSE) {
        return;
    }

    /*
     * Check if interrupt remapping is disabled.
     */
    if (immu_intrmap_enable == B_FALSE) {
        return;
    }

    psm_vt_ops = &intrmap_ops;

    immu = list_head(listp);
    for (; immu; immu = list_next(listp, immu)) {
        mutex_init(&(immu->immu_intrmap_lock), NULL,
            MUTEX_DEFAULT, NULL);
        mutex_enter(&(immu->immu_intrmap_lock));
        immu_init_inv_wait(&immu->immu_intrmap_inv_wait,
            "intrmapglobal", B_TRUE);
        immu->immu_intrmap_setup = B_TRUE;
        mutex_exit(&(immu->immu_intrmap_lock));
    }
}

void
immu_intrmap_startup(immu_t *immu)
{
    /* do nothing */
    mutex_enter(&(immu->immu_intrmap_lock));
    if (immu->immu_intrmap_setup == B_TRUE) {
        immu->immu_intrmap_running = B_TRUE;
    }
    mutex_exit(&(immu->immu_intrmap_lock));
}

/*
 * Register a Intel IOMMU unit (i.e. DMAR unit's)
 * interrupt handler
 */
void
immu_intr_register(immu_t *immu)
{
    int irq, vect;
    char intr_handler_name[IMMU_MAXNAMELEN];
    uint32_t msi_data;
    uint32_t uaddr;
    uint32_t msi_addr;
    uint32_t localapic_id = 0;

    if (psm_get_localapicid)
        localapic_id = psm_get_localapicid(0);

    msi_addr = (MSI_ADDR_HDR |
        ((localapic_id & 0xFF) << MSI_ADDR_DEST_SHIFT) |
        (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
        (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT));

    if (intrmap_apic_mode == LOCAL_X2APIC) {
        uaddr = localapic_id & 0xFFFFFF00;
    } else {
        uaddr = 0;
    }

    /* Dont need to hold immu_intr_lock since we are in boot */
    irq = vect = psm_get_ipivect(IMMU_INTR_IPL, -1);
    if (psm_xlate_vector_by_irq != NULL)
        vect = psm_xlate_vector_by_irq(irq);

    msi_data = ((MSI_DATA_DELIVERY_FIXED <<
        MSI_DATA_DELIVERY_SHIFT) | vect);

    (void) snprintf(intr_handler_name, sizeof (intr_handler_name),
        "%s-intr-handler", immu->immu_name);

    (void) add_avintr((void *)NULL, IMMU_INTR_IPL,
        (avfunc)(immu_intr_handler), intr_handler_name, irq,
        (caddr_t)immu, NULL, NULL, NULL);

    immu_regs_intr_enable(immu, msi_addr, msi_data, uaddr);

    (void) immu_intr_handler(immu);
}