io/pci/pci_iommu.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
 */

/*
 * PCI iommu initialization and configuration
 */

#include <sys/types.h>
#include <sys/kmem.h>
#include <sys/async.h>
#include <sys/sysmacros.h>
#include <sys/sunddi.h>
#include <sys/ddi_impldefs.h>
#include <sys/vmem.h>
#include <sys/machsystm.h>  /* lddphys() */
#include <sys/iommutsb.h>
#include <sys/pci/pci_obj.h>

/*LINTLIBRARY*/

static void iommu_tlb_flushall(iommu_t *iommu_p);
static void iommu_preserve_tsb(iommu_t *iommu_p);

void
iommu_create(pci_t *pci_p)
{
    dev_info_t *dip = pci_p->pci_dip;
    iommu_t *iommu_p;
    uintptr_t a;
    size_t cache_size;
    uint32_t tsb_entries;

    char map_name[32];
    extern uint64_t va_to_pa(void *);

    pci_dvma_range_prop_t   pci_dvma_range;

    /*
     * Allocate iommu state structure and link it to the
     * pci state structure.
     */
    iommu_p = (iommu_t *)kmem_zalloc(sizeof (iommu_t), KM_SLEEP);
    pci_p->pci_iommu_p = iommu_p;
    iommu_p->iommu_pci_p = pci_p;
    iommu_p->iommu_inst = ddi_get_instance(dip);

    /*
     * chip specific dvma_end, tsb_size & context support
     */
    iommu_p->iommu_dvma_end = pci_iommu_dvma_end;
    a = pci_iommu_setup(iommu_p);

    /*
     * Determine the virtual address of iommu registers.
     */
    iommu_p->iommu_ctrl_reg =
        (uint64_t *)(a + COMMON_IOMMU_CTRL_REG_OFFSET);
    iommu_p->iommu_tsb_base_addr_reg =
        (uint64_t *)(a + COMMON_IOMMU_TSB_BASE_ADDR_REG_OFFSET);
    iommu_p->iommu_flush_page_reg =
        (uint64_t *)(a + COMMON_IOMMU_FLUSH_PAGE_REG_OFFSET);

    /*
     * Configure the rest of the iommu parameters according to:
     * tsb_size and dvma_end
     */
    iommu_p->iommu_tsb_vaddr = /* retrieve TSB VA reserved by system */
        iommu_tsb_cookie_to_va(pci_p->pci_tsb_cookie);
    iommu_p->iommu_tsb_entries = tsb_entries =
        IOMMU_TSBSIZE_TO_TSBENTRIES(iommu_p->iommu_tsb_size);
    iommu_p->iommu_tsb_paddr = va_to_pa((caddr_t)iommu_p->iommu_tsb_vaddr);
    iommu_p->iommu_dvma_cache_locks =
        kmem_zalloc(pci_dvma_page_cache_entries, KM_SLEEP);

    iommu_p->iommu_dvma_base = iommu_p->iommu_dvma_end + 1
        - (tsb_entries * IOMMU_PAGE_SIZE);
    iommu_p->dvma_base_pg = IOMMU_BTOP(iommu_p->iommu_dvma_base);
    iommu_p->iommu_dvma_reserve = tsb_entries >> 1;
    iommu_p->dvma_end_pg = IOMMU_BTOP(iommu_p->iommu_dvma_end);
    iommu_p->iommu_dma_bypass_base = COMMON_IOMMU_BYPASS_BASE;
    iommu_p->iommu_dma_bypass_end = pci_iommu_bypass_end_configure();

    /*
     * export "virtual-dma" software property to support
     * child devices needing to know DVMA range
     */
    pci_dvma_range.dvma_base = (uint32_t)iommu_p->iommu_dvma_base;
    pci_dvma_range.dvma_len = (uint32_t)
        iommu_p->iommu_dvma_end - iommu_p->iommu_dvma_base + 1;
    (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
        "virtual-dma", (caddr_t)&pci_dvma_range,
        sizeof (pci_dvma_range));

    DEBUG2(DBG_ATTACH, dip, "iommu_create: ctrl=%p, tsb=%p\n",
        iommu_p->iommu_ctrl_reg, iommu_p->iommu_tsb_base_addr_reg);
    DEBUG2(DBG_ATTACH, dip, "iommu_create: page_flush=%p, ctx_flush=%p\n",
        iommu_p->iommu_flush_page_reg, iommu_p->iommu_flush_ctx_reg);
    DEBUG2(DBG_ATTACH, dip, "iommu_create: tsb vaddr=%p tsb_paddr=%p\n",
        iommu_p->iommu_tsb_vaddr, iommu_p->iommu_tsb_paddr);
    DEBUG1(DBG_ATTACH, dip, "iommu_create: allocated size=%x\n",
        iommu_tsb_cookie_to_size(pci_p->pci_tsb_cookie));
    DEBUG2(DBG_ATTACH, dip, "iommu_create: fast tsb tte addr: %x + %x\n",
        iommu_p->iommu_tsb_vaddr,
        pci_dvma_page_cache_entries * pci_dvma_page_cache_clustsz);
    DEBUG3(DBG_ATTACH, dip,
        "iommu_create: tsb size=%x, tsb entries=%x, dvma base=%x\n",
        iommu_p->iommu_tsb_size, iommu_p->iommu_tsb_entries,
        iommu_p->iommu_dvma_base);
    DEBUG2(DBG_ATTACH, dip,
        "iommu_create: dvma_cache_locks=%x cache_entries=%x\n",
        iommu_p->iommu_dvma_cache_locks, pci_dvma_page_cache_entries);

    /*
     * zero out the area to be used for iommu tsb
     */
    bzero(iommu_p->iommu_tsb_vaddr, tsb_entries << 3);

    /*
     * Create a virtual memory map for dvma address space.
     * Reserve 'size' bytes of low dvma space for fast track cache.
     */
    (void) snprintf(map_name, sizeof (map_name), "%s%d_dvma",
        ddi_driver_name(dip), ddi_get_instance(dip));

    cache_size = IOMMU_PTOB(pci_dvma_page_cache_entries *
        pci_dvma_page_cache_clustsz);
    iommu_p->iommu_dvma_fast_end = iommu_p->iommu_dvma_base +
        cache_size - 1;
    iommu_p->iommu_dvma_map = vmem_create(map_name,
        (void *)(iommu_p->iommu_dvma_fast_end + 1),
        IOMMU_PTOB(tsb_entries) - cache_size, IOMMU_PAGE_SIZE,
        NULL, NULL, NULL, IOMMU_PAGE_SIZE, VM_SLEEP);

    mutex_init(&iommu_p->dvma_debug_lock, NULL, MUTEX_DRIVER, NULL);

    /*
     * On detach, the TSB Base Address Register gets set to zero,
     * so if its zero here, there is no need to preserve TTEs.
     */
    if (pci_preserve_iommu_tsb && *iommu_p->iommu_tsb_base_addr_reg)
        iommu_preserve_tsb(iommu_p);

    iommu_configure(iommu_p);
}

void
iommu_destroy(pci_t *pci_p)
{
#ifdef DEBUG
    dev_info_t *dip = pci_p->pci_dip;
#endif
    iommu_t *iommu_p = pci_p->pci_iommu_p;
    volatile uint64_t ctl_val = *iommu_p->iommu_ctrl_reg;

    DEBUG0(DBG_DETACH, dip, "iommu_destroy:\n");

    /*
     * Disable the IOMMU by setting the TSB Base Address to zero
     * and the TSB Table size to the smallest possible.
     */
    ctl_val = ctl_val & ~(7 << COMMON_IOMMU_CTRL_TSB_SZ_SHIFT);

    *iommu_p->iommu_ctrl_reg = ctl_val;
    *iommu_p->iommu_tsb_base_addr_reg = 0;

    /*
     * Return the boot time allocated tsb.
     */
    iommu_tsb_free(pci_p->pci_tsb_cookie);

    /*
     * Teardown any implementation-specific structures set up in
     * pci_iommu_setup.
     */
    pci_iommu_teardown(iommu_p);

    if (DVMA_DBG_ON(iommu_p))
        pci_dvma_debug_fini(iommu_p);
    mutex_destroy(&iommu_p->dvma_debug_lock);

    /*
     * Free the dvma resource map.
     */
    vmem_destroy(iommu_p->iommu_dvma_map);

    kmem_free(iommu_p->iommu_dvma_cache_locks,
        pci_dvma_page_cache_entries);

    /*
     * Free the iommu state structure.
     */
    kmem_free(iommu_p, sizeof (iommu_t));
    pci_p->pci_iommu_p = NULL;
}

/*
 * re-program iommu on the fly while preserving on-going dma
 * transactions on the PCI bus.
 */
void
iommu_configure(iommu_t *iommu_p)
{
    pci_t *pci_p = iommu_p->iommu_pci_p;
    uint64_t cfgpa = pci_get_cfg_pabase(pci_p);
    dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
    dev_info_t *cdip = NULL;
    volatile uint64_t ctl_val = (uint64_t)
        ((iommu_p->iommu_tsb_size << COMMON_IOMMU_CTRL_TSB_SZ_SHIFT) |
        (0 /* 8k page */ << COMMON_IOMMU_CTRL_TBW_SZ_SHIFT) |
        COMMON_IOMMU_CTRL_ENABLE | COMMON_IOMMU_CTRL_DIAG_ENABLE |
        (pci_lock_tlb ? COMMON_IOMMU_CTRL_LCK_ENABLE : 0));

    DEBUG2(DBG_ATTACH, dip, "iommu_configure: iommu_ctl=%08x.%08x\n",
        HI32(ctl_val), LO32(ctl_val));
    if (!pci_preserve_iommu_tsb || !(*iommu_p->iommu_tsb_base_addr_reg)) {
        *iommu_p->iommu_ctrl_reg = COMMON_IOMMU_CTRL_DIAG_ENABLE;
        iommu_tlb_flushall(iommu_p);
        goto config;
    }
    cdip = ddi_get_child(dip);
    for (; cdip; cdip = ddi_get_next_sibling(cdip)) {
        uint32_t *reg_p;
        int reg_len;
        if (ddi_getlongprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
            "reg", (caddr_t)&reg_p, &reg_len) != DDI_PROP_SUCCESS)
            continue;
        cfgpa += (*reg_p) & (PCI_CONF_ADDR_MASK ^ PCI_REG_REG_M);
        kmem_free(reg_p, reg_len);
        break;
    }

config:
    pci_iommu_config(iommu_p, ctl_val, cdip ? cfgpa : 0);
}

void
iommu_map_pages(iommu_t *iommu_p, ddi_dma_impl_t *mp,
        dvma_addr_t dvma_pg, size_t npages, size_t pfn_index)
{
    int i;
    dvma_addr_t pg_index = dvma_pg - iommu_p->dvma_base_pg;
    uint64_t *tte_addr = iommu_p->iommu_tsb_vaddr + pg_index;
    size_t pfn_last = pfn_index + npages;
    uint64_t tte = PCI_GET_MP_TTE(mp->dmai_tte);
#ifdef DEBUG
    dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
#endif

    ASSERT(pfn_last <= mp->dmai_ndvmapages);

    DEBUG5(DBG_MAP_WIN, dip,
        "iommu_map_pages:%x+%x=%x npages=0x%x pfn_index=0x%x\n",
        (uint_t)iommu_p->dvma_base_pg, (uint_t)pg_index, dvma_pg,
        (uint_t)npages, (uint_t)pfn_index);

    for (i = pfn_index; i < pfn_last; i++, pg_index++, tte_addr++) {
        iopfn_t pfn = PCI_GET_MP_PFN(mp, i);
        volatile uint64_t cur_tte = IOMMU_PTOB(pfn) | tte;

        DEBUG3(DBG_MAP_WIN, dip, "iommu_map_pages: mp=%p pg[%x]=%x\n",
            mp, i, (uint_t)pfn);
        DEBUG3(DBG_MAP_WIN, dip,
            "iommu_map_pages: pg_index=%x tte=%08x.%08x\n",
            pg_index, HI32(cur_tte), LO32(cur_tte));
        ASSERT(TTE_IS_INVALID(*tte_addr));
        *tte_addr = cur_tte;
#ifdef DEBUG
        if (pfn == 0 && pci_warn_pp0)
            cmn_err(CE_WARN, "%s%d <%p> doing DMA to pp0\n",
                ddi_driver_name(mp->dmai_rdip),
                ddi_get_instance(mp->dmai_rdip), mp);
#endif
    }
    ASSERT(tte_addr == iommu_p->iommu_tsb_vaddr + pg_index);
#ifdef DEBUG
    if (HAS_REDZONE(mp)) {
        DEBUG1(DBG_MAP_WIN, dip, "iommu_map_pages: redzone pg=%x\n",
            pg_index);
        ASSERT(TTE_IS_INVALID(iommu_p->iommu_tsb_vaddr[pg_index]));
    }
#endif
    if (DVMA_DBG_ON(iommu_p))
        pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping,
            mp->dmai_size, mp);
}

/*
 * iommu_map_window - map a dvma window into the iommu
 *
 * used by: pci_dma_win(), pci_dma_ctlops() - DDI_DMA_MOVWIN
 *
 * return value: none
 */
/*ARGSUSED*/
void
iommu_map_window(iommu_t *iommu_p, ddi_dma_impl_t *mp, window_t win_no)
{
    uint32_t obj_pg0_off = mp->dmai_roffset;
    uint32_t win_pg0_off = win_no ? 0 : obj_pg0_off;
    size_t win_size = mp->dmai_winsize;
    size_t pfn_index = win_size * win_no;           /* temp value */
    size_t obj_off = win_no ? pfn_index - obj_pg0_off : 0;  /* xferred sz */
    dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping);
    size_t res_size = mp->dmai_object.dmao_size - obj_off + win_pg0_off;

    ASSERT(!(win_size & IOMMU_PAGE_OFFSET));
    if (win_no >= mp->dmai_nwin)
        return;
    if (res_size < win_size)        /* last window */
        win_size = res_size;        /* mp->dmai_winsize unchanged */

    mp->dmai_mapping = IOMMU_PTOB(dvma_pg) | win_pg0_off;
    mp->dmai_size = win_size - win_pg0_off; /* cur win xferrable size */
    mp->dmai_offset = obj_off;      /* win offset into object */
    pfn_index = IOMMU_BTOP(pfn_index);  /* index into pfnlist */
    iommu_map_pages(iommu_p, mp, dvma_pg, IOMMU_BTOPR(win_size), pfn_index);
}

void
iommu_unmap_pages(iommu_t *iommu_p, dvma_addr_t dvma_pg, uint_t npages)
{
    dvma_addr_t pg_index = IOMMU_PAGE_INDEX(iommu_p, dvma_pg);

    for (; npages; npages--, dvma_pg++, pg_index++) {
        DEBUG1(DBG_UNMAP_WIN|DBG_CONT, 0, " %x", dvma_pg);
        IOMMU_UNLOAD_TTE(iommu_p, pg_index);

        if (!tm_mtlb_gc)
            IOMMU_PAGE_FLUSH(iommu_p, dvma_pg);
    }
}

void
iommu_remap_pages(iommu_t *iommu_p, ddi_dma_impl_t *mp, dvma_addr_t dvma_pg,
    size_t npages, size_t pfn_index)
{
    iommu_unmap_pages(iommu_p, dvma_pg, npages);
    iommu_map_pages(iommu_p, mp, dvma_pg, npages, pfn_index);
}

/*
 * iommu_unmap_window
 *
 * This routine is called to break down the iommu mappings to a dvma window.
 * Non partial mappings are viewed as single window mapping.
 *
 * used by: pci_dma_unbindhdl(), pci_dma_window(),
 *  and pci_dma_ctlops() - DDI_DMA_FREE, DDI_DMA_MOVWIN, DDI_DMA_NEXTWIN
 *
 * return value: none
 */
/*ARGSUSED*/
void
iommu_unmap_window(iommu_t *iommu_p, ddi_dma_impl_t *mp)
{
    dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping);
    dvma_addr_t pg_index = IOMMU_PAGE_INDEX(iommu_p, dvma_pg);
    uint_t npages = IOMMU_BTOP(mp->dmai_winsize);
#ifdef DEBUG
    dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
#endif
    /*
     * Invalidate each page of the mapping in the tsb and flush
     * it from the tlb.
     */
    DEBUG2(DBG_UNMAP_WIN, dip, "mp=%p %x pfns:", mp, npages);
    if (mp->dmai_flags & DMAI_FLAGS_CONTEXT) {
        dvma_context_t ctx = MP2CTX(mp);
        for (; npages; npages--, pg_index++) {
            DEBUG1(DBG_UNMAP_WIN|DBG_CONT, dip, " %x", pg_index);
            IOMMU_UNLOAD_TTE(iommu_p, pg_index);
        }
        DEBUG1(DBG_UNMAP_WIN|DBG_CONT, dip, " (context %x)", ctx);
        *iommu_p->iommu_flush_ctx_reg = ctx;
    } else
        iommu_unmap_pages(iommu_p, dvma_pg, npages);

    DEBUG0(DBG_UNMAP_WIN|DBG_CONT, dip, "\n");

    if (DVMA_DBG_ON(iommu_p))
        pci_dvma_free_debug(iommu_p, (char *)mp->dmai_mapping,
            mp->dmai_size, mp);
}

int
pci_alloc_tsb(pci_t *pci_p)
{
    uint16_t tsbc;

    if ((tsbc = iommu_tsb_alloc(pci_p->pci_id)) == IOMMU_TSB_COOKIE_NONE) {
        cmn_err(CE_WARN, "%s%d: Unable to allocate IOMMU TSB.",
            ddi_driver_name(pci_p->pci_dip),
            ddi_get_instance(pci_p->pci_dip));
        return (DDI_FAILURE);
    }
    pci_p->pci_tsb_cookie = tsbc;
    return (DDI_SUCCESS);
}

void
pci_free_tsb(pci_t *pci_p)
{
    iommu_tsb_free(pci_p->pci_tsb_cookie);
}

#if 0
/*
 * The following data structure is used to map a tsb size
 * to a tsb size configuration parameter in the iommu
 * control register.
 * This is a hardware table. It is here for reference only.
 */
static int pci_iommu_tsb_sizes[] = {
    0x2000,     /* 0 - 8 mb */
    0x4000,     /* 1 - 16 mb */
    0x8000,     /* 2 - 32 mb */
    0x10000,    /* 3 - 64 mb */
    0x20000,    /* 4 - 128 mb */
    0x40000,    /* 5 - 256 mb */
    0x80000,    /* 6 - 512 mb */
    0x100000    /* 7 - 1 gb */
};
#endif

uint_t
iommu_tsb_size_encode(uint_t tsb_bytes)
{
    uint_t i;

    for (i = 7; i && (tsb_bytes < (0x2000 << i)); i--)
        /* empty */;
    return (i);
}

/*
 * invalidate IOMMU TLB entries through diagnostic registers.
 */
static void
iommu_tlb_flushall(iommu_t *iommu_p)
{
    int i;
    uint64_t base = (uint64_t)(iommu_p->iommu_ctrl_reg) -
        COMMON_IOMMU_CTRL_REG_OFFSET;
    volatile uint64_t *tlb_tag = (volatile uint64_t *)
        (base + COMMON_IOMMU_TLB_TAG_DIAG_ACC_OFFSET);
    volatile uint64_t *tlb_data = (volatile uint64_t *)
        (base + COMMON_IOMMU_TLB_DATA_DIAG_ACC_OFFSET);
    for (i = 0; i < IOMMU_TLB_ENTRIES; i++)
        tlb_tag[i] = tlb_data[i] = 0ull;
}

static void
iommu_preserve_tsb(iommu_t *iommu_p)
{
#ifdef DEBUG
    dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
#endif
    uint_t i, obp_tsb_entries, obp_tsb_size, base_pg_index;
    uint64_t ctl = *iommu_p->iommu_ctrl_reg;
    uint64_t obp_tsb_pa = *iommu_p->iommu_tsb_base_addr_reg;
    uint64_t *base_tte_addr;

    DEBUG3(DBG_ATTACH, dip,
        "iommu_tsb_base_addr_reg=0x%08x (0x%08x.0x%08x)\n",
        iommu_p->iommu_tsb_base_addr_reg,
        (uint32_t)(*iommu_p->iommu_tsb_base_addr_reg >> 32),
        (uint32_t)(*iommu_p->iommu_tsb_base_addr_reg & 0xffffffff));

    obp_tsb_size = IOMMU_CTL_TO_TSBSIZE(ctl);
    obp_tsb_entries = IOMMU_TSBSIZE_TO_TSBENTRIES(obp_tsb_size);
    base_pg_index = iommu_p->dvma_end_pg - obp_tsb_entries + 1;
    base_tte_addr = iommu_p->iommu_tsb_vaddr +
        (iommu_p->iommu_tsb_entries - obp_tsb_entries);

    /*
     * old darwin prom does not set tsb size correctly, bail out.
     */
    if ((obp_tsb_size == IOMMU_DARWIN_BOGUS_TSBSIZE) &&
        (CHIP_TYPE(iommu_p->iommu_pci_p) == PCI_CHIP_SABRE))
        return;

    DEBUG3(DBG_ATTACH, dip, "iommu_preserve_tsb: kernel info\n"
        "iommu_tsb_vaddr=%08x copy to base_tte_addr=%08x "
        "base_pg_index=%x\n", iommu_p->iommu_tsb_vaddr,
        base_tte_addr, base_pg_index);

    DEBUG3(DBG_ATTACH | DBG_CONT, dip, "iommu_preserve_tsb: obp info "
        "obp_tsb_entries=0x%x obp_tsb_pa=%08x.%08x\n", obp_tsb_entries,
        (uint32_t)(obp_tsb_pa >> 32), (uint32_t)obp_tsb_pa);

    for (i = 0; i < obp_tsb_entries; i++) {
        uint64_t tte = lddphys(obp_tsb_pa + i * 8);
        caddr_t va;

        if (TTE_IS_INVALID(tte)) {
            DEBUG0(DBG_ATTACH | DBG_CONT, dip, ".");
            continue;
        }

        base_tte_addr[i] = tte;
        DEBUG3(DBG_ATTACH | DBG_CONT, dip,
            "\npreserve_tsb: (%x)=%08x.%08x\n", base_tte_addr + i,
            (uint_t)(tte >> 32), (uint_t)(tte & 0xffffffff));

        /*
         * permanantly reserve this page from dvma address space
         * resource map
         */

        va = (caddr_t)(IOMMU_PTOB(base_pg_index + i));
        (void) vmem_xalloc(iommu_p->iommu_dvma_map, IOMMU_PAGE_SIZE,
            IOMMU_PAGE_SIZE, 0, 0, va, va + IOMMU_PAGE_SIZE,
            VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
    }
}