/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Portions Copyright (c) 2010, Oracle and/or its affiliates.
* All rights reserved.
*/
/*
* Copyright (c) 2009, Intel Corporation.
* All rights reserved.
*/
#include <sys/ddi.h>
#include <sys/archsystm.h>
#include <vm/hat_i86.h>
#include <sys/types.h>
#include <sys/cpu.h>
#include <sys/sysmacros.h>
#include <sys/immu.h>
/* invalidation queue table entry size */
#define QINV_ENTRY_SIZE 0x10
/* max value of Queue Size field of Invalidation Queue Address Register */
#define QINV_MAX_QUEUE_SIZE 0x7
/* status data size of invalidation wait descriptor */
#define QINV_SYNC_DATA_SIZE 0x4
/* invalidation queue head and tail */
#define QINV_IQA_HEAD(QH) BITX((QH), 18, 4)
#define QINV_IQA_TAIL_SHIFT 4
/* invalidation queue entry structure */
typedef struct qinv_inv_dsc {
uint64_t lo;
uint64_t hi;
} qinv_dsc_t;
/* physical contigous pages for invalidation queue */
typedef struct qinv_mem {
kmutex_t qinv_mem_lock;
ddi_dma_handle_t qinv_mem_dma_hdl;
ddi_acc_handle_t qinv_mem_acc_hdl;
caddr_t qinv_mem_vaddr;
paddr_t qinv_mem_paddr;
uint_t qinv_mem_size;
uint16_t qinv_mem_head;
uint16_t qinv_mem_tail;
} qinv_mem_t;
/*
* invalidation queue state
* This structure describes the state information of the
* invalidation queue table and related status memeory for
* invalidation wait descriptor
*
* qinv_table - invalidation queue table
* qinv_sync - sync status memory for invalidation wait descriptor
*/
typedef struct qinv {
qinv_mem_t qinv_table;
qinv_mem_t qinv_sync;
} qinv_t;
static void immu_qinv_inv_wait(immu_inv_wait_t *iwp);
static struct immu_flushops immu_qinv_flushops = {
immu_qinv_context_fsi,
immu_qinv_context_dsi,
immu_qinv_context_gbl,
immu_qinv_iotlb_psi,
immu_qinv_iotlb_dsi,
immu_qinv_iotlb_gbl,
immu_qinv_inv_wait
};
/* helper macro for making queue invalidation descriptor */
#define INV_DSC_TYPE(dsc) ((dsc)->lo & 0xF)
#define CC_INV_DSC_HIGH (0)
#define CC_INV_DSC_LOW(fm, sid, did, g) (((uint64_t)(fm) << 48) | \
((uint64_t)(sid) << 32) | \
((uint64_t)(did) << 16) | \
((uint64_t)(g) << 4) | \
1)
#define IOTLB_INV_DSC_HIGH(addr, ih, am) (((uint64_t)(addr)) | \
((uint64_t)(ih) << 6) | \
((uint64_t)(am)))
#define IOTLB_INV_DSC_LOW(did, dr, dw, g) (((uint64_t)(did) << 16) | \
((uint64_t)(dr) << 7) | \
((uint64_t)(dw) << 6) | \
((uint64_t)(g) << 4) | \
2)
#define DEV_IOTLB_INV_DSC_HIGH(addr, s) (((uint64_t)(addr)) | (s))
#define DEV_IOTLB_INV_DSC_LOW(sid, max_invs_pd) ( \
((uint64_t)(sid) << 32) | \
((uint64_t)(max_invs_pd) << 16) | \
3)
#define IEC_INV_DSC_HIGH (0)
#define IEC_INV_DSC_LOW(idx, im, g) (((uint64_t)(idx) << 32) | \
((uint64_t)(im) << 27) | \
((uint64_t)(g) << 4) | \
4)
#define INV_WAIT_DSC_HIGH(saddr) ((uint64_t)(saddr))
#define INV_WAIT_DSC_LOW(sdata, fn, sw, iflag) (((uint64_t)(sdata) << 32) | \
((uint64_t)(fn) << 6) | \
((uint64_t)(sw) << 5) | \
((uint64_t)(iflag) << 4) | \
5)
/*
* QS field of Invalidation Queue Address Register
* the size of invalidation queue is 1 << (qinv_iqa_qs + 8)
*/
static uint_t qinv_iqa_qs = 6;
/*
* the invalidate desctiptor type of queued invalidation interface
*/
static char *qinv_dsc_type[] = {
"Reserved",
"Context Cache Invalidate Descriptor",
"IOTLB Invalidate Descriptor",
"Device-IOTLB Invalidate Descriptor",
"Interrupt Entry Cache Invalidate Descriptor",
"Invalidation Wait Descriptor",
"Incorrect queue invalidation type"
};
#define QINV_MAX_DSC_TYPE (sizeof (qinv_dsc_type) / sizeof (char *))
/*
* the queued invalidation interface functions
*/
static void qinv_submit_inv_dsc(immu_t *immu, qinv_dsc_t *dsc);
static void qinv_context_common(immu_t *immu, uint8_t function_mask,
uint16_t source_id, uint_t domain_id, ctt_inv_g_t type);
static void qinv_iotlb_common(immu_t *immu, uint_t domain_id,
uint64_t addr, uint_t am, uint_t hint, tlb_inv_g_t type);
static void qinv_iec_common(immu_t *immu, uint_t iidx,
uint_t im, uint_t g);
static void immu_qinv_inv_wait(immu_inv_wait_t *iwp);
static void qinv_wait_sync(immu_t *immu, immu_inv_wait_t *iwp);
/*LINTED*/
static void qinv_dev_iotlb_common(immu_t *immu, uint16_t sid,
uint64_t addr, uint_t size, uint_t max_invs_pd);
/* submit invalidation request descriptor to invalidation queue */
static void
qinv_submit_inv_dsc(immu_t *immu, qinv_dsc_t *dsc)
{
qinv_t *qinv;
qinv_mem_t *qinv_table;
uint_t tail;
#ifdef DEBUG
uint_t count = 0;
#endif
qinv = (qinv_t *)immu->immu_qinv;
qinv_table = &(qinv->qinv_table);
mutex_enter(&qinv_table->qinv_mem_lock);
tail = qinv_table->qinv_mem_tail;
qinv_table->qinv_mem_tail++;
if (qinv_table->qinv_mem_tail == qinv_table->qinv_mem_size)
qinv_table->qinv_mem_tail = 0;
while (qinv_table->qinv_mem_head == qinv_table->qinv_mem_tail) {
#ifdef DEBUG
count++;
#endif
/*
* inv queue table exhausted, wait hardware to fetch
* next descriptor
*/
qinv_table->qinv_mem_head = QINV_IQA_HEAD(
immu_regs_get64(immu, IMMU_REG_INVAL_QH));
}
IMMU_DPROBE3(immu__qinv__sub, uint64_t, dsc->lo, uint64_t, dsc->hi,
uint_t, count);
bcopy(dsc, qinv_table->qinv_mem_vaddr + tail * QINV_ENTRY_SIZE,
QINV_ENTRY_SIZE);
immu_regs_put64(immu, IMMU_REG_INVAL_QT,
qinv_table->qinv_mem_tail << QINV_IQA_TAIL_SHIFT);
mutex_exit(&qinv_table->qinv_mem_lock);
}
/* queued invalidation interface -- invalidate context cache */
static void
qinv_context_common(immu_t *immu, uint8_t function_mask,
uint16_t source_id, uint_t domain_id, ctt_inv_g_t type)
{
qinv_dsc_t dsc;
dsc.lo = CC_INV_DSC_LOW(function_mask, source_id, domain_id, type);
dsc.hi = CC_INV_DSC_HIGH;
qinv_submit_inv_dsc(immu, &dsc);
}
/* queued invalidation interface -- invalidate iotlb */
static void
qinv_iotlb_common(immu_t *immu, uint_t domain_id,
uint64_t addr, uint_t am, uint_t hint, tlb_inv_g_t type)
{
qinv_dsc_t dsc;
uint8_t dr = 0;
uint8_t dw = 0;
if (IMMU_CAP_GET_DRD(immu->immu_regs_cap))
dr = 1;
if (IMMU_CAP_GET_DWD(immu->immu_regs_cap))
dw = 1;
switch (type) {
case TLB_INV_G_PAGE:
if (!IMMU_CAP_GET_PSI(immu->immu_regs_cap) ||
am > IMMU_CAP_GET_MAMV(immu->immu_regs_cap) ||
addr & IMMU_PAGEOFFSET) {
type = TLB_INV_G_DOMAIN;
goto qinv_ignore_psi;
}
dsc.lo = IOTLB_INV_DSC_LOW(domain_id, dr, dw, type);
dsc.hi = IOTLB_INV_DSC_HIGH(addr, hint, am);
break;
qinv_ignore_psi:
case TLB_INV_G_DOMAIN:
dsc.lo = IOTLB_INV_DSC_LOW(domain_id, dr, dw, type);
dsc.hi = 0;
break;
case TLB_INV_G_GLOBAL:
dsc.lo = IOTLB_INV_DSC_LOW(0, dr, dw, type);
dsc.hi = 0;
break;
default:
ddi_err(DER_WARN, NULL, "incorrect iotlb flush type");
return;
}
qinv_submit_inv_dsc(immu, &dsc);
}
/* queued invalidation interface -- invalidate dev_iotlb */
static void
qinv_dev_iotlb_common(immu_t *immu, uint16_t sid,
uint64_t addr, uint_t size, uint_t max_invs_pd)
{
qinv_dsc_t dsc;
dsc.lo = DEV_IOTLB_INV_DSC_LOW(sid, max_invs_pd);
dsc.hi = DEV_IOTLB_INV_DSC_HIGH(addr, size);
qinv_submit_inv_dsc(immu, &dsc);
}
/* queued invalidation interface -- invalidate interrupt entry cache */
static void
qinv_iec_common(immu_t *immu, uint_t iidx, uint_t im, uint_t g)
{
qinv_dsc_t dsc;
dsc.lo = IEC_INV_DSC_LOW(iidx, im, g);
dsc.hi = IEC_INV_DSC_HIGH;
qinv_submit_inv_dsc(immu, &dsc);
}
/*
* queued invalidation interface -- invalidation wait descriptor
* wait until the invalidation request finished
*/
static void
qinv_wait_sync(immu_t *immu, immu_inv_wait_t *iwp)
{
qinv_dsc_t dsc;
volatile uint32_t *status;
uint64_t paddr;
#ifdef DEBUG
uint_t count;
#endif
status = &iwp->iwp_vstatus;
paddr = iwp->iwp_pstatus;
*status = IMMU_INV_DATA_PENDING;
membar_producer();
/*
* sdata = IMMU_INV_DATA_DONE, fence = 1, sw = 1, if = 0
* indicate the invalidation wait descriptor completion by
* performing a coherent DWORD write to the status address,
* not by generating an invalidation completion event
*/
dsc.lo = INV_WAIT_DSC_LOW(IMMU_INV_DATA_DONE, 1, 1, 0);
dsc.hi = INV_WAIT_DSC_HIGH(paddr);
qinv_submit_inv_dsc(immu, &dsc);
if (iwp->iwp_sync) {
#ifdef DEBUG
count = 0;
while (*status != IMMU_INV_DATA_DONE) {
count++;
ht_pause();
}
DTRACE_PROBE2(immu__wait__sync, const char *, iwp->iwp_name,
uint_t, count);
#else
while (*status != IMMU_INV_DATA_DONE)
ht_pause();
#endif
}
}
static void
immu_qinv_inv_wait(immu_inv_wait_t *iwp)
{
volatile uint32_t *status = &iwp->iwp_vstatus;
#ifdef DEBUG
uint_t count;
count = 0;
while (*status != IMMU_INV_DATA_DONE) {
count++;
ht_pause();
}
DTRACE_PROBE2(immu__wait__async, const char *, iwp->iwp_name,
uint_t, count);
#else
while (*status != IMMU_INV_DATA_DONE)
ht_pause();
#endif
}
/*
* call ddi_dma_mem_alloc to allocate physical contigous
* pages for invalidation queue table
*/
static int
qinv_setup(immu_t *immu)
{
qinv_t *qinv;
size_t size;
ddi_dma_attr_t qinv_dma_attr = {
DMA_ATTR_V0,
0U,
0xffffffffffffffffULL,
0xffffffffU,
MMU_PAGESIZE, /* page aligned */
0x1,
0x1,
0xffffffffU,
0xffffffffffffffffULL,
1,
4,
0
};
ddi_device_acc_attr_t qinv_acc_attr = {
DDI_DEVICE_ATTR_V0,
DDI_NEVERSWAP_ACC,
DDI_STRICTORDER_ACC
};
mutex_init(&(immu->immu_qinv_lock), NULL, MUTEX_DRIVER, NULL);
mutex_enter(&(immu->immu_qinv_lock));
immu->immu_qinv = NULL;
if (!IMMU_ECAP_GET_QI(immu->immu_regs_excap) ||
immu_qinv_enable == B_FALSE) {
mutex_exit(&(immu->immu_qinv_lock));
return (DDI_SUCCESS);
}
if (qinv_iqa_qs > QINV_MAX_QUEUE_SIZE)
qinv_iqa_qs = QINV_MAX_QUEUE_SIZE;
qinv = kmem_zalloc(sizeof (qinv_t), KM_SLEEP);
if (ddi_dma_alloc_handle(root_devinfo,
&qinv_dma_attr, DDI_DMA_SLEEP, NULL,
&(qinv->qinv_table.qinv_mem_dma_hdl)) != DDI_SUCCESS) {
ddi_err(DER_WARN, root_devinfo,
"alloc invalidation queue table handler failed");
goto queue_table_handle_failed;
}
if (ddi_dma_alloc_handle(root_devinfo,
&qinv_dma_attr, DDI_DMA_SLEEP, NULL,
&(qinv->qinv_sync.qinv_mem_dma_hdl)) != DDI_SUCCESS) {
ddi_err(DER_WARN, root_devinfo,
"alloc invalidation queue sync mem handler failed");
goto sync_table_handle_failed;
}
qinv->qinv_table.qinv_mem_size = (1 << (qinv_iqa_qs + 8));
size = qinv->qinv_table.qinv_mem_size * QINV_ENTRY_SIZE;
/* alloc physical contiguous pages for invalidation queue */
if (ddi_dma_mem_alloc(qinv->qinv_table.qinv_mem_dma_hdl,
size,
&qinv_acc_attr,
DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED,
DDI_DMA_SLEEP,
NULL,
&(qinv->qinv_table.qinv_mem_vaddr),
&size,
&(qinv->qinv_table.qinv_mem_acc_hdl)) != DDI_SUCCESS) {
ddi_err(DER_WARN, root_devinfo,
"alloc invalidation queue table failed");
goto queue_table_mem_failed;
}
ASSERT(!((uintptr_t)qinv->qinv_table.qinv_mem_vaddr & MMU_PAGEOFFSET));
bzero(qinv->qinv_table.qinv_mem_vaddr, size);
/* get the base physical address of invalidation request queue */
qinv->qinv_table.qinv_mem_paddr = pfn_to_pa(
hat_getpfnum(kas.a_hat, qinv->qinv_table.qinv_mem_vaddr));
qinv->qinv_table.qinv_mem_head = qinv->qinv_table.qinv_mem_tail = 0;
qinv->qinv_sync.qinv_mem_size = qinv->qinv_table.qinv_mem_size;
size = qinv->qinv_sync.qinv_mem_size * QINV_SYNC_DATA_SIZE;
/* alloc status memory for invalidation wait descriptor */
if (ddi_dma_mem_alloc(qinv->qinv_sync.qinv_mem_dma_hdl,
size,
&qinv_acc_attr,
DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED,
DDI_DMA_SLEEP,
NULL,
&(qinv->qinv_sync.qinv_mem_vaddr),
&size,
&(qinv->qinv_sync.qinv_mem_acc_hdl)) != DDI_SUCCESS) {
ddi_err(DER_WARN, root_devinfo,
"alloc invalidation queue sync mem failed");
goto sync_table_mem_failed;
}
ASSERT(!((uintptr_t)qinv->qinv_sync.qinv_mem_vaddr & MMU_PAGEOFFSET));
bzero(qinv->qinv_sync.qinv_mem_vaddr, size);
qinv->qinv_sync.qinv_mem_paddr = pfn_to_pa(
hat_getpfnum(kas.a_hat, qinv->qinv_sync.qinv_mem_vaddr));
qinv->qinv_sync.qinv_mem_head = qinv->qinv_sync.qinv_mem_tail = 0;
mutex_init(&(qinv->qinv_table.qinv_mem_lock), NULL, MUTEX_DRIVER, NULL);
mutex_init(&(qinv->qinv_sync.qinv_mem_lock), NULL, MUTEX_DRIVER, NULL);
immu->immu_qinv = qinv;
mutex_exit(&(immu->immu_qinv_lock));
return (DDI_SUCCESS);
sync_table_mem_failed:
ddi_dma_mem_free(&(qinv->qinv_table.qinv_mem_acc_hdl));
queue_table_mem_failed:
ddi_dma_free_handle(&(qinv->qinv_sync.qinv_mem_dma_hdl));
sync_table_handle_failed:
ddi_dma_free_handle(&(qinv->qinv_table.qinv_mem_dma_hdl));
queue_table_handle_failed:
kmem_free(qinv, sizeof (qinv_t));
mutex_exit(&(immu->immu_qinv_lock));
return (DDI_FAILURE);
}
/*
* ###########################################################################
*
* Functions exported by immu_qinv.c
*
* ###########################################################################
*/
/*
* initialize invalidation request queue structure.
*/
int
immu_qinv_setup(list_t *listp)
{
immu_t *immu;
int nerr;
if (immu_qinv_enable == B_FALSE) {
return (DDI_FAILURE);
}
nerr = 0;
immu = list_head(listp);
for (; immu; immu = list_next(listp, immu)) {
if (qinv_setup(immu) == DDI_SUCCESS) {
immu->immu_qinv_setup = B_TRUE;
} else {
nerr++;
break;
}
}
return (nerr > 0 ? DDI_FAILURE : DDI_SUCCESS);
}
void
immu_qinv_startup(immu_t *immu)
{
qinv_t *qinv;
uint64_t qinv_reg_value;
if (immu->immu_qinv_setup == B_FALSE) {
return;
}
qinv = (qinv_t *)immu->immu_qinv;
qinv_reg_value = qinv->qinv_table.qinv_mem_paddr | qinv_iqa_qs;
immu_regs_qinv_enable(immu, qinv_reg_value);
immu->immu_flushops = &immu_qinv_flushops;
immu->immu_qinv_running = B_TRUE;
}
/*
* queued invalidation interface
* function based context cache invalidation
*/
void
immu_qinv_context_fsi(immu_t *immu, uint8_t function_mask,
uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp)
{
qinv_context_common(immu, function_mask, source_id,
domain_id, CTT_INV_G_DEVICE);
qinv_wait_sync(immu, iwp);
}
/*
* queued invalidation interface
* domain based context cache invalidation
*/
void
immu_qinv_context_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)
{
qinv_context_common(immu, 0, 0, domain_id, CTT_INV_G_DOMAIN);
qinv_wait_sync(immu, iwp);
}
/*
* queued invalidation interface
* invalidation global context cache
*/
void
immu_qinv_context_gbl(immu_t *immu, immu_inv_wait_t *iwp)
{
qinv_context_common(immu, 0, 0, 0, CTT_INV_G_GLOBAL);
qinv_wait_sync(immu, iwp);
}
/*
* queued invalidation interface
* paged based iotlb invalidation
*/
void
immu_qinv_iotlb_psi(immu_t *immu, uint_t domain_id,
uint64_t dvma, uint_t count, uint_t hint, immu_inv_wait_t *iwp)
{
uint_t am = 0;
uint_t max_am;
max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap);
/* choose page specified invalidation */
if (IMMU_CAP_GET_PSI(immu->immu_regs_cap)) {
while (am <= max_am) {
if ((ADDR_AM_OFFSET(IMMU_BTOP(dvma), am) + count)
<= ADDR_AM_MAX(am)) {
qinv_iotlb_common(immu, domain_id,
dvma, am, hint, TLB_INV_G_PAGE);
break;
}
am++;
}
if (am > max_am) {
qinv_iotlb_common(immu, domain_id,
dvma, 0, hint, TLB_INV_G_DOMAIN);
}
/* choose domain invalidation */
} else {
qinv_iotlb_common(immu, domain_id, dvma,
0, hint, TLB_INV_G_DOMAIN);
}
qinv_wait_sync(immu, iwp);
}
/*
* queued invalidation interface
* domain based iotlb invalidation
*/
void
immu_qinv_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)
{
qinv_iotlb_common(immu, domain_id, 0, 0, 0, TLB_INV_G_DOMAIN);
qinv_wait_sync(immu, iwp);
}
/*
* queued invalidation interface
* global iotlb invalidation
*/
void
immu_qinv_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp)
{
qinv_iotlb_common(immu, 0, 0, 0, 0, TLB_INV_G_GLOBAL);
qinv_wait_sync(immu, iwp);
}
/* queued invalidation interface -- global invalidate interrupt entry cache */
void
immu_qinv_intr_global(immu_t *immu, immu_inv_wait_t *iwp)
{
qinv_iec_common(immu, 0, 0, IEC_INV_GLOBAL);
qinv_wait_sync(immu, iwp);
}
/* queued invalidation interface -- invalidate single interrupt entry cache */
void
immu_qinv_intr_one_cache(immu_t *immu, uint_t iidx, immu_inv_wait_t *iwp)
{
qinv_iec_common(immu, iidx, 0, IEC_INV_INDEX);
qinv_wait_sync(immu, iwp);
}
/* queued invalidation interface -- invalidate interrupt entry caches */
void
immu_qinv_intr_caches(immu_t *immu, uint_t iidx, uint_t cnt,
immu_inv_wait_t *iwp)
{
uint_t i, mask = 0;
ASSERT(cnt != 0);
/* requested interrupt count is not a power of 2 */
if (!ISP2(cnt)) {
for (i = 0; i < cnt; i++) {
qinv_iec_common(immu, iidx + cnt, 0, IEC_INV_INDEX);
}
qinv_wait_sync(immu, iwp);
return;
}
while ((2 << mask) < cnt) {
mask++;
}
if (mask > IMMU_ECAP_GET_MHMV(immu->immu_regs_excap)) {
for (i = 0; i < cnt; i++) {
qinv_iec_common(immu, iidx + cnt, 0, IEC_INV_INDEX);
}
qinv_wait_sync(immu, iwp);
return;
}
qinv_iec_common(immu, iidx, mask, IEC_INV_INDEX);
qinv_wait_sync(immu, iwp);
}
void
immu_qinv_report_fault(immu_t *immu)
{
uint16_t head;
qinv_dsc_t *dsc;
qinv_t *qinv;
/* access qinv data */
mutex_enter(&(immu->immu_qinv_lock));
qinv = (qinv_t *)(immu->immu_qinv);
head = QINV_IQA_HEAD(
immu_regs_get64(immu, IMMU_REG_INVAL_QH));
dsc = (qinv_dsc_t *)(qinv->qinv_table.qinv_mem_vaddr
+ (head * QINV_ENTRY_SIZE));
/* report the error */
ddi_err(DER_WARN, immu->immu_dip,
"generated a fault when fetching a descriptor from the"
"\tinvalidation queue, or detects that the fetched"
"\tdescriptor is invalid. The head register is "
"0x%" PRIx64
"\tthe type is %s",
head,
qinv_dsc_type[MIN(INV_DSC_TYPE(dsc), QINV_MAX_DSC_TYPE)]);
mutex_exit(&(immu->immu_qinv_lock));
}