8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/*
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * CDDL HEADER START
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus *
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * The contents of this file are subject to the terms of the
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * Common Development and Distribution License (the "License").
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * You may not use this file except in compliance with the License.
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus *
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * or http://www.opensolaris.org/os/licensing.
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * See the License for the specific language governing permissions
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * and limitations under the License.
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus *
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * When distributing Covered Code, include this CDDL HEADER in each
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * If applicable, add the following below this CDDL HEADER, with the
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * fields enclosed by brackets "[]" replaced with your own identifying
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * information: Portions Copyright [yyyy] [name of copyright owner]
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus *
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * CDDL HEADER END
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/*
eca2601cae391051acb146d28fba04237fe1eb85Randy Fishel * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * Use is subject to license terms.
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#ifndef _SYS_IOAT_H
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define _SYS_IOAT_H
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#ifdef __cplusplus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusextern "C" {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#endif
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#include <sys/types.h>
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#include <sys/dcopy.h>
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#include <sys/dcopy_device.h>
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* ioat ioctls */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOATIOC ('T'<< 8)
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef enum {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus IOAT_IOCTL_WRITE_REG = (IOATIOC | 0x0),
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus IOAT_IOCTL_READ_REG = (IOATIOC | 0x1),
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus IOAT_IOCTL_TEST = (IOATIOC | 0x2)
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus} ioat_ioctl_enum_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef struct ioat_ioctl_reg_s {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t size;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t addr;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t data;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus} ioat_ioctl_reg_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef ioat_ioctl_reg_t ioat_ioctl_wrreg_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef ioat_ioctl_reg_t ioat_ioctl_rdreg_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#ifdef _KERNEL
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* *** Driver Private Below *** */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* IOAT_DMACAPABILITY flags */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DMACAP_PAGEBREAK 0x1
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DMACAP_CRC 0x2
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DMACAP_MARKERSKIP 0x4
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DMACAP_XOR 0x8
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DMACAP_DCA 0x10
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* IOAT_INTRCTL bits */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_INTRCTL_MASTER_EN 0x1
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_INTRCTL_INTR_STAT 0x2
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* MMIO Registers */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHANCNT 0x0 /* 8-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_XFERCAP 0x1 /* 8-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_GENCTRL 0x2 /* 8-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_INTRCTL 0x3 /* 8-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_ATTNSTATUS 0x4 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CBVER 0x8 /* 8-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_PERPORT_OFF 0xA /* 16-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_INTRDELAY 0xC /* 16-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CSSTATUS 0xE /* 16-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DMACAPABILITY 0x10 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHANNELREG_OFFSET 0x80
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* Channel Registers */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHAN_CTL 0x0 /* 16-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHAN_COMP 0x2 /* 16-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHAN_CMPL_LO 0x18 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHAN_CMPL_HI 0x1C /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHAN_ERR 0x28 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHAN_ERRMASK 0x2C /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHAN_DCACTRL 0x30 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V1_CHAN_STS_LO 0x4 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V1_CHAN_STS_HI 0x8 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V1_CHAN_ADDR_LO 0x0C /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V1_CHAN_ADDR_HI 0x10 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V1_CHAN_CMD 0x14 /* 8-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V2_CHAN_CMD 0x4 /* 8-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V2_CHAN_CNT 0x6 /* 16-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V2_CHAN_STS_LO 0x8 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V2_CHAN_STS_HI 0xC /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V2_CHAN_ADDR_LO 0x10 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_V2_CHAN_ADDR_HI 0x14 /* 32-bit */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHAN_STS_ADDR_MASK 0xFFFFFFFFFFFFFFC0
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHAN_STS_XFER_MASK 0x3F
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CHAN_STS_FAIL_MASK 0x6
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CMPL_INDEX(channel) \
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus (((*channel->ic_cmpl & IOAT_CHAN_STS_ADDR_MASK) - \
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ring->cr_phys_desc) >> 6)
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_CMPL_FAILED(channel) \
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus (*channel->ic_cmpl & IOAT_CHAN_STS_FAIL_MASK)
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef struct ioat_chan_desc_s {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint32_t dd_res0;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint32_t dd_ctrl;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res1;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res2;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_next_desc;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res4;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res5;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res6;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res7;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus} ioat_chan_desc_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* dca dd_ctrl bits */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DESC_CTRL_OP_CNTX ((uint32_t)0xFF << 24)
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DESC_CTRL_CNTX_CHNG 0x1
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef struct ioat_chan_dca_desc_s {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint32_t dd_cntx;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint32_t dd_ctrl;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res1;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res2;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_next_desc;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res4;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res5;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res6;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res7;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus} ioat_chan_dca_desc_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* dma dd_ctrl bits */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DESC_CTRL_OP_DMA (0x0 << 24)
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DESC_DMACTRL_NULL 0x20
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DESC_CTRL_FENCE 0x10
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DESC_CTRL_CMPL 0x8
eca2601cae391051acb146d28fba04237fe1eb85Randy Fishel#define IOAT_DESC_CTRL_NODSTSNP 0x4
eca2601cae391051acb146d28fba04237fe1eb85Randy Fishel#define IOAT_DESC_CTRL_NOSRCSNP 0x2
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#define IOAT_DESC_CTRL_INTR 0x1
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef struct ioat_chan_dma_desc_s {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint32_t dd_size;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint32_t dd_ctrl;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_src_paddr;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_dest_paddr;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_next_desc;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_next_src_paddr; /* v2 only */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_next_dest_paddr; /* v2 only */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res6;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t dd_res7;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus} ioat_chan_dma_desc_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef enum {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus IOAT_CBv1,
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus IOAT_CBv2
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus} ioat_version_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* ioat private data per command */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef struct ioat_cmd_private_s {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t ip_generation;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t ip_index;
eca2601cae391051acb146d28fba04237fe1eb85Randy Fishel uint64_t ip_start;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus dcopy_cmd_t ip_next;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus} ioat_cmd_private_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* descriptor ring state */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef struct ioat_channel_ring_s {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* protects cr_cmpl_gen & cr_cmpl_last */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus kmutex_t cr_cmpl_mutex;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* desc ring generation for the last completion we saw */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t cr_cmpl_gen;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* last descriptor index we saw complete */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t cr_cmpl_last;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* protects cr_desc_* */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus kmutex_t cr_desc_mutex;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /*
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * last descriptor posted. used to update its next pointer when we
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * add a new desc. Also used to tack the completion (See comment for
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * cr_desc_gen_prev).
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t cr_desc_prev;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* where to put the next descriptor */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t cr_desc_next;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* what the current desc ring generation is */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t cr_desc_gen;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /*
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * used during cmd_post to track the last desc posted. cr_desc_next
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * and cr_desc_gen will be pointing to the next free desc after
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * writing the descriptor to the ring. But we want to track the
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * completion for the last descriptor posted.
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t cr_desc_gen_prev;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* the last desc in the ring (for wrap) */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t cr_desc_last;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* pointer to the head of the ring */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ioat_chan_desc_t *cr_desc;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* physical address of the head of the ring */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t cr_phys_desc;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* back pointer to the channel state */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus struct ioat_channel_s *cr_chan;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* for CB v2, number of desc posted (written to IOAT_V2_CHAN_CNT) */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t cr_post_cnt;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus} ioat_channel_ring_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* track channel state so we can handle a failure */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef enum {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus IOAT_CHANNEL_OK = 0,
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus IOAT_CHANNEL_IN_FAILURE = 1
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus} ic_channel_state_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef struct ioat_channel_s *ioat_channel_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusstruct ioat_channel_s {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* channel's ring state */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ioat_channel_ring_t *ic_ring;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* IOAT_CBv1 || IOAT_CBv2 */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ioat_version_t ic_ver;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /*
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * state to determine if it's OK to post the the channel and if all
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus * future polls should return failure.
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ic_channel_state_t ic_channel_state;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* channel command cache (*_cmd_alloc, *_cmd_free, etc) */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus kmem_cache_t *ic_cmd_cache;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* dcopy state for dcopy_device_channel_notify() call */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus dcopy_handle_t ic_dcopy_handle;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* location in memory where completions are DMA'ed into */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus volatile uint64_t *ic_cmpl;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* channel specific registers */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint8_t *ic_regs;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* if this channel is using DCA */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus boolean_t ic_dca_active;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* DCA ID the channel is currently pointing to */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint32_t ic_dca_current;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* devices channel number */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t ic_chan_num;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* number of descriptors in ring */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t ic_chan_desc_cnt;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* descriptor ring alloc state */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ddi_dma_handle_t ic_desc_dma_handle;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus size_t ic_desc_alloc_size;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ddi_acc_handle_t ic_desc_handle;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ddi_dma_cookie_t ic_desc_cookies;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* completion buffer alloc state */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ddi_dma_handle_t ic_cmpl_dma_handle;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus size_t ic_cmpl_alloc_size;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ddi_acc_handle_t ic_cmpl_handle;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ddi_dma_cookie_t ic_cmpl_cookie;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint64_t ic_phys_cmpl;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* if inuse, we need to re-init the channel during resume */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus boolean_t ic_inuse;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* backpointer to driver state */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus struct ioat_state_s *ic_state;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus};
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef struct ioat_rs_s *ioat_rs_hdl_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus/* driver state */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutustypedef struct ioat_state_s {
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus dev_info_t *is_dip;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus int is_instance;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus kmutex_t is_mutex;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* register handle and pointer to registers */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ddi_acc_handle_t is_reg_handle;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint8_t *is_genregs;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* IOAT_CBv1 || IOAT_CBv2 */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ioat_version_t is_ver;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* channel state */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ioat_channel_t is_channel;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus size_t is_chansize;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ioat_rs_hdl_t is_channel_rs;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ddi_iblock_cookie_t is_iblock_cookie;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* device info */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t is_chanoff;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t is_num_channels;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t is_maxxfer;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t is_cbver;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t is_intrdelay;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t is_status;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t is_capabilities;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus /* dcopy_device_register()/dcopy_device_unregister() state */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus dcopy_device_handle_t is_device_handle;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus dcopy_device_info_t is_deviceinfo;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus} ioat_state_t;
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusint ioat_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred,
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus int *rval);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusvoid ioat_rs_init(ioat_state_t *state, uint_t min_val, uint_t max_val,
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus ioat_rs_hdl_t *handle);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusvoid ioat_rs_fini(ioat_rs_hdl_t *handle);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusint ioat_rs_alloc(ioat_rs_hdl_t handle, uint_t *rs);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusvoid ioat_rs_free(ioat_rs_hdl_t handle, uint_t rs);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusint ioat_channel_init(ioat_state_t *state);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusvoid ioat_channel_fini(ioat_state_t *state);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusvoid ioat_channel_suspend(ioat_state_t *state);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusint ioat_channel_resume(ioat_state_t *state);
193974072f41a843678abf5f61979c748687e66bSherry Moorevoid ioat_channel_quiesce(ioat_state_t *);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusint ioat_channel_alloc(void *device_private, dcopy_handle_t handle, int flags,
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus uint_t size, dcopy_query_channel_t *info, void *channel_private);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusvoid ioat_channel_free(void *channel_private);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusvoid ioat_channel_intr(ioat_channel_t channel);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusint ioat_cmd_alloc(void *channel, int flags, dcopy_cmd_t *cmd);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusvoid ioat_cmd_free(void *channel, dcopy_cmd_t *cmd);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusint ioat_cmd_post(void *channel, dcopy_cmd_t cmd);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusint ioat_cmd_poll(void *channel, dcopy_cmd_t cmd);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutusvoid ioat_unregister_complete(void *device_private, int status);
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#endif /* _KERNEL */
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#ifdef __cplusplus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus}
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#endif
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus
8e50dcc9f00b393d43e6aa42b820bcbf1d3e1ce4brutus#endif /* _SYS_IOAT_H */