9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or http://www.opensolaris.org/os/licensing.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use is subject to license terms.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#ifndef _DAPL_TAVOR_HW_H
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define _DAPL_TAVOR_HW_H
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapl_tavor_hw.h
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Contains all the structure definitions and #defines for all Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware resources and registers.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Most of these definitions have been replicated from the tavor_hw.h
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * header file used by the tavor device driver.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#ifdef __cplusplus
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern "C" {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#include "dapl.h"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#include "dapl_tavor_ibtf.h"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Ownership flags used to define hardware or software ownership for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * various Tavor resources
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_HW_OWNER 0x1U
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_SW_OWNER 0x0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor Completion Queue Entries (CQE)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Each CQE contains enough information for the software to associate the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion with the Work Queue Element (WQE) to which it corresponds.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: The following structure is not #define'd with both little-endian
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and big-endian definitions. This is because each CQE's individual
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields are not directly accessed except through the macros defined below.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following defines are used for Tavor CQ error handling. Note: For
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQEs which correspond to error events, the Tavor device requires some
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * special handling by software. These defines are used to identify and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * extract the necessary information from each error CQE, including status
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * code (above), doorbell count, and whether a error completion is for a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * send or receive work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_ERR_STATUS_SHIFT 24
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_ERR_STATUS_MASK 0xFF
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_ERR_DBDCNT_MASK 0xFFFF
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SEND_ERR_OPCODE 0xFF
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_RECV_ERR_OPCODE 0xFE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQ_SYNC_AND_DB 0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQ_RECYCLE_ENTRY 1
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * These are the defines for the Tavor CQ entry types. They are also
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * specified by the Tavor register specification. They indicate what type
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of work request is completing (for successful completions). Note: The
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "SND" or "RCV" in each define is used to indicate whether the completion
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request was from the Send work queue or the Receive work queue on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the associated QP.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SND_RDMAWR 0x8
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SND_RDMAWR_IMM 0x9
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SND_SEND 0xA
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SND_SEND_IMM 0xB
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SND_RDMARD 0x10
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SND_ATOMIC_CS 0x11
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SND_ATOMIC_FA 0x12
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SND_BIND_MW 0x18
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_RCV_RECV_IMM 0x3
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_RCV_RECV_IMM2 0x5
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_RCV_RECV 0x2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_RCV_RECV2 0x4
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_RCV_RDMAWR_IMM 0x9
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_RCV_RDMAWR_IMM2 0xB
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * These are the defines for the Tavor CQ completion statuses. They are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * specified by the Tavor register specification.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SUCCESS 0x0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_LOC_LEN_ERR 0x1
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_LOC_OP_ERR 0x2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_LOC_EEC_ERR 0x3 /* unsupported: RD */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_LOC_PROT_ERR 0x4
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_WR_FLUSHED_ERR 0x5
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_MW_BIND_ERR 0x6
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_BAD_RESPONSE_ERR 0x10
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_LOCAL_ACCESS_ERR 0x11
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_REM_INV_REQ_ERR 0x12
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_REM_ACC_ERR 0x13
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_REM_OP_ERR 0x14
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_TRANS_TO_ERR 0x15
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_RNRNAK_TO_ERR 0x16
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_LOCAL_RDD_VIO_ERR 0x20 /* unsupported: RD */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_REM_INV_RD_REQ_ERR 0x21 /* unsupported: RD */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_EEC_REM_ABORTED_ERR 0x22 /* unsupported: RD */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_INV_EEC_NUM_ERR 0x23 /* unsupported: RD */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_INV_EEC_STATE_ERR 0x24 /* unsupported: RD */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_cqe_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t ver :4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t my_qpn :24;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :8;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t my_ee :24;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :8;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t rqpn :24;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t sl :4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t grh :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t ml_path :7;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t rlid :16;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t imm_eth_pkey_cred;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t byte_cnt;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t wqe_addr :26;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t wqe_sz :6;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t opcode :8;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t send_or_recv :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :15;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t owner :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t status :7;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_cqe_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_COMPLETION_RECV 0x0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_COMPLETION_SEND 0x1
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_DEFAULT_VERSION 0x0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following macros are used for extracting (and in some cases filling in)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information from CQEs
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_QPNUM_MASK 0x00FFFFFF
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_QPNUM_SHIFT 0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_DQPN_MASK 0x00FFFFFF
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_DQPN_SHIFT 0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SL_MASK 0xF0000000
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SL_SHIFT 28
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_GRH_MASK 0x00800000
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_GRH_SHIFT 23
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_PATHBITS_MASK 0x007F0000
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_PATHBITS_SHIFT 16
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_DLID_MASK 0x0000FFFF
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_DLID_SHIFT 0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_OPCODE_MASK 0xFF000000
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_OPCODE_SHIFT 24
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SENDRECV_MASK 0x00800000
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SENDRECV_SHIFT 23
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_OWNER_MASK 0x00000080
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_OWNER_SHIFT 7
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_QPNUM_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((BETOH_32(((uint32_t *)(cqe))[0]) & TAVOR_CQE_QPNUM_MASK) >> \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_CQE_QPNUM_SHIFT)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_DQPN_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((BETOH_32(((uint32_t *)(cqe))[2]) & TAVOR_CQE_DQPN_MASK) >> \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_CQE_DQPN_SHIFT)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SL_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((BETOH_32(((uint32_t *)(cqe))[3]) & TAVOR_CQE_SL_MASK) >> \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_CQE_SL_SHIFT)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_GRH_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((BETOH_32(((uint32_t *)(cqe))[3]) & TAVOR_CQE_GRH_MASK) >> \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_CQE_GRH_SHIFT)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_PATHBITS_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((BETOH_32(((uint32_t *)(cqe))[3]) & TAVOR_CQE_PATHBITS_MASK) >>\
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_CQE_PATHBITS_SHIFT)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_DLID_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((BETOH_32(((uint32_t *)(cqe))[3]) & TAVOR_CQE_DLID_MASK) >> \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_CQE_DLID_SHIFT)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_IMM_ETH_PKEY_CRED_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (BETOH_32(((uint32_t *)(cqe))[4]))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_IMM_ETH_PKEY_CRED_SET(cqe, arg) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (((uint32_t *)(cqe))[4] = HTOBE_32((arg)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_BYTECNT_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (BETOH_32(((uint32_t *)(cqe))[5]))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_WQEADDRSZ_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (BETOH_32(((uint32_t *)(cqe))[6]))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_WQEADDRSZ_SET(cqe, arg) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (((uint32_t *)(cqe))[6] = HTOBE_32((arg)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_OPCODE_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((BETOH_32(((uint32_t *)(cqe))[7]) & TAVOR_CQE_OPCODE_MASK) >> \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_CQE_OPCODE_SHIFT)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_SENDRECV_GET(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((BETOH_32(((uint32_t *)(cqe))[7]) & TAVOR_CQE_SENDRECV_MASK) >>\
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_CQE_SENDRECV_SHIFT)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_OWNER_IS_SW(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (((BETOH_32(((uint32_t *)(cqe))[7]) & TAVOR_CQE_OWNER_MASK) >> \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_CQE_OWNER_SHIFT) == TAVOR_SW_OWNER)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQE_OWNER_SET_HW(cqe) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (((uint32_t *)(cqe))[7] = \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor BETOH_32((TAVOR_HW_OWNER << TAVOR_CQE_OWNER_SHIFT) & \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_CQE_OWNER_MASK))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor User Access Region (UAR)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor doorbells are each rung by writing to the doorbell registers that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * form a User Access Region (UAR). A doorbell is a write-only hardware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * register which enables passing information from software to hardware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with minimum software latency. A write operation from the host software
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to these doorbell registers passes information about the HCA resources
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and initiates processing of the doorbell data. There are 6 types of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * doorbells in Tavor.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Send Doorbell" for synchronizing the attachment of a WQE (or a chain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of WQEs) to the send queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "RD Send Doorbell" (Same as above, except for RD QPs) is not supported.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Receive Doorbell" for synchronizing the attachment of a WQE (or a chain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of WQEs) to the receive queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "CQ Doorbell" for updating the CQ consumer index and requesting
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion notifications.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "EQ Doorbell" for updating the EQ consumer index, arming interrupt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * triggering, and disarming CQ notification requests.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "InfiniBlast" (which would have enabled access to the "InfiniBlast
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * buffer") is not supported.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: The tavor_hw_uar_t below is the container for all of the various
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * doorbell types. Below we first define several structures which make up
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the contents of those doorbell types.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note also: The following structures are not #define'd with both little-
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * endian and big-endian definitions. This is because each doorbell type
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is not directly accessed except through a single ddi_put64() operation
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (see tavor_qp_send_doorbell, tavor_qp_recv_doorbell, tavor_cq_doorbell,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or tavor_eq_doorbell)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_uar_send_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t nda :26;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t fence :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t nopcode :5;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t qpn :24;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :2;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t nds :6;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_uar_send_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QPSNDDB_NDA_MASK 0xFFFFFFC0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QPSNDDB_NDA_SHIFT 0x20
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QPSNDDB_F_SHIFT 0x25
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QPSNDDB_NOPCODE_SHIFT 0x20
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QPSNDDB_QPN_SHIFT 0x8
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_uar_recv_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t nda :26;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t nds :6;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t qpn :24;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t credits :8;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_uar_recv_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QPRCVDB_NDA_MASK 0xFFFFFFC0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QPRCVDB_NDA_SHIFT 0x20
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QPRCVDB_NDS_SHIFT 0x20
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QPRCVDB_QPN_SHIFT 0x8
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* Max descriptors per Tavor doorbell */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QP_MAXDESC_PER_DB 256
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_uar_cq_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t cmd :8;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t cqn :24;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t param;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_uar_cq_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQDB_CMD_SHIFT 0x38
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQDB_CQN_SHIFT 0x20
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQDB_INCR_CONSINDX 0x01
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQDB_NOTIFY_CQ 0x02
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQDB_NOTIFY_CQ_SOLICIT 0x03
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQDB_SET_CONSINDX 0x04
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQDB_NOTIFY_NCQ 0x05
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* Default value for use in NOTIFY_CQ doorbell */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_CQDB_DEFAULT_PARAM 0xFFFFFFFF
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_uar_eq_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t cmd :8;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :18;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t eqn :6;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t param;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_uar_eq_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_uar_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t rsrv0[4]; /* "RD Send" unsupported */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t send; /* tavor_hw_uar_send_t */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t recv; /* tavor_hw_uar_recv_t */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t cq; /* tavor_hw_uar_cq_t */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t eq; /* tavor_hw_uar_eq_t */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t rsrv1[244];
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t iblast[256]; /* "InfiniBlast" unsupported */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_uar_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_uar32_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t rsrv0[4]; /* "RD Send" unsupported */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t send[2]; /* tavor_hw_uar_send_t */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t recv[2]; /* tavor_hw_uar_recv_t */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t cq[2]; /* tavor_hw_uar_cq_t */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t eq[2]; /* tavor_hw_uar_eq_t */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t rsrv1[244];
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t iblast[256]; /* "InfiniBlast" unsupported */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_uar32_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor Send Work Queue Element (WQE)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * A Tavor Send WQE is built of the following segments, each of which is a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * multiple of 16 bytes. Note: Each individual WQE may contain only a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * subset of these segments described below (according to the operation type
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and transport type of the QP).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The first 16 bytes of ever WQE are formed from the "Next/Ctrl" segment.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This segment contains the address of the next WQE to be executed and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information required in order to allocate the resources to execute the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * next WQE. The "Ctrl" part of this segment contains the control
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information required to execute the WQE, including the opcode and other
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * control information.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The "Datagram" segment contains address information required in order to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * form a UD message.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The "Bind" segment contains the parameters required for a Bind Memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Window operation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The "Remote Address" segment is present only in RDMA or Atomic WQEs and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * specifies remote virtual addresses and RKey, respectively. Length of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the remote access is calculated from the scatter/gather list (for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * RDMA-write/RDMA-read) or set to eight (for Atomic).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The "Atomic" segment is present only in Atomic WQEs and specifies
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Swap/Add and Compare data.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: The following structures are not #define'd with both little-endian
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and big-endian definitions. This is because their individual fields are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not directly accessed except through macros defined below.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_snd_wqe_nextctrl_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t next_wqe_addr :26;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t nopcode :5;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t next_eec :24;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t dbd :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t fence :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t nds :6;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :28;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t c :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t e :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t s :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t i :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t immediate :32;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_snd_wqe_nextctrl_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_NDA_MASK 0x00000000FFFFFFC0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_NDS_MASK 0x3F
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_DBD_MASK 0x80
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_FENCE_MASK 0x40
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_NOPCODE_RDMAW 0x8
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_NOPCODE_RDMAWI 0x9
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_NOPCODE_SEND 0xA
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_NOPCODE_SENDI 0xB
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_NOPCODE_RDMAR 0x10
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_NOPCODE_ATMCS 0x11
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_NOPCODE_ATMFA 0x12
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_NOPCODE_BIND 0x18
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_SIGNALED_MASK 0x800000000ULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_EVENT_MASK 0x400000000ULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_SOLICIT_MASK 0x200000000ULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SEND_IMMEDIATE_MASK 0x100000000ULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SENDHDR_UD_AV_MASK 0xFFFFFFFFFFFFFFE0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SENDHDR_UD_DQPN_MASK 0xFFFFFF
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_snd_wqe_bind_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t ae :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t rw :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t rr :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :29;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :32;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t new_rkey;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t reg_lkey;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_snd_wqe_bind_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SENDHDR_BIND_ATOM 0x8000000000000000ULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SENDHDR_BIND_WR 0x4000000000000000ULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SENDHDR_BIND_RD 0x2000000000000000ULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_snd_wqe_remaddr_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t vaddr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t rkey;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :32;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_snd_wqe_remaddr_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor Receive Work Queue Element (WQE)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Like the Send WQE, the Receive WQE is built of 16-byte segments. The
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segment is the "Next/Ctrl" segment (defined below). It is followed by
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * some number of scatter list entries for the incoming message.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The format of the scatter-gather list entries is also shown below. For
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Receive WQEs the "inline_data" field must be cleared (i.e. data segments
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cannot contain inline data).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_rcv_wqe_nextctrl_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t next_wqe_addr :26;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :5;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t one :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :24;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t dbd :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t nds :6;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :28;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t c :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t e :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :2;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t :32;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_rcv_wqe_nextctrl_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This bit must be set in the next/ctrl field of all Receive WQEs
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as a workaround to a Tavor hardware erratum related to having
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the first 32-bits in the WQE set to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_RCV_WQE_NDA0_WA_MASK 0x0000000100000000ULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_RCV_SIGNALED_MASK 0x800000000ULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_RCV_EVENT_MASK 0x400000000ULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_hw_wqe_sgl_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t inline_data :1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t byte_cnt :31;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t lkey;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_hw_wqe_sgl_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SGL_BYTE_CNT_MASK 0x7FFFFFFF
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_SGL_INLINE_MASK 0x80000000
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The tavor_sw_wqe_dbinfo_t structure is used internally by the Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * driver to return information (from the tavor_wqe_mlx_build_nextctl() and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wqe_send_build_nextctl() routines) regarding the type of Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * doorbell necessary.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortypedef struct tavor_sw_wqe_dbinfo_s {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint_t db_nopcode;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint_t db_fence;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor} tavor_sw_wqe_dbinfo_t;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following macros are used for building each of the individual
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments that can make up a Tavor WQE. Note: We try not to use the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * structures (with their associated bitfields) here, instead opting to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * build and put 64-bit or 32-bit chunks to the WQEs as appropriate,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * primarily because using the bitfields appears to force more read-modify-
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * write operations.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_BUILD_REMADDR - Builds Remote Address Segment using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * RDMA info from the work request
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_BUILD_BIND - Builds the Bind Memory Window
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Segment using bind info from the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_LINKNEXT - Links the current WQE to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * previous one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_LINKFIRST - Links the first WQE on the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * chain to the previous WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_BUILD_REMADDR(ra, wr_rdma) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor{ \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t *tmp; \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp = (uint64_t *)(ra); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp[0] = HTOBE_64((wr_rdma)->rdma_raddr); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp[1] = HTOBE_64((uint64_t)(wr_rdma)->rdma_rkey << 32); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor}
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_BUILD_BIND(bn, wr_bind) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor{ \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t *tmp; \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t bn0_tmp; \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_bind_flags_t bind_flags; \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp = (uint64_t *)(bn); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bind_flags = (wr_bind)->bind_flags; \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bn0_tmp = (bind_flags & IBT_WR_BIND_ATOMIC) ? \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_SENDHDR_BIND_ATOM : 0; \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bn0_tmp |= (bind_flags & IBT_WR_BIND_WRITE) ? \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_SENDHDR_BIND_WR : 0; \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bn0_tmp |= (bind_flags & IBT_WR_BIND_READ) ? \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_SENDHDR_BIND_RD : 0; \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp[0] = HTOBE_64(bn0_tmp); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp[1] = HTOBE_64(((uint64_t)(wr_bind)->bind_rkey_out << 32) | \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (wr_bind)->bind_lkey); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp[2] = HTOBE_64((wr_bind)->bind_va); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp[3] = HTOBE_64((wr_bind)->bind_len); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor}
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_BUILD_DATA_SEG(ds, sgl) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor{ \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t *tmp; \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp = (uint64_t *)(ds); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp[0] = HTOBE_64(((uint64_t)((sgl)->ds_len & \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_SGL_BYTE_CNT_MASK) << 32) | (sgl)->ds_key); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp[1] = HTOBE_64((sgl)->ds_va); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor}
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_LINKNEXT(prev, ctrl, next) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor{ \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t *)(prev))[1] = HTOBE_64((ctrl)); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t *)(prev))[0] = HTOBE_64((next)); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor}
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_WQE_LINKFIRST(prev, next) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor{ \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t *)(prev))[0] = HTOBE_64((next)); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor}
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following macro is used to convert WQE address and size into the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "wqeaddrsz" value needed in the tavor_wrid_entry_t (see below).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QP_WQEADDRSZ(addr, size) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((((uintptr_t)(addr)) & ~TAVOR_WQE_NDS_MASK) | \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((size) & TAVOR_WQE_NDS_MASK))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following macros are used to calculate pointers to the Send or Receive
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQEs on a given QP, respectively
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QP_SQ_ENTRY(qp, tail) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t *)((uintptr_t)((qp)->qp_sq_buf) + \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((tail) * (qp)->qp_sq_wqesz)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QP_SQ_DESC(qp, tail) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint32_t)((qp)->qp_sq_desc_addr + \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((tail) * (qp)->qp_sq_wqesz)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QP_RQ_ENTRY(qp, tail) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t *)((uintptr_t)((qp)->qp_rq_buf) + \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((tail) * (qp)->qp_rq_wqesz)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_QP_RQ_DESC(qp, tail) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint32_t)((qp)->qp_rq_desc_addr + \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((tail) * (qp)->qp_rq_wqesz)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_SRQ_RQ_ENTRY(srq, tail) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t *)((uintptr_t)((srq)->srq_wq_buf) + \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((tail) * (srq)->srq_wq_wqesz)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_SRQ_RQ_DESC(srq, tail) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint32_t)((srq)->srq_wq_desc_addr + \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((tail) * (srq)->srq_wq_wqesz)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_SRQ_WQ_INDEX(srq_wq_desc_addr, desc_addr, wqesz) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint32_t)(((desc_addr) - (srq_wq_desc_addr)) / (wqesz)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_SRQ_WQ_ENTRY(srq, index) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t *)(((uintptr_t)(srq)->srq_addr) + \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((index) * (srq)->srq_wq_wqesz)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Maximum header before the data bytes when inlining data.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Header" includes the link (nextctrl) struct, a remote address struct
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (only for RDMA Write, not for Send) and the 32-bit byte count field.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_INLINE_HEADER_SIZE_MAX 0x40 /* from tavor driver */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_INLINE_HEADER_SIZE_RDMAW \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (sizeof (tavor_hw_snd_wqe_nextctrl_t) + \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (tavor_hw_snd_wqe_remaddr_t) + \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (uint32_t))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define TAVOR_INLINE_HEADER_SIZE_SEND \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (sizeof (tavor_hw_snd_wqe_nextctrl_t) + \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (uint32_t))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/*
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Function signatures
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern int dapls_tavor_max_inline(void);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#ifdef __cplusplus
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor}
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* _DAPL_TAVOR_HW_H */