Lines Matching defs:qp
53 static int tavor_wqe_send_build(tavor_state_t *state, tavor_qphdl_t qp,
57 uint64_t *prev_desc, tavor_sw_wqe_dbinfo_t *dbinfo, tavor_qphdl_t qp);
58 static int tavor_wqe_mlx_build(tavor_state_t *state, tavor_qphdl_t qp,
62 tavor_qphdl_t qp);
63 static int tavor_wqe_recv_build(tavor_state_t *state, tavor_qphdl_t qp,
66 uint64_t *prev, tavor_qphdl_t qp);
86 static void tavor_wrid_wqhdr_lock_both(tavor_qphdl_t qp);
87 static void tavor_wrid_wqhdr_unlock_both(tavor_qphdl_t qp);
96 tavor_post_send(tavor_state_t *state, tavor_qphdl_t qp,
119 if (qp->qp_is_umap) {
129 mutex_enter(&qp->qp_lock);
135 if ((qp->qp_state == TAVOR_QP_RESET) ||
136 (qp->qp_state == TAVOR_QP_INIT) ||
137 (qp->qp_state == TAVOR_QP_RTR)) {
138 mutex_exit(&qp->qp_lock);
146 mutex_enter(&qp->qp_sq_wqhdr->wq_wrid_wql->wql_lock);
147 wridlist = qp->qp_sq_wqhdr->wq_wrid_post;
150 qsize_msk = qp->qp_sq_wqhdr->wq_size - 1;
151 tail = qp->qp_sq_wqhdr->wq_tail;
152 head = qp->qp_sq_wqhdr->wq_head;
175 prev = TAVOR_QP_SQ_ENTRY(qp, tail);
198 if (qp->qp_sq_wqhdr->wq_full != 0) {
213 qp->qp_sq_wqhdr->wq_full = 1;
220 desc = TAVOR_QP_SQ_ENTRY(qp, tail);
228 status = tavor_wqe_send_build(state, qp,
242 ((uint64_t)(uintptr_t)desc - qp->qp_desc_off),
244 if ((qp->qp_sq_sigtype == TAVOR_QP_SQ_ALL_SIGNALED) ||
250 tavor_wrid_add_entry(qp->qp_sq_wqhdr,
264 if (qp->qp_is_special) {
266 desc, desc_sz, prev, NULL, qp);
270 prev, NULL, qp);
302 if (qp->qp_is_special) {
304 currindx - 1], NULL, 0, prev, NULL, qp);
308 prev, NULL, qp);
315 tavor_wqe_sync(qp, sync_from, sync_to, TAVOR_WR_SEND,
323 if (qp->qp_is_special) {
325 qp->qp_sq_lastwqeaddr, &dbinfo, qp);
328 first, first_sz, qp->qp_sq_lastwqeaddr,
329 &dbinfo, qp);
338 if (qp->qp_sq_lastwqeaddr != NULL) {
341 tavor_wqe_sync(qp, sync_from, sync_to,
364 qp->qp_sq_lastwqeaddr = desc;
365 qp->qp_sq_wqhdr->wq_tail = tail;
369 (uint32_t)((uintptr_t)first - qp->qp_desc_off),
370 first_sz, qp->qp_qpnum, dbinfo.db_fence,
383 mutex_exit(&qp->qp_sq_wqhdr->wq_wrid_wql->wql_lock);
384 mutex_exit(&qp->qp_lock);
396 tavor_post_recv(tavor_state_t *state, tavor_qphdl_t qp,
416 if (qp->qp_is_umap) {
426 mutex_enter(&qp->qp_lock);
431 if (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
432 mutex_exit(&qp->qp_lock);
442 if (qp->qp_state == TAVOR_QP_RESET) {
443 mutex_exit(&qp->qp_lock);
451 mutex_enter(&qp->qp_rq_wqhdr->wq_wrid_wql->wql_lock);
454 qsize_msk = qp->qp_rq_wqhdr->wq_size - 1;
455 tail = qp->qp_rq_wqhdr->wq_tail;
456 head = qp->qp_rq_wqhdr->wq_head;
479 prev = TAVOR_QP_RQ_ENTRY(qp, tail);
501 if (qp->qp_rq_wqhdr->wq_full != 0) {
516 qp->qp_rq_wqhdr->wq_full = 1;
523 desc = TAVOR_QP_RQ_ENTRY(qp, tail);
531 status = tavor_wqe_recv_build(state, qp, &wr[wrindx],
548 ((uint64_t)(uintptr_t)desc - qp->qp_desc_off),
552 tavor_wrid_add_entry(qp->qp_rq_wqhdr,
563 qp);
591 tavor_wqe_recv_linknext(NULL, 0, prev, qp);
597 tavor_wqe_sync(qp, sync_from, sync_to, TAVOR_WR_RECV,
605 qp->qp_rq_lastwqeaddr, qp);
613 if (qp->qp_rq_lastwqeaddr != NULL) {
616 tavor_wqe_sync(qp, sync_from, sync_to,
621 qp->qp_rq_lastwqeaddr = desc;
622 qp->qp_rq_wqhdr->wq_tail = tail;
626 (uint32_t)((uintptr_t)first - qp->qp_desc_off),
627 first_sz, qp->qp_qpnum, (chainlen % maxdb));
639 mutex_exit(&qp->qp_rq_wqhdr->wq_wrid_wql->wql_lock);
640 mutex_exit(&qp->qp_lock);
943 tavor_wqe_send_build(tavor_state_t *state, tavor_qphdl_t qp,
959 ASSERT(MUTEX_HELD(&qp->qp_lock));
975 if (qp->qp_serv_type != TAVOR_QP_UD) {
998 if (qp->qp_is_special) {
999 status = tavor_wqe_mlx_build(state, qp, wr, desc, size);
1026 TAVOR_WQE_BUILD_UD(qp, ud, ah, wr);
1036 if (qp->qp_serv_type != TAVOR_QP_RC) {
1081 TAVOR_WQE_BUILD_REMADDR(qp, rc, &wr->wr.rc.rcwr.rdma);
1106 TAVOR_WQE_BUILD_RC_ATOMIC_REMADDR(qp, rc, wr);
1107 TAVOR_WQE_BUILD_ATOMIC(qp, at, wr->wr.rc.rcwr.atomic);
1146 TAVOR_WQE_BUILD_BIND(qp, bn, wr->wr.rc.rcwr.bind);
1162 if (qp->qp_serv_type != TAVOR_QP_UC) {
1204 TAVOR_WQE_BUILD_REMADDR(qp, uc, &wr->wr.uc.ucwr.rdma);
1235 TAVOR_WQE_BUILD_BIND(qp, bn, wr->wr.uc.ucwr.bind);
1261 if (nds > qp->qp_sq_sgl) {
1287 TAVOR_WQE_BUILD_DATA_SEG(qp, &ds[num_ds], &sgl[i]);
1306 tavor_sw_wqe_dbinfo_t *dbinfo, tavor_qphdl_t qp)
1358 - qp->qp_desc_off);
1401 TAVOR_WQE_LINKFIRST(qp, prev_desc, next);
1423 TAVOR_WQE_LINKNEXT(qp, prev_desc, ctrl, next);
1433 tavor_wqe_mlx_build(tavor_state_t *state, tavor_qphdl_t qp,
1452 ASSERT(MUTEX_HELD(&qp->qp_lock));
1485 (qp->qp_is_special == TAVOR_QP_GSI)) {
1510 TAVOR_WQE_BUILD_INLINE(qp, &ds[0], desc_sz);
1534 TAVOR_WQE_BUILD_MLX_LRH(lrh, qp, udav, pktlen);
1547 if (qp->qp_is_special == TAVOR_QP_SMI) {
1554 TAVOR_WQE_BUILD_MLX_GRH(state, grh, qp, udav, pktlen);
1568 TAVOR_WQE_BUILD_MLX_BTH(state, bth, qp, wr);
1574 TAVOR_WQE_BUILD_MLX_DETH(deth, qp);
1588 if (nds > qp->qp_sq_sgl) {
1616 TAVOR_WQE_BUILD_DATA_SEG(qp, &ds[num_ds], &sgl[i]);
1625 if (qp->qp_is_special == TAVOR_QP_SMI) {
1647 if (qp->qp_is_special == TAVOR_QP_SMI) {
1657 TAVOR_WQE_BUILD_INLINE_ICRC(qp, &ds[num_ds], 4, 0);
1675 tavor_qphdl_t qp)
1698 (uintptr_t)curr_desc - qp->qp_desc_off);
1738 TAVOR_WQE_LINKFIRST(qp, prev_desc, next);
1760 if (qp->qp_is_special == TAVOR_QP_SMI) {
1780 if (qp->qp_is_special != TAVOR_QP_SMI) {
1794 TAVOR_WQE_LINKNEXT(qp, prev_desc, ctrl, next);
1805 tavor_wqe_recv_build(tavor_state_t *state, tavor_qphdl_t qp,
1813 ASSERT(MUTEX_HELD(&qp->qp_lock));
1816 if ((qp->qp_serv_type != TAVOR_QP_UD) &&
1817 (qp->qp_serv_type != TAVOR_QP_RC) &&
1818 (qp->qp_serv_type != TAVOR_QP_UC)) {
1831 if (wr->wr_nds > qp->qp_rq_sgl) {
1857 TAVOR_WQE_BUILD_DATA_SEG(qp, &ds[num_ds], &wr->wr_sgl[i]);
1875 uint64_t *prev_desc, tavor_qphdl_t qp)
1893 qp->qp_desc_off);
1910 TAVOR_WQE_LINKNEXT(qp, prev_desc, 0, next);
2080 tavor_qphdl_t qp;
2098 qp = (tavor_qphdl_t)hdl;
2099 is_sync_req = qp->qp_sync;
2101 dmahdl = qp->qp_mrhdl->mr_bindinfo.bi_dmahdl;
2115 wqe_from = TAVOR_QP_SQ_ENTRY(qp, sync_from);
2116 wqe_to = TAVOR_QP_SQ_ENTRY(qp, sync_to);
2117 qsize = qp->qp_sq_bufsz;
2119 wqe_base = TAVOR_QP_SQ_ENTRY(qp, 0);
2120 wqe_top = TAVOR_QP_SQ_ENTRY(qp, qsize);
2122 wqe_from = TAVOR_QP_RQ_ENTRY(qp, sync_from);
2123 wqe_to = TAVOR_QP_RQ_ENTRY(qp, sync_to);
2124 qsize = qp->qp_rq_bufsz;
2126 wqe_base = TAVOR_QP_RQ_ENTRY(qp, 0);
2127 wqe_top = TAVOR_QP_RQ_ENTRY(qp, qsize);
2328 tavor_wrid_from_reset_handling(tavor_state_t *state, tavor_qphdl_t qp)
2344 tavor_wrid_wqhdr_lock_both(qp);
2345 swq = tavor_wrid_wqhdr_find(qp->qp_sq_cqhdl, qp->qp_qpnum,
2350 swq = tavor_wrid_wqhdr_create(state, qp->qp_sq_cqhdl,
2351 qp->qp_qpnum, TAVOR_WR_SEND, create_wql);
2357 tavor_wrid_wqhdr_unlock_both(qp);
2365 qp->qp_sq_wqhdr = swq;
2366 swq->wq_size = qp->qp_sq_bufsz;
2383 tavor_cq_wqhdr_remove(qp->qp_sq_cqhdl, swq);
2386 tavor_wrid_wqhdr_unlock_both(qp);
2400 qp_srq_en = qp->qp_srq_en;
2403 mutex_enter(&qp->qp_srqhdl->srq_lock);
2406 mutex_enter(&qp->qp_srqhdl->srq_lock);
2415 rwq = tavor_wrid_wqhdr_find(qp->qp_rq_cqhdl, qp->qp_qpnum,
2429 qp->qp_srqhdl->srq_wrid_wql != NULL) {
2433 rwq = tavor_wrid_wqhdr_create(state, qp->qp_rq_cqhdl,
2434 qp->qp_qpnum, TAVOR_WR_RECV, create_wql);
2446 tavor_cq_wqhdr_remove(qp->qp_sq_cqhdl,
2451 mutex_exit(&qp->qp_srqhdl->srq_lock);
2454 mutex_exit(&qp->qp_srqhdl->srq_lock);
2458 tavor_wrid_wqhdr_unlock_both(qp);
2477 rwq->wq_size = qp->qp_srqhdl->srq_wq_bufsz;
2478 if (qp->qp_srqhdl->srq_wrid_wql == NULL) {
2479 qp->qp_srqhdl->srq_wrid_wql = rwq->wq_wrid_wql;
2481 rwq->wq_wrid_wql = qp->qp_srqhdl->srq_wrid_wql;
2483 tavor_wql_refcnt_inc(qp->qp_srqhdl->srq_wrid_wql);
2486 rwq->wq_size = qp->qp_rq_bufsz;
2489 qp->qp_rq_wqhdr = rwq;
2504 qp->qp_srqhdl->srq_wridlist != NULL) {
2506 r_wridlist = qp->qp_srqhdl->srq_wridlist;
2528 tavor_cq_wqhdr_remove(qp->qp_sq_cqhdl, swq);
2531 tavor_cq_wqhdr_remove(qp->qp_rq_cqhdl, rwq);
2535 mutex_exit(&qp->qp_srqhdl->srq_lock);
2538 mutex_exit(&qp->qp_srqhdl->srq_lock);
2542 tavor_wrid_wqhdr_unlock_both(qp);
2569 qp->qp_srqhdl->srq_wridlist == NULL) {
2572 qp->qp_srqhdl->srq_wridlist = r_wridlist;
2575 if (qp->qp_srqhdl->srq_is_umap == 0) {
2577 tavor_wrid_list_srq_init(r_wridlist, qp->qp_srqhdl, 0);
2590 mutex_exit(&qp->qp_srqhdl->srq_lock);
2593 mutex_exit(&qp->qp_srqhdl->srq_lock);
2602 tavor_wrid_wqhdr_unlock_both(qp);
2613 tavor_wrid_to_reset_handling(tavor_state_t *state, tavor_qphdl_t qp)
2627 mutex_enter(&qp->qp_rq_cqhdl->cq_lock);
2628 tavor_wrid_wqhdr_lock_both(qp);
2629 tavor_wrid_reaplist_add(qp->qp_sq_cqhdl, qp->qp_sq_wqhdr);
2640 if (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
2646 mutex_enter(&qp->qp_rq_wqhdr->wq_wrid_wql->wql_lock);
2647 tavor_cq_srq_entries_flush(state, qp);
2650 tavor_wrid_wqhdr_remove(qp->qp_rq_wqhdr,
2651 qp->qp_rq_wqhdr->wq_wrid_post);
2654 if (qp->qp_rq_wqhdr->wq_wrid_post == NULL) {
2660 mutex_exit(&qp->qp_rq_wqhdr->wq_wrid_wql->wql_lock);
2664 tavor_cq_wqhdr_remove(qp->qp_rq_cqhdl, qp->qp_rq_wqhdr);
2667 tavor_wrid_reaplist_add(qp->qp_rq_cqhdl, qp->qp_rq_wqhdr);
2669 tavor_wrid_wqhdr_unlock_both(qp);
2670 mutex_exit(&qp->qp_rq_cqhdl->cq_lock);
3583 tavor_wrid_wqhdr_lock_both(tavor_qphdl_t qp)
3587 sq_cq = qp->qp_sq_cqhdl;
3588 rq_cq = qp->qp_rq_cqhdl;
3619 tavor_wrid_wqhdr_unlock_both(tavor_qphdl_t qp)
3623 sq_cq = qp->qp_sq_cqhdl;
3624 rq_cq = qp->qp_rq_cqhdl;