30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * CDDL HEADER START
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * The contents of this file are subject to the terms of the
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Common Development and Distribution License (the "License").
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * You may not use this file except in compliance with the License.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * See the License for the specific language governing permissions
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * and limitations under the License.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * When distributing Covered Code, include this CDDL HEADER in each
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * If applicable, add the following below this CDDL HEADER, with the
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * fields enclosed by brackets "[]" replaced with your own identifying
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * information: Portions Copyright [yyyy] [name of copyright owner]
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * CDDL HEADER END
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Use is subject to license terms.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Routines for completion queue handlers for iSER.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapstatic void iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapint iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapstatic int iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapstatic int iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan * Poll for work request completion while successful. If the
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan * queue empties or otherwise becomes invalid, stop polling.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap status = iser_ib_poll_send_completions(cq_hdl, iser_chan);
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan /* We've emptied the CQ, rearm it before we're done here */
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION);
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan /* Unexpected error */
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan /* Now, check for more completions after the rearm */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan /* Poll ISER_IB_SCQ_POLL_MAX completions from the CQ */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap status = ibt_poll_cq(cq_hdl, wc, ISER_IB_SCQ_POLL_MAX, &npoll);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Unexpected error */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: ibt_poll_cq "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* CQ is empty. Either way, move along... */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Handle each of the completions we've polled
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap for (i = 0; i < npoll; i++) {
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap DTRACE_PROBE3(iser__send__cqe, iser_chan_t *, iser_chan,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ibt_wc_t *, &wc[i], ibt_wc_status_t, wc[i].wc_status);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Grab the wrid of the completion */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Decrement this channel's SQ posted count */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Pull in the wr handle */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Set an idm_status for return to IDM */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap idm_status = (wc[i].wc_status == IBT_WC_SUCCESS) ?
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * A non-success status here indicates the QP went
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * into an error state while this WR was being
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * processed. This can also happen when the
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * channel is closed on the remote end. Clean up
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * the resources, then push CE_TRANSPORT_FAIL
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Free the resources attached to this
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * completion.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Free iser_msg handle */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Complete the PDU */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Invoke buffer callback */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap } else { /* ISER_WR_RDMAR */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Free the iser wr handle */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Tell IDM that the channel has gone down,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * unless he already knows.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Move onto the next completion */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * For a success status, just invoke the PDU or
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * buffer completion. We use our WR handle's
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * "iw_type" here so that we can properly process
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * because the CQE's opcode is invalid if the status
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * is failed.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Free the msg handle */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* This is a hello exchange message */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * We're on the target side,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * and have just successfully
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * sent the HelloReply msg.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* This is a normal control message */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Free the wr handle */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Invoke the appropriate callback;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * the buffer will be freed there.
a668b114487acbb725f522170849c39f8e844673Priya Krishnan mr = ((iser_buf_t *)idb->idb_buf_private)->iser_mr;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap idm_buf_rx_from_ini_done(idt, idb, idm_status);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Free the wr handle */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan * Poll for work request completion while successful. If the
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan * queue empties or otherwise becomes invalid, stop polling.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap status = iser_ib_poll_recv_completions(cq_hdl, iser_chan);
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan /* We've emptied the CQ, rearm it before we're done here */
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION);
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan /* Unexpected error */
a82ec3cf2b07d0f2ebb2e60f41370b7c39a5e71ePriya Krishnan /* Now, check for more completions after the rearm */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* CQ is empty, return */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Unexpected error */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Free the msg handle (if we got it back) */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap if ((msg = (iser_msg_t *)(uintptr_t)wc.wc_id) != NULL) {
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Retrieve the iSER msg handle */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Decrement the posted level in the RQ, then check
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * to see if we need to fill the RQ back up (or if
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * we are already on the taskq).
72cf314316bed51cd2e5fd0cb021a9725316a6b0peter dunlap (iser_chan->ic_conn->ic_stage >= ISER_CONN_STAGE_IC_CONNECTED) &&
72cf314316bed51cd2e5fd0cb021a9725316a6b0peter dunlap (iser_chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN)) {
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Set the pending flag and fire off a post_recv */
cf8c0ebaf84c824d8f14486e47457119c138ce3cPeter Dunlap status = iser_ib_post_recv_async(iser_chan->ic_chanhdl);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "task dispatch failed");
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Failure to launch, unset the pending flag */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap DTRACE_PROBE3(iser__recv__cqe, iser_chan_t *, iser_chan,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ibt_wc_t *, &wc, ibt_wc_status_t, wc.wc_status);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Tell IDM that the channel has gone down,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * unless he already knows.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * We have an iSER message in, let's handle it.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * We will free the iser_msg_t later in this path,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * depending upon the action required.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_msg_handle(iser_chan_t *chan, iser_msg_t *msg)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap hdr = (iser_ctrl_hdr_t *)(uintptr_t)msg->msg_ds.ds_va;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Handle an iSCSI Control PDU iSER message.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Note we'll free the msg handle in the PDU callback.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * We are unable to handle this message, and
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * have no way to recover from this. Fail the
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * transport.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "iser_iscsihdr_handle");
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap } else if (opcode == ISER_OPCODE_HELLO_MSG) { /* at the target */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * We are currently not supporting Hello Exchange,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * since OFED iSER does not. May be revisited.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap if (iser_conn->ic_type != ISER_CONN_TYPE_TGT) {
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_hello_hdr_t *hello_hdr = (iser_hello_hdr_t *)hdr;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "received Hello message: opcode[%d], "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "maxver[%d], minver[%d], iser_ird[%d], msg (0x%p)",
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap hello_hdr->opcode, hello_hdr->maxver, hello_hdr->minver,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap if (iser_conn->ic_stage != ISER_CONN_STAGE_HELLO_WAIT) {
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* target is not expected to receive a Hello */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_SENT;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Prepare and send a HelloReply message */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Free this msg handle */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap } else if (opcode == ISER_OPCODE_HELLOREPLY_MSG) { /* at initiator */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * We are currently not supporting Hello Exchange,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * since OFED iSER does not. May be revisited.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap if (iser_conn->ic_type != ISER_CONN_TYPE_INI) {
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_helloreply_hdr_t *hello_hdr = (iser_helloreply_hdr_t *)hdr;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "received Hello Reply message: opcode[%d], "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "maxver[%d], curver[%d], iser_ord[%d], msg (0x%p)",
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap hello_hdr->opcode, hello_hdr->maxver, hello_hdr->curver,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Free this msg handle */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Signal the receipt of HelloReply to the waiting thread
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * so that the initiator can proceed to the Full Feature
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_RCV;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Protocol error: free the msg handle and fail the session */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_msg_handle: unsupported opcode (0x%x): "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "terminating session on IDM handle (0x%p)", opcode,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap idm_conn_event(iser_conn->ic_idmc, CE_TRANSPORT_FAIL,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap/* network to host translation for 24b integers */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap return ((ptr[0] << 16) | (ptr[1] << 8) | ptr[2]);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap/* ARGSUSED */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_rx_pdu_cb(idm_pdu_t *pdu, idm_status_t status)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Free the iser msg handle and the PDU handle */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_msg_free((iser_msg_t *)pdu->isp_transport_private);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap pdu = idm_pdu_alloc_nosleep(sizeof (iscsi_hdr_t), 0);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Set the iser_msg handle into the transport-private field */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Set up a pointer in the pdu handle to the iSER header */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_hdrp = (uint8_t *)(uintptr_t)msg->msg_ds.ds_va;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iser_hdrp is NULL");
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap pdu->isp_transport_hdrlen = ISER_HEADER_LENGTH;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Set up a pointer to the iSCSI header, which is directly
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * after the iSER header in the message.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iscsi_hdrp = ((uint8_t *)(uintptr_t)msg->msg_ds.ds_va) +
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iscsi_hdrp is NULL");
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap pdu->isp_hdr = (iscsi_hdr_t *)(uintptr_t)iscsi_hdrp;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Fill in the BHS */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * If datalen > 0, then non-scsi data may be present. Allocate
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * space in the PDU handle and set a pointer to the data.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap pdu->isp_data = ((uint8_t *)(uintptr_t)pdu->isp_hdr) +
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Process RX PDU */