d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi/* TODO: remove this temporary solution for solaris / linux compilation conflict, linux needs the
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * first option, solaris the latter */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi db.header.data |= (TOE_CONNECTION_TYPE << DOORBELL_HDR_T_CONN_TYPE_SHIFT);\
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi volatile struct toe_tx_db_data *db_data = tx_con->db_data.tx;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi db_data->bds_prod += nbds; /* nbds should be written before nbytes (FW assumption) */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf((db_data->bds_prod & 0xff) == 0);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(db_data->flags & (TOE_TX_DB_DATA_FIN << TOE_TX_DB_DATA_FIN_SHIFT));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi db_data->flags |= (TOE_TX_DB_DATA_FIN << TOE_TX_DB_DATA_FIN_SHIFT);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (!(tx_con->flags & TCP_TX_DB_BLOCKED)) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi "ringing tx doorbell: cid=%d, (nbytes+=%d, nbds+=%d, fin=%d)\n",
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* define a policy for ringing the doorbell */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->db_more_bytes >= MAX_BYTES_PER_TX_DB ||
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->db_more_bds >= MAX_BDS_PER_TX_DB) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi _lm_tcp_tx_write_db(pdev, tx_con, tcp->cid, tx_con->db_more_bytes, tx_con->db_more_bds, 0);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* assert if the new addition will make the cyclic counter post_cnt smaller than comp_cnt */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(S64_SUB(tx_con->bytes_post_cnt + tx_con->db_more_bytes, tx_con->bytes_comp_cnt) < 0);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->bytes_post_cnt += tx_con->db_more_bytes;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->buffer_post_cnt += tx_con->db_more_bufs;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->db_more_bytes = tx_con->db_more_bds = tx_con->db_more_bufs = 0;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi "skipped doorbell ringing for cid=%d\n", tcp->cid);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi/** Description:
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * Post a single tcp buffer to the Tx bd chain
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * Assumptions:
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * - caller initiated tcp_buf->flags field with BUFFER_START/BUFFER_END appropriately
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * - SUCCESS - tcp buf was successfully attached to the bd chain
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * - RESOURCE - not enough available BDs on bd chain for given tcp buf
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * - CONNECTION_CLOSED - whenever connection's flag are marked as 'POST BLOCKED' */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, VERBOSEl4tx, "###lm_tcp_tx_post_buf\n");
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(!(pdev && tcp && tcp_buf && frag_list));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(tcp->cid && (tcp != lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, tcp->cid)));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(tx_con->flags & TCP_FIN_REQ_POSTED);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* check if tx con is already closed */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, WARNl4tx, "post tx buf failed, posting is blocked (cid=%d, con->flags=%x)\n",
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* check bd chain availability (including additional bd that should
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * be kept available for future fin request) */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if(lm_bd_chain_avail_bds(tx_chain) < frag_list->cnt + 1) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, INFORMl4tx, "post tx buf failed, tx chain is full (cid=%d, avail bds=%d, buf nfrags=%d)\n",
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->cid, lm_bd_chain_avail_bds(tx_chain), frag_list->cnt);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi LM_COMMON_DRV_STATS_ATOMIC_INC_TOE(pdev, tx_no_l4_bd);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* if doorbell ringing was deferred (e.g. until an end of
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * application buffer), it can no longer be deferred since
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * the place in the bd chain is now required */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi dbg_bytes_prod_seq = tx_con->db_data.tx->bytes_prod_seq + tx_con->db_more_bytes;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* "attach" the frags to the bd chain */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi for(i = 0; i < frag_list->cnt; i++, frag++) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(frag->size > 0xffff || frag->size == 0); /* hw limit: each bd can point to a buffer with max size of 64KB */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_bd = (struct toe_tx_bd *)lm_toe_bd_chain_produce_bd(tx_chain);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_bd->nextBdStartSeq = dbg_bytes_prod_seq;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* Support for FW Nagle Algorithm:
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * This bit, is to be set for every bd which is part of a tcp buffer which is equal to or larger than an mss.
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if ((u32_t)frag_list->size >= tx_con->u.tx.mss) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, VERBOSEl4tx, "Setting Tx BD, addr_lo=0x%x, addr_hi=0x%x, size=%d\n",
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_bd->addr_lo, tx_bd->addr_hi, tx_bd->size);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp_buf->bd_used = frag_list->cnt & 0xffff;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp_buf->size = tcp_buf->more_to_comp = (u32_t)frag_list->size;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(!(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_START ?
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->app_buf_bytes_acc_post += tcp_buf->size;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(S16_SUB(new_prod, old_prod) < tcp_buf->bd_used);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->db_more_bds += S16_SUB(new_prod, old_prod);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* Support for FW Nagle Algorithm:
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * This bit, is to be set for every bd which is part of a tcp buffer which is equal to or larger than an mss.
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* special care in case of last tcp buffer of an application buffer */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp_buf->app_buf_xferred = 0; /* just for safety */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp_buf->app_buf_size = tx_con->app_buf_bytes_acc_post;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* special care for the last bd: */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi "Setting Tx BD, last bd of app buf, flags=%d\n", tx_bd->flags);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* make sure there aren't 'too many' bds without completion flag */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->u.tx.bds_without_comp_flag += tcp_buf->bd_used;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (tx_con->u.tx.bds_without_comp_flag > (tx_chain->capacity - MAX_FRAG_CNT_PER_TB)) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi s_list_push_tail(&tx_con->active_tb_list, &tcp_buf->link);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi lm_tcp_tx_write_db(pdev, tcp, tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* network reachability (NOT IMPLEMENTED):
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if(lm_neigh_is_cache_entry_staled(tcp->path->neigh))
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi lm_neigh_indicate_staled_cache_entry(tcp->path->neigh);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, VERBOSEl4tx, "posted tx buf for cid=%d, buf size=%d, bd used=%d, buf flags=%x, app_buf_size=%d\n",
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->cid, tcp_buf->size, tcp_buf->bd_used, tcp_buf->flags, tcp_buf->app_buf_size);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, VERBOSEl4tx, "after posting tx buf, tx_con->active_tb_list=%d\n",
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi s_list_entry_cnt(&tx_con->active_tb_list));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi} /* lm_tcp_tx_post_buf */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi/** Description
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * indicates graceful disconnect completion to client.
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * Assumtpions:
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * tx-lock is taken by caller
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchistatic __inline void lm_tcp_tx_graceful_disconnect_complete(lm_device_t * pdev, lm_tcp_state_t * tcp)
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(!s_list_is_empty(&tcp->tx_con->active_tb_list));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(tcp->tx_con->flags & TCP_FIN_REQ_COMPLETED);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->flags |= TCP_FIN_REQ_COMPLETED;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, INFORMl4tx, "fin request completed (cid=%d)\n", tcp->cid);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tcp_state_calc.fin_completed_time = mm_get_current_time(pdev);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (!(tcp->tx_con->u.tx.flags & TCP_CON_FIN_REQ_LM_INTERNAL)) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi ip_version = (tcp->path->path_const.ip_version == IP_VERSION_IPV4)? STATS_IP_4_IDX : STATS_IP_6_IDX;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi LM_COMMON_DRV_STATS_ATOMIC_INC_TOE(pdev, ipv[ip_version].out_fin);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi mm_tcp_graceful_disconnect_done(pdev,tcp, LM_STATUS_SUCCESS);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi u32_t actual_completed; /* number of bytes actually completed (could be different than completed in case of fin) */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, VERBOSEl4tx, "##lm_tcp_tx_app_cmp_process, cid=%d, completed_bytes=%d\n",
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(tx_con->flags & TCP_TX_COMP_BLOCKED);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(S64_SUB(tx_con->bytes_post_cnt, tx_con->bytes_comp_cnt) < 0);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi actual_completed = lm_tcp_complete_nbytes(pdev, tcp, tcp->tx_con, completed_bytes, FALSE);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(actual_completed > completed_bytes);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf((completed_bytes - actual_completed) != 1);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(!(tx_con->flags & TCP_FIN_REQ_POSTED));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(tx_con->bytes_post_cnt != tx_con->bytes_comp_cnt);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* fin completed */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_FIN_CMP;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_FIN_REQ_COMPLETED */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* network reachability (NOT IMPLEMENTED):
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi lm_neigh_update_nic_reachability_time(tcp->path->neigh)
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi} /* lm_tcp_tx_app_cmp_process */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchiu8_t lm_toe_is_tx_completion(lm_device_t *pdev, u8_t drv_toe_rss_id)
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(!(pdev && ARRSIZE(pdev->toe_info.scqs) > drv_toe_rss_id));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi scq = &pdev->toe_info.scqs[drv_toe_rss_id];
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi *scq->hw_con_idx_ptr != lm_bd_chain_cons_idx(&scq->bd_chain) )
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, INFORMl4int, "lm_toe_is_tx_completion(): result is:%s\n", result? "TRUE" : "FALSE");
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, VERBOSEl4tx, "##lm_tcp_tx_inc_aborted_count, cid=%d, aborted_bytes=%d\n",
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_con->bytes_trm_aborted_cnt += aborted_bytes;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi} /* lm_tcp_tx_inc_aborted_count */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi/** Description
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * completes the fast-path operations for a certain connection
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * Assumption:
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * fp-tx lock is taken
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchivoid lm_tcp_tx_complete_tcp_fp(lm_device_t * pdev, lm_tcp_state_t * tcp, lm_tcp_con_t * con)
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /**** Client completing : may result in lock-release *****/
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* during lock-release, due to this function being called from service_deferred, more
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * cqes can be processed. We don't want to mix. This function is mutually exclusive, so
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * any processing makes it's way to being completed by calling this function.
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * the following define a "fast-path completion"
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * (i) RQ buffers to be completed
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * defined by dpc_completed_tail and are collected during lm_tcp_complete_bufs BEFORE lock
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * is released, so no more buffer processing can make it's way into this buffer completion.
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * (ii) Fin to be completed
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * determined by the flags, since dpc_flags CAN be modified during processing we copy
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * them to a snapshot_flags parameter, which is initialized in this function only, so no fin
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * can can make its way in while we release the lock.
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * (iv) Remainders for sp
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * all sp operations are logged in dpc_flags. for the same reason as (iii) no sp commands can
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * make their way in during this fp-completion, all sp-processing after will relate to this point in time.
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi con->dpc_info.snapshot_flags = con->dpc_info.dpc_flags;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* complete buffers to client */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (con->dpc_info.dpc_completed_tail != NULL) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* Graceful Disconnect */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (con->dpc_info.snapshot_flags & LM_TCP_DPC_FIN_CMP) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi con->dpc_info.snapshot_flags &= ~LM_TCP_DPC_FIN_CMP;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi lm_tcp_tx_graceful_disconnect_complete(pdev, con->tcp_state);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* get the cmd from cqe */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi cmd = ((cqe->params & TOE_TX_CQE_COMPLETION_OPCODE) >> TOE_TX_CQE_COMPLETION_OPCODE_SHIFT);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, INFORMl4tx, "###lm_tcp_tx_process_cqe cid=%d cmd=%d\n", tcp->cid, cmd);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* Check that the cqe len make sense, we could have got here by chance... */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIfAll(cqe->len & 0xc0000000); /* two upper bits on show a completion larger than 1GB - a bit odd...*/
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* Three types of completios: fast-path, reset-recv, ramrod-cmp. All completions may have a
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * fast-path part (nbytes completed) which will be handled in any case that cqe->len > 0 */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* complete data if anything needs to be complete */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi ((tcp->tx_con->dpc_info.dpc_flags & LM_TCP_DPC_RESET_RECV /* RST recv on this DPC on a previous CQE */ ) ||
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi (tcp->tx_con->flags & TCP_REMOTE_RST_RECEIVED /* RST recv on previous DPC */ )))
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* 10/28/08 - Since in exterme cases current FW may not complete all sent+acked bytes
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi on RST recv cqe and do so only later on one of the following ramrod completions,
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi we need to ignore this too late completed bytes thus we nullify cqe->len */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf((cmd != RAMROD_OPCODE_TOE_RESET_SEND) &&
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi lm_tcp_tx_inc_trm_aborted_bytes(pdev, tcp, cqe->len);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(tcp->tx_con->dpc_info.dpc_comp_blocked);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi lm_tcp_tx_cmp_process(pdev, tcp, cqe->len);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RESET_RECV;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_REMOTE_RST_RECEIVED */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(tcp->sp_request->type != SP_REQUEST_ABORTIVE_DISCONNECT);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_RST_REQ_COMPLETED */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(tcp->sp_request->type != SP_REQUEST_INVALIDATE);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_INV_REQ_COMPLETED */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(tcp->sp_request->type != SP_REQUEST_TERMINATE1_OFFLOAD);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_TRM_REQ_COMPLETED */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf((tcp->sp_request->type != SP_REQUEST_PENDING_LOCAL_FIN_DISCONNECT) &&
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi (tcp->sp_request->type != SP_REQUEST_PENDING_REMOTE_DISCONNECT) &&
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi (tcp->sp_request->type != SP_REQUEST_PENDING_TX_RST));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, FATAL, "unexpected tx cqe opcode=%d\n", cmd);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi/** Description
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * Assumptions
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * connections is initialzed with a dummy head.
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchivoid lm_tcp_tx_process_cqes(lm_device_t *pdev, u8_t drv_toe_rss_id, s_list_t * connections)
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, VERBOSEl4int , "###lm_tcp_tx_process_cqes\n");
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi scq = &pdev->toe_info.scqs[drv_toe_rss_id];
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi cq_old_idx = lm_bd_chain_cons_idx(&scq->bd_chain);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(S16_SUB(cq_new_idx, cq_old_idx) <= 0);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* save statistics */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi scq->num_cqes_last_dpc = S16_SUB(cq_new_idx, cq_old_idx);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (scq->num_cqes_last_dpc) { /* Exclude zeroed value from statistics*/
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if(scq->max_cqes_per_dpc < scq->num_cqes_last_dpc) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi scq->max_cqes_per_dpc = scq->num_cqes_last_dpc;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* we don't want to wrap around...*/
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if ((scq->sum_cqes_last_x_dpcs + scq->num_cqes_last_dpc) < scq->sum_cqes_last_x_dpcs) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi scq->sum_cqes_last_x_dpcs += scq->num_cqes_last_dpc;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi scq->avg_cqes_per_dpc = scq->sum_cqes_last_x_dpcs / avg_dpc_cnt;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(S16_SUB(cq_new_idx, cq_old_idx) <= 0);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* get next consumed cqe */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi cqe = lm_toe_bd_chain_consume_bd(&scq->bd_chain);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* get tcp state from cqe */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi cid = SW_CID(((cqe->params & TOE_TX_CQE_CID) >> TOE_TX_CQE_CID_SHIFT));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp = lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, cid);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* save cqe in history_cqes */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi hist_cqe = (struct toe_tx_cqe *)lm_tcp_qe_buffer_next_cqe_override(&tcp->tx_con->history_cqes);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi defer_cqe = ((tcp->tx_con->flags & TCP_TX_COMP_DEFERRED) == TCP_TX_COMP_DEFERRED);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* if we're deferring completions - just store the cqe and continue to the next one */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* Return if we are still deferred (may have changed since initial check was w/o a lock */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* check again under lock if we're deferred */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi defer_cqe = ((tcp->tx_con->flags & TCP_TX_COMP_DEFERRED) == TCP_TX_COMP_DEFERRED);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp->tx_con->flags |= TCP_DEFERRED_PROCESSING;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* connections will always be initialized to a dummy, so once a tcp connection is added to the
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * list, it's link will be initialized to point to another link other than NULL */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (s_list_next_entry(&tcp->tx_con->dpc_info.link) == NULL) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi s_list_push_head(connections, &tcp->tx_con->dpc_info.link);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi cq_old_idx = lm_bd_chain_cons_idx(&scq->bd_chain);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* GilR 5/12/2006 - TODO - decide with Alon if reading the hw_con again is required */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi //cq_new_idx = *(scq->hw_con_idx_ptr);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* The fact that we post the producer here before we've handled any slow-path completions assures that
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * the sp-ring will always be updated AFTER the producer was. */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi lm_toe_bd_chain_bds_produced(&scq->bd_chain, num_to_reproduce);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* GilR 5/13/2006 - TBA - save some stats? */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* notify the fw of the prod of the SCQ */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi LM_INTMEM_WRITE16(pdev, CSTORM_TOE_CQ_PROD_OFFSET(LM_TOE_FW_RSS_ID(pdev,drv_toe_rss_id) , PORT_ID(pdev)),
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi lm_bd_chain_prod_idx(&scq->bd_chain), BAR_CSTRORM_INTMEM);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchivoid lm_toe_service_tx_intr(lm_device_t *pdev, u8_t drv_toe_rss_id)
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, VERBOSEl4int , "###lm_toe_service_tx_intr\n");
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(!(pdev && ARRSIZE(pdev->toe_info.scqs) > drv_toe_rss_id));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* process the cqes and initialize connections with all the connections that appeared
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * in the DPC */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi lm_tcp_tx_process_cqes(pdev,drv_toe_rss_id,&connections);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* complete the fp/sp parts of the connections remember to ignore the last one */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi con = (lm_tcp_con_t *)s_list_peek_head(&connections);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi while (s_list_next_entry(&con->dpc_info.link) != NULL) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi lm_tcp_tx_complete_tcp_fp(pdev, con->tcp_state, con);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi con = (lm_tcp_con_t *)s_list_next_entry(&con->dpc_info.link);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* SP : traverse the connections. remember to ignore the last one */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi con = (lm_tcp_con_t *)s_list_pop_head(&connections);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi s_list_next_entry(&con->dpc_info.link) = NULL;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi while (s_list_entry_cnt(&connections) > 0) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* we access snapshot and not dpc, since once the dpc_flags were copied
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi * to snapshot they were zeroized */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi con = (lm_tcp_con_t *)s_list_pop_head(&connections);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi s_list_next_entry(&con->dpc_info.link) = NULL;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgMessage(pdev, INFORMl4tx, "###lm_tcp_graceful_disconnect\n");
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if ( tcp_con->flags & TCP_TX_POST_BLOCKED ) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf( (tcp_con->app_buf_bytes_acc_post != 0) ||
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi old_prod = lm_bd_chain_prod_idx(&(tcp_con->bd_chain));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* Post FIN BD on Tx chain */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_bd = (struct toe_tx_bd *)lm_toe_bd_chain_produce_bd(&(tcp_con->bd_chain));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_bd->flags = TOE_TX_BD_FIN; /* Vladz: Pay attention when u move this
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi line - there is an assignment to flags, NOT bitwise OR */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* For a safety */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi dbg_bytes_prod_seq = tcp_con->db_data.tx->bytes_prod_seq + tcp_con->db_more_bytes;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tx_bd->nextBdStartSeq = dbg_bytes_prod_seq;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi new_prod = lm_bd_chain_prod_idx(&(tcp_con->bd_chain));
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(S16_SUB(new_prod, old_prod) >= 3);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(S16_SUB(new_prod, old_prod) <= 0);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi DbgBreakIf(tcp_con->flags & TCP_FIN_REQ_POSTED);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* Update fin request time, if not already set by the caller */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (!tcp_state->tcp_state_calc.fin_request_time) {
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp_state->tcp_state_calc.fin_request_time = mm_get_current_time(pdev);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi if (tcp_state->tcp_state_calc.fin_request_time == tcp_state->tcp_state_calc.fin_reception_time){
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi tcp_state->tcp_state_calc.fin_reception_time -= 1;
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* Doorbell FIN */
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi _lm_tcp_tx_write_db(pdev, tcp_con, tcp_state->cid, 0, (u16_t)S16_SUB(new_prod, old_prod), 1);
d14abf155341d55053c76eeec58b787a456b753bRobert Mustacchi /* assert if the new addition will make the cyclic counter post_cnt smaller than comp_cnt */