7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * CDDL HEADER START
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * The contents of this file are subject to the terms of the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Common Development and Distribution License (the "License").
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * You may not use this file except in compliance with the License.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * or http://www.opensolaris.org/os/licensing.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * See the License for the specific language governing permissions
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * and limitations under the License.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * When distributing Covered Code, include this CDDL HEADER in each
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * If applicable, add the following below this CDDL HEADER, with the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * fields enclosed by brackets "[]" replaced with your own identifying
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * information: Portions Copyright [yyyy] [name of copyright owner]
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * CDDL HEADER END
8e51227711fb29b69b8f42a3953e759963432065WENTAO YANG * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * This file contains the implementation of RxDringData transfer mode of VIO
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Protocol in vnet. The functions in this file are invoked from vnet_gen.c
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * after RxDringData mode is negotiated with the peer during attribute phase of
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * handshake. This file contains functions that setup the transmit and receive
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * descriptor rings, and associated resources in RxDringData mode. It also
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * contains the transmit and receive data processing functions that are invoked
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * in RxDringData mode. The data processing routines in this file have the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * suffix '_shm' to indicate the shared memory mechanism used in RxDringData
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna/* Functions exported to vnet_gen.c */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaint vgen_create_rx_dring(vgen_ldc_t *ldcp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavoid vgen_destroy_rx_dring(vgen_ldc_t *ldcp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaint vgen_map_tx_dring(vgen_ldc_t *ldcp, void *pkt);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavoid vgen_unmap_tx_dring(vgen_ldc_t *ldcp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaint vgen_map_data(vgen_ldc_t *ldcp, void *pkt);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaint vgen_dringsend_shm(void *arg, mblk_t *mp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaint vgen_handle_dringdata_shm(void *arg1, void *arg2);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnamblk_t *vgen_poll_rcv_shm(vgen_ldc_t *ldcp, int bytes_to_pickup);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaint vgen_send_dringack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna uint32_t start, int32_t end, uint8_t pstate);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna/* Internal functions */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnastatic int vgen_handle_dringdata_info_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnastatic int vgen_handle_dringdata_ack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnastatic int vgen_handle_dringdata_nack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnastatic int vgen_intr_rcv_shm(vgen_ldc_t *ldcp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnastatic int vgen_receive_packet(vgen_ldc_t *ldcp, mblk_t **bp, uint_t *size);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnastatic int vgen_send_dringdata_shm(vgen_ldc_t *ldcp, uint32_t start,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnastatic int vgen_sendmsg_shm(vgen_ldc_t *ldcp, caddr_t msg, size_t msglen);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna/* Functions imported from vnet_gen.c */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaextern int vgen_handle_evt_read(vgen_ldc_t *ldcp, vgen_caller_t caller);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaextern int vgen_handle_evt_reset(vgen_ldc_t *ldcp, vgen_caller_t caller);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaextern void vgen_handle_pkt_data(void *arg1, void *arg2, uint32_t msglen);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaextern void vgen_destroy_rxpools(void *arg);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaextern void vgen_debug_printf(const char *fname, vgen_t *vgenp,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vgen_ldc_t *ldcp, const char *fmt, ...);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnaextern boolean_t vgen_inject_error(vgen_ldc_t *ldcp, int error);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Allocate receive resources for the channel. The resources consist of a
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * receive descriptor ring and an associated receive buffer area.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rxdsize = sizeof (vnet_rx_dringdata_desc_t);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->num_rxds = vnet_num_descriptors;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Create the receive descriptor ring */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = ldc_mem_dring_create(ldcp->num_rxds, rxdsize,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "ldc_mem_dring_create() failed\n");
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Get the addr of descriptor ring */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = ldc_mem_dring_info(ldcp->rx_dring_handle, &minfo);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "ldc_mem_dring_info() failed\n");
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->rxdp = (vnet_rx_dringdata_desc_t *)(minfo.vaddr);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna bzero(ldcp->rxdp, sizeof (*rxdp) * (ldcp->num_rxds));
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Allocate a table that maps descriptor to its associated buffer;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * used while receiving to validate that the peer has not changed the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * buffer offset provided in the descriptor.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rxdp_to_vmp = kmem_zalloc(ldcp->num_rxds * sizeof (uintptr_t),
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Allocate a single large buffer that serves as the rx buffer area.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * We allocate a ldc memory handle and export the buffer area as shared
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * memory. We send the ldc memcookie for this buffer space to the peer,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * as part of dring registration phase during handshake. We manage this
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * buffer area as individual buffers of max_frame_size and provide
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * specific buffer offsets in each descriptor to the peer. Note that
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * the factor used to compute the # of buffers (above) must be > 1 to
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * ensure that there are more buffers than the # of descriptors. This
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * is needed because, while the shared memory buffers are sent up our
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * stack during receive, the sender needs additional buffers that can
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * be used for further transmits. This also means there is no one to
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * one correspondence between the descriptor index and buffer offset.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * The sender has to read the buffer offset in the descriptor and use
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * the specified offset to copy the tx data into the shared buffer. We
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * (receiver) manage the individual buffers and their state (see
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * VIO_MBLK_STATEs in vio_util.h).
34f94fbc7a730740933e4776ade5f74009afe4ceWENTAO YANG data_sz = RXDRING_DBLK_SZ(vgenp->max_frame_size);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->rx_data_sz = data_sz * ldcp->num_rbufs;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna datap = kmem_zalloc(ldcp->rx_data_sz, KM_SLEEP);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Allocate a ldc memhandle for the entire rx data area */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = ldc_mem_alloc_handle(ldcp->ldc_handle, &ldcp->rx_data_handle);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Allocate memory for the data cookies */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->rx_data_cookie = kmem_zalloc(VNET_DATA_AREA_COOKIES *
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Bind ldc memhandle to the corresponding rx data area.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = ldc_mem_bind_handle(ldcp->rx_data_handle, (caddr_t)datap,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->rx_data_sz, LDC_DIRECT_MAP, LDC_MEM_W,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if ((ncookies == 0) || (ncookies > VNET_DATA_AREA_COOKIES)) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Successful in binding the handle to rx data area. Now setup mblks
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * around each data buffer and setup the descriptors to point to these
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * rx data buffers. We associate each descriptor with a buffer
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * by specifying the buffer offset in the descriptor. When the peer
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * needs to transmit data, this offset is read by the peer to determine
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * the buffer in the mapped buffer area where the data to be
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * transmitted should be copied, for a specific descriptor.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vio_create_mblks(ldcp->num_rbufs, data_sz, (uint8_t *)datap,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna for (i = 0; i < ldcp->num_rxds; i++) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* allocate an mblk around this data buffer */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rxdp->data_buf_offset = VIO_MBLK_DATA_OFF(vmp) + VNET_IPALIGN;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * The descriptors and the associated buffers are all ready;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * now bind descriptor ring to the channel.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = ldc_mem_dring_bind(ldcp->ldc_handle, ldcp->rx_dring_handle,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna LDC_DIRECT_MAP | LDC_SHADOW_MAP, LDC_MEM_RW,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "ldc_mem_dring_bind failed "
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* initialize rx seqnum and index */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Free receive resources for the channel.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_destroy_rx_dring(vgen_ldc_t *ldcp)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* We first unbind the descriptor ring */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (void) ldc_mem_dring_unbind(ldcp->rx_dring_handle);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Destroy the mblks that are wrapped around the rx data buffers */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (vio_destroy_mblks(ldcp->rx_vmp) != 0) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * If we can't destroy the rx pool for this channel,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * dispatch a task to retry and clean up. Note that we
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * don't need to wait for the task to complete. If the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * vnet device itself gets detached, it will wait for
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * the task to complete implicitly in
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * ddi_taskq_destroy().
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (void) ddi_taskq_dispatch(vgenp->rxp_taskq,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vgen_destroy_rxpools, ldcp->rx_vmp, DDI_SLEEP);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Free rx data area cookies */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna kmem_free(ldcp->rx_data_cookie, VNET_DATA_AREA_COOKIES *
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Unbind rx data area memhandle */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (void) ldc_mem_unbind_handle(ldcp->rx_data_handle);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Free rx data area memhandle */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (void) ldc_mem_free_handle(ldcp->rx_data_handle);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Now free the rx data area itself */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* prealloc'd rx data buffer */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna kmem_free(ldcp->rx_datap, ldcp->rx_data_sz);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Finally, free the receive descriptor ring */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (void) ldc_mem_dring_destroy(ldcp->rx_dring_handle);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Reset rx index and seqnum */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Map the receive descriptor ring exported
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * by the peer, as our transmit descriptor ring.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_map_tx_dring(vgen_ldc_t *ldcp, void *pkt)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Sanity check.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (num_desc < VGEN_NUM_DESCRIPTORS_MIN ||
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna desc_size < sizeof (vnet_rx_dringdata_desc_t) ||
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna bcopy(&msg->cookie[0], &dcookie, sizeof (ldc_mem_cookie_t));
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Map the remote dring */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = ldc_mem_dring_map(ldcp->ldc_handle, &dcookie, ncookies, num_desc,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna desc_size, LDC_DIRECT_MAP, &(ldcp->tx_dring_handle));
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Sucessfully mapped; now try to get info about the mapped dring
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = ldc_mem_dring_info(ldcp->tx_dring_handle, &minfo);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Save ring address, number of descriptors.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->mtxdp = (vnet_rx_dringdata_desc_t *)(minfo.vaddr);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna bcopy(&dcookie, &(ldcp->tx_dring_cookie), sizeof (dcookie));
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Initialize tx dring indexes and seqnum */
0e26330710421d79f1fbb73c4f5f75086785b207WENTAO YANG ldcp->next_txi = ldcp->cur_txi = ldcp->resched_peer_txi = 0;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Save peer's dring_info values */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna bcopy(&dcookie, &(ldcp->peer_hparams.dring_cookie),
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->peer_hparams.num_desc = num_desc;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->peer_hparams.desc_size = desc_size;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->peer_hparams.dring_ncookies = ncookies;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Set dring_ident for the peer */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->peer_hparams.dring_ident = (uint64_t)ldcp->mtxdp;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Return the dring_ident in ack msg */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna msg->dring_ident = (uint64_t)ldcp->mtxdp;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Mark the descriptor state as 'done'. This is implementation specific
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * and not required by the protocol. In our implementation, we only
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * need the descripor to be in 'done' state to be used by the transmit
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * function and the peer is not aware of it. As the protocol requires
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * that during initial registration the exporting end point mark the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * dstate as 'free', we change it 'done' here. After this, the dstate
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * in our implementation will keep moving between 'ready', set by our
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * transmit function; and and 'done', set by the peer (per protocol)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * after receiving data.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Setup on_trap() protection before accessing dring shared memory area.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Data access fault occured down the code path below while
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * accessing the descriptors. Return failure.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna for (i = 0; i < num_desc; i++) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (void) ldc_mem_dring_unmap(ldcp->tx_dring_handle);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Unmap the transmit descriptor ring.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Unmap mapped tx data area */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (void) ldc_mem_unmap(ldcp->tx_data_handle);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Free tx data area handle */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (void) ldc_mem_free_handle(ldcp->tx_data_handle);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Free tx data area cookies */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna kmem_free(ldcp->tx_data_cookie, ldcp->tx_data_ncookies *
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Unmap peer's dring */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (void) ldc_mem_dring_unmap(ldcp->tx_dring_handle);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* clobber tx ring members */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna bzero(&ldcp->tx_dring_cookie, sizeof (ldcp->tx_dring_cookie));
0e26330710421d79f1fbb73c4f5f75086785b207WENTAO YANG ldcp->next_txi = ldcp->cur_txi = ldcp->resched_peer_txi = 0;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Map the shared memory data buffer area exported by the peer.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_map_data(vgen_ldc_t *ldcp, void *pkt)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vio_dring_reg_msg_t *msg = (vio_dring_reg_msg_t *)pkt;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna uint8_t *buf = (uint8_t *)msg->cookie;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* skip over dring cookies */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna buf += (msg->ncookies * sizeof (ldc_mem_cookie_t));
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna emsg = (vio_dring_reg_ext_msg_t *)buf;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (emsg->data_ncookies > VNET_DATA_AREA_COOKIES) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* save # of data area cookies */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->tx_data_ncookies = emsg->data_ncookies;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* save data area size */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->tx_data_sz = emsg->data_area_size;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* allocate ldc mem handle for data area */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = ldc_mem_alloc_handle(ldcp->ldc_handle, &ldcp->tx_data_handle);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "ldc_mem_alloc_handle() failed: %d\n", rv);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* map the data area */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = ldc_mem_map(ldcp->tx_data_handle, emsg->data_cookie,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna emsg->data_ncookies, LDC_DIRECT_MAP, LDC_MEM_W,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "ldc_mem_map() failed: %d\n", rv);
34f94fbc7a730740933e4776ade5f74009afe4ceWENTAO YANG /* get the map info */
34f94fbc7a730740933e4776ade5f74009afe4ceWENTAO YANG rv = ldc_mem_info(ldcp->tx_data_handle, &minfo);
34f94fbc7a730740933e4776ade5f74009afe4ceWENTAO YANG DWARN(vgenp, ldcp, "ldc_mem_info() failed: %d\n", rv);
34f94fbc7a730740933e4776ade5f74009afe4ceWENTAO YANG DWARN(vgenp, ldcp, "mtype(%d) is not direct map\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* allocate memory for data area cookies */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->tx_data_cookie = kmem_zalloc(emsg->data_ncookies *
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* save data area cookies */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna bcopy(emsg->data_cookie, ldcp->tx_data_cookie,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna emsg->data_ncookies * sizeof (ldc_mem_cookie_t));
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * This function transmits normal data frames (non-priority) over the channel.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * It queues the frame into the transmit descriptor ring and sends a
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * VIO_DRING_DATA message if needed, to wake up the peer to (re)start
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_dringsend_shm(void *arg, mblk_t *mp)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vgen_hparams_t *lp = &ldcp->local_hparams;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Drop the packet if ldc is not up or handshake is not done */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DBG2(vgenp, ldcp, "status(%d), dropping packet\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "hphase(%x), dropping packet\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "invalid size(%d)\n", size);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ehp = (struct ether_header *)mp->b_rptr;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Setup on_trap() protection before accessing shared memory areas
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * (descriptor and data buffer). Note that we enable this protection a
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * little early and turn it off slightly later, than keeping it enabled
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * strictly at the points in code below where the descriptor and data
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * buffer are accessed. This is done for performance reasons:
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * (a) to avoid calling the trap protection code while holding mutex.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * (b) to avoid multiple on/off steps for descriptor and data accesses.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Data access fault occured down the code path below while
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * accessing either the descriptor or the data buffer. Release
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * any locks that we might have acquired in the code below and
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * return failure.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DERR(vgenp, ldcp, "data access fault occured\n");
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Allocate a descriptor
0e26330710421d79f1fbb73c4f5f75086785b207WENTAO YANG if (txdp->dstate != VIO_DESC_DONE) { /* out of descriptors */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->tx_blocked_lbolt = ddi_get_lbolt();
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Update descriptor ring index */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Ensure load ordering of dstate (above) and data_buf_offset. */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Get the offset of the buffer to be used */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Access the buffer using the offset */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna dst = (caddr_t)ldcp->tx_datap + buf_offset;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Copy data into mapped transmit buffer */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna for (bp = mp; bp != NULL; bp = bp->b_cont) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Set the size of data in the descriptor */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Ensure store ordering of nbytes and dstate (below); so that the peer
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * sees the right nbytes value after it checks that the dstate is READY.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Mark the descriptor ready */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Check if peer needs wake up (handled below) */
0e26330710421d79f1fbb73c4f5f75086785b207WENTAO YANG if (ldcp->resched_peer == B_TRUE && ldcp->resched_peer_txi == txi) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Update tx stats */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * We are done accessing shared memory; clear trap protection.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Need to wake up the peer ?
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vgen_send_dringdata_shm(ldcp, (uint32_t)txi, -1);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* error: drop the packet */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "failed sending dringdata msg "
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (rv == ECONNRESET || rv == EACCES) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (void) vgen_handle_evt_reset(ldcp, VGEN_OTHER);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Process dring data messages (info/ack/nack)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_handle_dringdata_shm(void *arg1, void *arg2)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vgen_ldc_t *ldcp = (vgen_ldc_t *)arg1;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vio_msg_tag_t *tagp = (vio_msg_tag_t *)arg2;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * To reduce the locking contention, release the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * cblock here and re-acquire it once we are done
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * receiving packets.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vgen_handle_dringdata_info_shm(ldcp, tagp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "handle_data_info failed(%d)\n", rv);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vgen_handle_dringdata_ack_shm(ldcp, tagp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "handle_data_ack failed(%d)\n", rv);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vgen_handle_dringdata_nack_shm(ldcp, tagp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "handle_data_nack failed(%d)\n", rv);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_handle_dringdata_info_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vio_dring_msg_t *dringmsg = (vio_dring_msg_t *)tagp;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DBG1(vgenp, ldcp, "INFO: start(%d), end(%d)\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ((end != -1) && !(CHECK_RXI(end, ldcp)))) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "Invalid Rx start(%d) or end(%d)\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* drop the message if invalid index */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* validate dring_ident */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (dringmsg->dring_ident != ldcp->peer_hparams.dring_ident) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "Invalid dring ident 0x%x\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* invalid dring_ident, drop the msg */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * If we are in polling mode, return from here without processing the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * dring. We will process the dring in the context of polling thread.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Process the dring and receive packets in intr context.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "vgen_intr_rcv_shm() failed\n");
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Process the rx descriptor ring in the context of interrupt thread
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * (vgen_ldc_cb() callback) and send the received packets up the stack.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vio_net_rx_cb_t vrx_cb = ldcp->portp->vcb.vio_net_rx_cb;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vgen_receive_packet(ldcp, &mp, &mblk_sz);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Invalid descriptor error; get next */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DTRACE_PROBE1(vgen_intr_nopkts, vgen_ldc_t *, ldcp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Build a chain of received packets */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* first pkt */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * We are receiving the packets in interrupt context. If we
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * have gathered vgen_chain_len (tunable) # of packets in the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * chain, send them up. (See vgen_poll_rcv_shm() for receiving
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * in polling thread context).
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DTRACE_PROBE2(vgen_intr_pkts, vgen_ldc_t *, ldcp,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Stop further processing if we processed the entire dring
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * once; otherwise continue.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna } while (total_count < ldcp->num_rxds);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DTRACE_PROBE2(vgen_intr_pkts, vgen_ldc_t *, ldcp, int, count);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * We send a stopped message to peer (sender) while we are in
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * intr mode only; allowing the peer to send further data intrs
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * (dring data msgs) to us.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna msg.dring_ident = ldcp->peer_hparams.dring_ident;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vgen_send_dringack_shm(ldcp, (vio_msg_tag_t *)&msg,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna VNET_START_IDX_UNSPEC, end_ix, VIO_DP_STOPPED);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Process the rx descriptor ring in the context of mac polling thread. Receive
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * packets upto the limit specified by bytes_to_pickup or until there are no
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * more packets, whichever occurs first. Return the chain of received packets.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_poll_rcv_shm(vgen_ldc_t *ldcp, int bytes_to_pickup)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Channel is being reset and handshake not complete */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vgen_receive_packet(ldcp, &mp, &mblk_sz);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Invalid descriptor error; get next */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DTRACE_PROBE1(vgen_poll_nopkts, vgen_ldc_t *, ldcp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Build a chain of received packets */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* first pkt */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Compute total size accumulated */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Reached the bytes limit; we are done. */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * We prepend any high priority packets to the chain of packets; note
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * that if we are already at the bytes_to_pickup limit, we might
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * slightly exceed that in such cases. That should be ok, as these pkts
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * are expected to be small in size and arrive at an interval in the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * the order of a few seconds.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (ldcp->rx_pktdata == vgen_handle_pkt_data &&
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ldcp->rx_pri_head = ldcp->rx_pri_tail = NULL;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DTRACE_PROBE2(vgen_poll_pkts, vgen_ldc_t *, ldcp, int, count);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DTRACE_PROBE2(vgen_poll_bytes, vgen_ldc_t *, ldcp, uint_t, sz);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Process the next index in the rx dring and receive the associated packet.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * bp: Success: The received packet.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Failure: NULL
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * size: Success: Size of received packet.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Failure: EAGAIN: Descriptor not ready
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * EIO: Descriptor contents invalid.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_receive_packet(vgen_ldc_t *ldcp, mblk_t **bp, uint_t *size)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vgen_hparams_t *lp = &ldcp->local_hparams;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Descriptor is not ready.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DTRACE_PROBE1(vgen_noready_rxds, vgen_ldc_t *, ldcp);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Ensure load ordering of dstate and nbytes.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna (VIO_MBLK_DATA_OFF(vmp) + VNET_IPALIGN))) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Descriptor contents invalid.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Now allocate a new buffer for this descriptor before sending up the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * buffer being processed. If that fails, stop processing; as we are
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * out of receive buffers.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Process the current buffer being received.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * We failed to get a new mapped buffer that is needed to
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * refill the descriptor. In that case, leave the current
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * buffer bound to the descriptor; allocate an mblk dynamically
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * and copy the contents of the buffer to the mblk. Then send
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * up this mblk. This way the sender has the same buffer as
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * before that can be used to send new data.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna dmp = allocb(nbytes + VNET_IPALIGN, BPRI_MED);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Mark the status of the current rbuf */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Set the offset of the new buffer in the descriptor */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna VIO_MBLK_DATA_OFF(new_vmp) + VNET_IPALIGN;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Ensure store ordering of data_buf_offset and dstate; so that the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * peer sees the right data_buf_offset after it checks that the dstate
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Now mark the descriptor 'done' */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Update stats */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna ehp = (struct ether_header *)mp->b_rptr;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Update the next index to be processed */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Save the new recv index */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Return the packet received */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_handle_dringdata_ack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vio_dring_msg_t *dringmsg = (vio_dring_msg_t *)tagp;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Received an ack for our transmits upto a certain dring index. This
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * enables us to reclaim descriptors. We also send a new dring data msg
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * to the peer to restart processing if there are pending transmit pkts.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DBG2(vgenp, ldcp, "ACK: start(%d), end(%d)\n", start, end);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * In RxDringData mode (v1.6), start index of -1 can be used by the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * peer to indicate that it is unspecified. However, the end index
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * must be set correctly indicating the last descriptor index processed.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (((start != VNET_START_IDX_UNSPEC) && !(CHECK_TXI(start, ldcp))) ||
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* drop the message if invalid index */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "Invalid Tx ack start(%d) or end(%d)\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Validate dring_ident */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (dringmsg->dring_ident != ldcp->local_hparams.dring_ident) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* invalid dring_ident, drop the msg */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "Invalid dring ident 0x%x\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Clear transmit flow control condition
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * as some descriptors should be free now.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (dringmsg->dring_process_state != VIO_DP_STOPPED) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Receiver continued processing
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * dring after sending us the ack.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Receiver stopped processing descriptors.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Setup on_trap() protection before accessing dring shared memory area.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Data access fault occured down the code path below while
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * accessing the descriptors. Release any locks that we might
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * have acquired in the code below and return failure.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Determine if there are any pending tx descriptors ready to be
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * processed by the receiver(peer) and if so, send a message to the
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * peer to restart receiving.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * We are done accessing shared memory; clear trap protection.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * No ready tx descriptors. Set the flag to send a message to
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * the peer when tx descriptors are ready in transmit routine.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * We have some tx descriptors ready to be processed by the receiver.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Send a dring data message to the peer to restart processing.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vgen_send_dringdata_shm(ldcp, txi, -1);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_handle_dringdata_nack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vio_dring_msg_t *dringmsg = (vio_dring_msg_t *)tagp;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Peer sent a NACK msg (to indicate bad descriptors ?). The start and
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * end correspond to the range of descriptors which are being nack'd.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "NACK: start(%d), end(%d)\n", start, end);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * In RxDringData mode (v1.6), start index of -1 can be used by
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * the peer to indicate that it is unspecified. However, the end index
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * must be set correctly indicating the last descriptor index processed.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (((start != VNET_START_IDX_UNSPEC) && !(CHECK_TXI(start, ldcp))) ||
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* drop the message if invalid index */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "Invalid Tx nack start(%d) or end(%d)\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* Validate dring_ident */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (dringmsg->dring_ident != ldcp->local_hparams.dring_ident) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* invalid dring_ident, drop the msg */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "Invalid dring ident 0x%x\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Setup on_trap() protection before accessing dring shared memory area.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Data access fault occured down the code path below while
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * accessing the descriptors. Release any locks that we might
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * have acquired in the code below and return failure.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna /* We just mark the descrs as free so they can be reused */
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * We are done accessing shared memory; clear trap protection.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Send descriptor ring data message to the peer over LDC.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_send_dringdata_shm(vgen_ldc_t *ldcp, uint32_t start, int32_t end)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vio_dring_msg_t dringmsg, *msgp = &dringmsg;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna if (vgen_inject_error(ldcp, VGEN_ERR_TXTIMEOUT)) {
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna tagp->vio_subtype_env = VIO_DRING_DATA;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna msgp->dring_ident = ldcp->local_hparams.dring_ident;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna msgp->seq_num = atomic_inc_32_nv(&ldcp->dringdata_msgid);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vgen_sendmsg_shm(ldcp, (caddr_t)tagp, sizeof (dringmsg));
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "vgen_sendmsg_shm() failed\n");
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DBG2(vgenp, ldcp, "DRING_DATA_SENT \n");
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Send dring data ack message.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_send_dringack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp, uint32_t start,
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna vio_dring_msg_t *msgp = (vio_dring_msg_t *)tagp;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna tagp->vio_subtype_env = VIO_DRING_DATA;
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna msgp->seq_num = atomic_inc_32_nv(&ldcp->dringdata_msgid);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = vgen_sendmsg_shm(ldcp, (caddr_t)tagp, sizeof (*msgp));
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "vgen_sendmsg_shm() failed\n");
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna * Send dring data msgs (info/ack/nack) over LDC.
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatnavgen_sendmsg_shm(vgen_ldc_t *ldcp, caddr_t msg, size_t msglen)
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna rv = ldc_write(ldcp->ldc_handle, (caddr_t)msg, &len);
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "ldc_write failed: rv(%d) msglen(%d)\n",
7bd3a2e26cc8569257b88c1691d559138e1d32d0Sriharsha Basavapatna DWARN(vgenp, ldcp, "ldc_write failed: rv(%d) msglen (%d)\n",