Lines Matching defs:ring
96 int ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring,
99 static void ioat_cmd_post_copy(ioat_channel_ring_t *ring, uint64_t src_addr,
101 static void ioat_cmd_post_dca(ioat_channel_ring_t *ring, uint32_t dca_id);
295 ioat_channel_ring_t *ring;
303 ring = channel->ic_ring;
332 /* Re-initialize the ring */
333 bzero(ring->cr_desc, channel->ic_desc_alloc_size);
338 (uint32_t)(ring->cr_phys_desc & 0xffffffff));
341 (uint32_t)(ring->cr_phys_desc >> 32));
346 (uint32_t)(ring->cr_phys_desc & 0xffffffff));
349 (uint32_t)(ring->cr_phys_desc >> 32));
631 ioat_channel_ring_t *ring;
640 ring = kmem_zalloc(sizeof (ioat_channel_ring_t), KM_SLEEP);
641 channel->ic_ring = ring;
642 ring->cr_chan = channel;
643 ring->cr_post_cnt = 0;
645 mutex_init(&ring->cr_cmpl_mutex, NULL, MUTEX_DRIVER,
647 mutex_init(&ring->cr_desc_mutex, NULL, MUTEX_DRIVER,
651 * allocate memory for the ring, zero it out, and get the paddr.
665 ring->cr_desc_last = channel->ic_chan_desc_cnt - 1;
671 (caddr_t *)&ring->cr_desc, &real_length, &channel->ic_desc_handle);
675 bzero(ring->cr_desc, channel->ic_desc_alloc_size);
677 (caddr_t)ring->cr_desc, channel->ic_desc_alloc_size,
686 ring->cr_phys_desc = channel->ic_desc_cookies.dmac_laddress;
692 (uint32_t)(ring->cr_phys_desc & 0xffffffff));
695 (uint32_t)(ring->cr_phys_desc >> 32));
700 (uint32_t)(ring->cr_phys_desc & 0xffffffff));
703 (uint32_t)(ring->cr_phys_desc >> 32));
713 mutex_destroy(&ring->cr_desc_mutex);
714 mutex_destroy(&ring->cr_cmpl_mutex);
760 * write the first descriptor in the ring.
765 ioat_channel_ring_t *ring;
772 ring = channel->ic_ring;
775 ring->cr_cmpl_gen = 0x0;
776 ring->cr_cmpl_last = 0x0;
779 ring->cr_post_cnt++;
781 ring->cr_desc_gen = 0;
782 ring->cr_desc_prev = 0;
783 ring->cr_desc_next = 1;
794 ASSERT(ring->cr_chan->ic_ver == IOAT_CBv2);
796 &ring->cr_desc[ring->cr_desc_next];
798 &ring->cr_desc[ring->cr_desc_prev];
803 prev->dd_next_desc = ring->cr_phys_desc +
804 (ring->cr_desc_next << 6);
815 * Make the ring loop for CB v1
816 * This function assume we are in the ring->cr_desc_mutex mutex context
819 ioat_ring_loop(ioat_channel_ring_t *ring, dcopy_cmd_t cmd)
827 channel = ring->cr_chan;
832 * in descriptor ring. Return error if not continuous.
855 * descriptor ring. And make sure there is space for cancel operation.
862 curr = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_prev];
863 curr->dd_next_desc = ring->cr_phys_desc + (currpriv->ip_start << 6);
867 ring->cr_desc_prev << 6, 64, DDI_DMA_SYNC_FORDEV);
960 ioat_channel_ring_t *ring;
980 ring = channel->ic_ring;
993 mutex_enter(&ring->cr_desc_mutex);
999 } else if (mutex_tryenter(&ring->cr_desc_mutex) == 0) {
1005 mutex_exit(&ring->cr_desc_mutex);
1010 e = ioat_ring_reserve(channel, ring, cmd);
1012 mutex_exit(&ring->cr_desc_mutex);
1019 ioat_cmd_post_dca(ring, cmd->dp_dca_id);
1031 priv->ip_start = ring->cr_desc_next;
1073 ioat_cmd_post_copy(ring, src_paddr, dest_paddr, xfer_size,
1083 priv->ip_generation = ring->cr_desc_gen_prev;
1084 priv->ip_index = ring->cr_desc_prev;
1089 * Link the ring to a loop (currently only for FIPE).
1092 e = ioat_ring_loop(ring, cmd);
1094 mutex_exit(&ring->cr_desc_mutex);
1107 (uint16_t)(ring->cr_post_cnt & 0xFFFF));
1111 mutex_exit(&ring->cr_desc_mutex);
1121 ioat_cmd_post_dca(ioat_channel_ring_t *ring, uint32_t dca_id)
1132 channel = ring->cr_chan;
1133 desc = (ioat_chan_dca_desc_t *)&ring->cr_desc[ring->cr_desc_next];
1134 prev = (ioat_chan_dca_desc_t *)&ring->cr_desc[ring->cr_desc_prev];
1137 ring->cr_post_cnt++;
1159 prev_offset = ring->cr_desc_prev << 6;
1160 next_offset = ring->cr_desc_next << 6;
1161 next_desc_phys = ring->cr_phys_desc + next_offset;
1164 ring->cr_desc_prev = ring->cr_desc_next;
1165 ring->cr_desc_gen_prev = ring->cr_desc_gen;
1168 ring->cr_desc_next++;
1169 if (ring->cr_desc_next > ring->cr_desc_last) {
1170 ring->cr_desc_next = 0;
1171 ring->cr_desc_gen++;
1178 if (ring->cr_chan->ic_ver == IOAT_CBv2) {
1180 &ring->cr_desc[ring->cr_desc_next];
1182 &ring->cr_desc[ring->cr_desc_prev];
1186 ring->cr_desc_next << 6, 64, DDI_DMA_SYNC_FORDEV);
1187 prev->dd_next_desc = ring->cr_phys_desc +
1188 (ring->cr_desc_next << 6);
1211 ioat_cmd_post_copy(ioat_channel_ring_t *ring, uint64_t src_addr,
1223 channel = ring->cr_chan;
1224 desc = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_next];
1225 prev = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_prev];
1228 ring->cr_post_cnt++;
1242 prev_offset = ring->cr_desc_prev << 6;
1243 next_offset = ring->cr_desc_next << 6;
1244 next_desc_phy = ring->cr_phys_desc + next_offset;
1247 ring->cr_desc_prev = ring->cr_desc_next;
1248 ring->cr_desc_gen_prev = ring->cr_desc_gen;
1251 ring->cr_desc_next++;
1252 if (ring->cr_desc_next > ring->cr_desc_last) {
1253 ring->cr_desc_next = 0;
1254 ring->cr_desc_gen++;
1261 if (ring->cr_chan->ic_ver == IOAT_CBv2) {
1263 &ring->cr_desc[ring->cr_desc_next];
1265 &ring->cr_desc[ring->cr_desc_prev];
1270 ring->cr_desc_next << 6, 64, DDI_DMA_SYNC_FORDEV);
1271 prev->dd_next_desc = ring->cr_phys_desc +
1272 (ring->cr_desc_next << 6);
1296 ioat_channel_ring_t *ring;
1306 ring = channel->ic_ring;
1307 ASSERT(ring != NULL);
1310 mutex_enter(&ring->cr_cmpl_mutex);
1316 } else if (mutex_tryenter(&ring->cr_cmpl_mutex) == 0) {
1323 mutex_exit(&ring->cr_cmpl_mutex);
1330 * as indexes into the ring since post uses VAs and the H/W returns
1336 if (last_cmpl != ring->cr_cmpl_last) {
1338 * if we wrapped the ring, increment the generation. Store
1340 * ring.
1342 if (last_cmpl < ring->cr_cmpl_last) {
1343 ring->cr_cmpl_gen++;
1345 ring->cr_cmpl_last = last_cmpl;
1346 generation = ring->cr_cmpl_gen;
1349 generation = ring->cr_cmpl_gen;
1352 mutex_exit(&ring->cr_cmpl_mutex);
1356 * consumer pointer (ring->cr_cmpl_last).
1383 ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring,
1435 desc = ring->cr_desc_next;
1439 * if this is the last descriptor in the ring, see if the
1442 if (desc == ring->cr_desc_last) {
1443 if (ring->cr_cmpl_last == 0) {
1445 * if we think the ring is full, update where
1451 if (ring->cr_cmpl_last == 0) {
1463 * if this is not the last descriptor in the ring, see if
1467 if ((desc + 1) == ring->cr_cmpl_last) {
1469 * if we think the ring is full, update where
1475 if ((desc + 1) == ring->cr_cmpl_last) {