Lines Matching refs:channel
88 static int ioat_completion_alloc(ioat_channel_t channel);
89 static void ioat_completion_free(ioat_channel_t channel);
90 static void ioat_channel_start(ioat_channel_t channel);
91 static void ioat_channel_reset(ioat_channel_t channel);
93 int ioat_ring_alloc(ioat_channel_t channel, uint_t desc_cnt);
94 void ioat_ring_free(ioat_channel_t channel);
95 void ioat_ring_seed(ioat_channel_t channel, ioat_chan_dma_desc_t *desc);
96 int ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring,
113 * initialize each dma channel's state which doesn't change across
114 * channel alloc/free.
155 struct ioat_channel_s *channel;
168 /* allocate a H/W channel */
174 channel = &state->is_channel[chan_num];
175 channel->ic_inuse = B_TRUE;
176 channel->ic_chan_num = chan_num;
177 channel->ic_ver = state->is_ver;
178 channel->ic_dca_active = B_FALSE;
179 channel->ic_channel_state = IOAT_CHANNEL_OK;
180 channel->ic_dcopy_handle = handle;
186 if (channel->ic_ver == IOAT_CBv2) {
188 (uint16_t *)&channel->ic_regs[IOAT_CHAN_COMP]);
195 * Configure DMA channel
202 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x011C);
204 /* check channel error register, clear any errors */
206 (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]);
209 cmn_err(CE_CONT, "cleared errors (0x%x) before channel (%d) "
210 "enable\n", estat, channel->ic_chan_num);
213 (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR], estat);
217 e = ioat_ring_alloc(channel, size);
223 e = ioat_completion_alloc(channel);
233 state->is_instance, channel->ic_chan_num);
234 channel->ic_cmd_cache = kmem_cache_create(chanstr, cmd_size, 64,
236 if (channel->ic_cmd_cache == NULL) {
240 /* start-up the channel */
241 ioat_channel_start(channel);
243 /* fill in the channel info returned to dcopy */
248 info->qc_chan_num = (uint64_t)channel->ic_chan_num;
249 if (channel->ic_ver == IOAT_CBv1) {
259 *chan = channel;
264 ioat_completion_free(channel);
266 ioat_ring_free(channel);
296 ioat_channel_t channel;
302 channel = &state->is_channel[i];
303 ring = channel->ic_ring;
305 if (!channel->ic_inuse) {
310 * Configure DMA channel
317 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x011C);
319 /* check channel error register, clear any errors */
321 (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]);
324 cmn_err(CE_CONT, "cleared errors (0x%x) before channel"
325 " (%d) enable\n", estat, channel->ic_chan_num);
328 (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR],
333 bzero(ring->cr_desc, channel->ic_desc_alloc_size);
335 if (channel->ic_ver == IOAT_CBv1) {
337 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO],
340 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI],
343 ASSERT(channel->ic_ver == IOAT_CBv2);
345 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO],
348 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI],
353 bzero((void *)channel->ic_cmpl, channel->ic_cmpl_alloc_size);
356 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO],
357 (uint32_t)(channel->ic_phys_cmpl & 0xffffffff));
359 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI],
360 (uint32_t)(channel->ic_phys_cmpl >> 32));
362 /* start-up the channel */
363 ioat_channel_start(channel);
390 ioat_channel_t channel = state->is_channel + i;
392 if (!channel->ic_inuse)
397 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL],
400 ioat_channel_reset(channel);
411 struct ioat_channel_s *channel;
418 channel = *chan;
420 state = channel->ic_state;
421 chan_num = channel->ic_chan_num;
425 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x0);
427 ioat_channel_reset(channel);
430 kmem_cache_destroy(channel->ic_cmd_cache);
433 ioat_completion_free(channel);
434 ioat_ring_free(channel);
436 channel->ic_inuse = B_FALSE;
449 ioat_channel_intr(ioat_channel_t channel)
457 state = channel->ic_state;
459 if (channel->ic_ver == IOAT_CBv1) {
461 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_STS_LO]);
463 ASSERT(channel->ic_ver == IOAT_CBv2);
465 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_STS_LO]);
468 /* if that status isn't ACTIVE or IDLE, the channel has failed */
471 (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]);
472 cmn_err(CE_WARN, "channel(%d) fatal failure! "
474 channel->ic_chan_num, status, chanerr);
475 channel->ic_channel_state = IOAT_CHANNEL_IN_FAILURE;
476 ioat_channel_reset(channel);
486 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL]);
488 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], chanctrl);
490 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL]);
492 /* tell dcopy we have seen a completion on this channel */
493 dcopy_device_channel_notify(channel->ic_dcopy_handle, DCOPY_COMPLETION);
501 ioat_channel_start(ioat_channel_t channel)
513 ioat_ring_seed(channel, &desc);
521 ioat_channel_reset(ioat_channel_t channel)
525 state = channel->ic_state;
528 if (channel->ic_ver == IOAT_CBv1) {
530 &channel->ic_regs[IOAT_V1_CHAN_CMD], 0x20);
532 ASSERT(channel->ic_ver == IOAT_CBv2);
534 &channel->ic_regs[IOAT_V2_CHAN_CMD], 0x20);
543 ioat_completion_alloc(ioat_channel_t channel)
551 state = channel->ic_state;
558 DDI_DMA_SLEEP, NULL, &channel->ic_cmpl_dma_handle);
562 channel->ic_cmpl_alloc_size = 64;
563 e = ddi_dma_mem_alloc(channel->ic_cmpl_dma_handle,
564 channel->ic_cmpl_alloc_size, &ioat_acc_attr,
566 (caddr_t *)&channel->ic_cmpl, &real_length,
567 &channel->ic_cmpl_handle);
571 bzero((void *)channel->ic_cmpl, channel->ic_cmpl_alloc_size);
572 e = ddi_dma_addr_bind_handle(channel->ic_cmpl_dma_handle, NULL,
573 (caddr_t)channel->ic_cmpl, channel->ic_cmpl_alloc_size,
575 &channel->ic_cmpl_cookie, &cookie_cnt);
580 ASSERT(channel->ic_cmpl_cookie.dmac_size ==
581 channel->ic_cmpl_alloc_size);
582 channel->ic_phys_cmpl = channel->ic_cmpl_cookie.dmac_laddress;
586 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO],
587 (uint32_t)(channel->ic_phys_cmpl & 0xffffffff));
589 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI],
590 (uint32_t)(channel->ic_phys_cmpl >> 32));
595 ddi_dma_mem_free(&channel->ic_desc_handle);
597 ddi_dma_free_handle(&channel->ic_desc_dma_handle);
607 ioat_completion_free(ioat_channel_t channel)
611 state = channel->ic_state;
615 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO], 0x0);
617 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI], 0x0);
620 (void) ddi_dma_unbind_handle(channel->ic_cmpl_dma_handle);
621 ddi_dma_mem_free(&channel->ic_cmpl_handle);
622 ddi_dma_free_handle(&channel->ic_cmpl_dma_handle);
629 ioat_ring_alloc(ioat_channel_t channel, uint_t desc_cnt)
638 state = channel->ic_state;
641 channel->ic_ring = ring;
642 ring->cr_chan = channel;
646 channel->ic_state->is_iblock_cookie);
648 channel->ic_state->is_iblock_cookie);
656 DDI_DMA_SLEEP, NULL, &channel->ic_desc_dma_handle);
664 channel->ic_chan_desc_cnt = ((desc_cnt + 1) + 3) & ~0x3;
665 ring->cr_desc_last = channel->ic_chan_desc_cnt - 1;
666 channel->ic_desc_alloc_size = channel->ic_chan_desc_cnt *
668 e = ddi_dma_mem_alloc(channel->ic_desc_dma_handle,
669 channel->ic_desc_alloc_size, &ioat_acc_attr,
671 (caddr_t *)&ring->cr_desc, &real_length, &channel->ic_desc_handle);
675 bzero(ring->cr_desc, channel->ic_desc_alloc_size);
676 e = ddi_dma_addr_bind_handle(channel->ic_desc_dma_handle, NULL,
677 (caddr_t)ring->cr_desc, channel->ic_desc_alloc_size,
679 &channel->ic_desc_cookies, &cookie_cnt);
684 ASSERT(channel->ic_desc_cookies.dmac_size ==
685 channel->ic_desc_alloc_size);
686 ring->cr_phys_desc = channel->ic_desc_cookies.dmac_laddress;
689 if (channel->ic_ver == IOAT_CBv1) {
691 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO],
694 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI],
697 ASSERT(channel->ic_ver == IOAT_CBv2);
699 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO],
702 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI],
709 ddi_dma_mem_free(&channel->ic_desc_handle);
711 ddi_dma_free_handle(&channel->ic_desc_dma_handle);
715 kmem_free(channel->ic_ring, sizeof (ioat_channel_ring_t));
725 ioat_ring_free(ioat_channel_t channel)
730 state = channel->ic_state;
733 if (channel->ic_ver == IOAT_CBv1) {
735 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO], 0x0);
737 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI], 0x0);
739 ASSERT(channel->ic_ver == IOAT_CBv2);
741 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO], 0x0);
743 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI], 0x0);
747 (void) ddi_dma_unbind_handle(channel->ic_desc_dma_handle);
748 ddi_dma_mem_free(&channel->ic_desc_handle);
749 ddi_dma_free_handle(&channel->ic_desc_dma_handle);
751 mutex_destroy(&channel->ic_ring->cr_desc_mutex);
752 mutex_destroy(&channel->ic_ring->cr_cmpl_mutex);
753 kmem_free(channel->ic_ring, sizeof (ioat_channel_ring_t));
763 ioat_ring_seed(ioat_channel_t channel, ioat_chan_dma_desc_t *in_desc)
771 state = channel->ic_state;
772 ring = channel->ic_ring;
780 channel->ic_ring->cr_desc[0] = *(ioat_chan_desc_t *)in_desc;
785 if (channel->ic_ver == IOAT_CBv1) {
788 &channel->ic_regs[IOAT_V1_CHAN_CMD], 0x1);
807 (uint16_t *)&channel->ic_regs[IOAT_V2_CHAN_CNT],
822 ioat_channel_t channel;
827 channel = ring->cr_chan;
828 ASSERT(channel->ic_ver == IOAT_CBv1);
835 cmd != NULL && count <= channel->ic_chan_desc_cnt;
841 channel->ic_chan_desc_cnt) {
846 count += channel->ic_chan_desc_cnt
857 if (count >= channel->ic_chan_desc_cnt) {
866 (void) ddi_dma_sync(channel->ic_desc_dma_handle,
880 ioat_channel_t channel;
885 channel = (ioat_channel_t)private;
896 *cmd = kmem_cache_alloc(channel->ic_cmd_cache, kmflag);
932 ioat_channel_t channel;
937 channel = (ioat_channel_t)private;
947 kmem_cache_free(channel->ic_cmd_cache, cmd);
962 ioat_channel_t channel;
976 channel = (ioat_channel_t)private;
979 state = channel->ic_state;
980 ring = channel->ic_ring;
987 (channel->ic_ver != IOAT_CBv1 ||
1003 /* if the channel has had a fatal failure, return failure */
1004 if (channel->ic_channel_state == IOAT_CHANNEL_IN_FAILURE) {
1010 e = ioat_ring_reserve(channel, ring, cmd);
1017 if ((channel->ic_ver == IOAT_CBv2) &&
1099 if (channel->ic_ver == IOAT_CBv1) {
1101 (uint8_t *)&channel->ic_regs[IOAT_V1_CHAN_CMD],
1104 ASSERT(channel->ic_ver == IOAT_CBv2);
1106 (uint16_t *)&channel->ic_regs[IOAT_V2_CHAN_CNT],
1126 ioat_channel_t channel;
1132 channel = ring->cr_chan;
1141 * this channel, or if the id doesn't match the last id used on this
1142 * channel, set CONTEXT_CHANGE bit and dca id, set dca state to active,
1147 if (!channel->ic_dca_active || (channel->ic_dca_current != dca_id)) {
1148 channel->ic_dca_active = B_TRUE;
1149 channel->ic_dca_current = dca_id;
1185 (void) ddi_dma_sync(channel->ic_desc_dma_handle,
1196 (void) ddi_dma_sync(channel->ic_desc_dma_handle, next_offset, 64,
1201 (void) ddi_dma_sync(channel->ic_desc_dma_handle, prev_offset, 64,
1217 ioat_channel_t channel;
1223 channel = ring->cr_chan;
1269 (void) ddi_dma_sync(channel->ic_desc_dma_handle,
1280 (void) ddi_dma_sync(channel->ic_desc_dma_handle, next_offset, 64,
1285 (void) ddi_dma_sync(channel->ic_desc_dma_handle, prev_offset, 64,
1298 ioat_channel_t channel;
1303 channel = (ioat_channel_t)private;
1306 ring = channel->ic_ring;
1320 /* if the channel had a fatal failure, fail all polls */
1321 if ((channel->ic_channel_state == IOAT_CHANNEL_IN_FAILURE) ||
1322 IOAT_CMPL_FAILED(channel)) {
1333 (void) ddi_dma_sync(channel->ic_cmpl_dma_handle, 0, 0,
1335 last_cmpl = IOAT_CMPL_INDEX(channel);
1383 ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring,
1402 if ((channel->ic_ver == IOAT_CBv2) &&
1449 (void) ioat_cmd_poll(channel, cmd);
1473 (void) ioat_cmd_poll(channel, cmd);