Lines Matching refs:mp
53 * right after (accessed through (ddi_dma_attr_t *)(mp + 1)) is used
62 register ddi_dma_impl_t *mp;
66 if ((mp = kmem_alloc(sizeof (px_dma_hdl_t), sleep)) == 0) {
72 return (mp);
75 mp->dmai_rdip = rdip;
76 mp->dmai_flags = 0;
77 mp->dmai_pfnlst = NULL;
78 mp->dmai_winlst = NULL;
82 * mp->dmai_mapping = 0;
83 * mp->dmai_size = 0;
84 * mp->dmai_offset = 0;
85 * mp->dmai_minxfer = 0;
86 * mp->dmai_burstsizes = 0;
87 * mp->dmai_ndvmapages = 0;
88 * mp->dmai_pool/roffset = 0;
89 * mp->dmai_rflags = 0;
90 * mp->dmai_inuse/flags
91 * mp->dmai_nwin = 0;
92 * mp->dmai_winsize = 0;
93 * mp->dmai_nexus_private/tte = 0;
94 * mp->dmai_iopte/pfnlst
95 * mp->dmai_sbi/pfn0 = 0;
96 * mp->dmai_minfo/winlst/fdvma
97 * mp->dmai_rdip
98 * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t));
99 * bzero(&mp->dmai_attr, sizeof (ddi_dma_attr_t));
100 * mp->dmai_cookie = 0;
103 mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION;
104 mp->dmai_attr.dma_attr_flags = (uint_t)0;
105 mp->dmai_fault = 0;
106 mp->dmai_fault_check = NULL;
107 mp->dmai_fault_notify = NULL;
109 mp->dmai_error.err_ena = 0;
110 mp->dmai_error.err_status = DDI_FM_OK;
111 mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
112 mp->dmai_error.err_ontrap = NULL;
113 mp->dmai_error.err_fep = NULL;
114 mp->dmai_error.err_cf = NULL;
124 mp->dmai_bdf = ((intptr_t)ddi_get_parent_data(rdip) == 1) ?
127 ndi_fmc_insert(rdip, DMA_HANDLE, mp, NULL);
128 return (mp);
132 px_dma_freemp(ddi_dma_impl_t *mp)
134 ndi_fmc_remove(mp->dmai_rdip, DMA_HANDLE, mp);
135 if (mp->dmai_ndvmapages > 1)
136 px_dma_freepfn(mp);
137 if (mp->dmai_winlst)
138 px_dma_freewin(mp);
139 kmem_free(mp, sizeof (px_dma_hdl_t));
143 px_dma_freepfn(ddi_dma_impl_t *mp)
145 void *addr = mp->dmai_pfnlst;
147 size_t npages = mp->dmai_ndvmapages;
150 mp->dmai_pfnlst = NULL;
152 mp->dmai_ndvmapages = 0;
157 * and convert dmareq->dmar_limits to mp->dmai_attr
161 * mp->dmai_minxfer - dev
162 * mp->dmai_burstsizes - dev
163 * mp->dmai_flags - no limit? peer-to-peer only?
167 * mp->dmai_attr.dma_attr_addr_lo - dev lo, sys lo
168 * mp->dmai_attr.dma_attr_addr_hi - dev hi, sys hi
169 * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta
170 * mp->dmai_attr.dma_attr_seg - 0 (no nocross restriction)
171 * mp->dmai_attr.dma_attr_align - 1 (no alignment restriction)
179 ddi_dma_impl_t *mp;
195 if (!(mp = px_dma_allocmp(dip, rdip, dmareq->dmar_fp,
200 attr_p = PX_DEV_ATTR(mp);
207 mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY;
211 mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT |
215 mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT;
218 mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX;
220 /* store augumented dev input to mp->dmai_attr */
221 mp->dmai_burstsizes = lim_p->dlim_burstsizes;
222 attr_p = &mp->dmai_attr;
225 return (mp);
267 px_dma_attr2hdl(px_t *px_p, ddi_dma_impl_t *mp)
272 ddi_dma_attr_t *attrp = PX_DEV_ATTR(mp);
297 mp->dmai_flags |= PX_DMAI_FLAGS_BYPASSREQ;
312 dev_info_t *rdip = mp->dmai_rdip;
322 dev_info_t *rdip = mp->dmai_rdip;
340 if ((mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ||
344 mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY;
349 mp->dmai_flags |= PX_DMAI_FLAGS_NOSYSLIMIT |
354 mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT;
356 if (PX_DMA_NOCTX(mp->dmai_rdip))
357 mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX;
359 mp->dmai_burstsizes = attrp->dma_attr_burstsizes;
360 attrp = &mp->dmai_attr;
378 * mp->dmai_object - dmareq->dmar_object
379 * mp->dmai_rflags - consistent?, nosync?, dmareq->dmar_flags
380 * mp->dmai_flags - DMA type
381 * mp->dmai_pfn0 - 1st page pfn (if va/size pair and not shadow)
382 * mp->dmai_roffset - initialized to starting MMU page offset
383 * mp->dmai_ndvmapages - # of total MMU pages of entire object
386 px_dma_type(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
395 mp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
398 mp->dmai_rflags |= DMP_NOSYNC;
409 mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN;
421 mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN;
428 NAMEINST(mp->dmai_rdip), dobj_p->dmao_type);
438 mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP32;
442 mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP64;
445 if (PX_DMA_ISPEERONLY(mp)) {
446 dev_info_t *rdip = mp->dmai_rdip;
451 redzone = (mp->dmai_rflags & DDI_DMA_REDZONE) ||
452 (mp->dmai_flags & PX_DMAI_FLAGS_MAP_BUFZONE) ?
455 mp->dmai_flags |= (mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ?
458 mp->dmai_object = *dobj_p; /* whole object */
459 mp->dmai_pfn0 = (void *)pfn0; /* cache pfn0 */
460 mp->dmai_roffset = offset; /* win0 pg0 offset */
461 mp->dmai_ndvmapages = MMU_BTOPR(offset + mp->dmai_object.dmao_size);
471 px_dma_pgpfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages)
476 switch (mp->dmai_object.dmao_type) {
479 page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv;
484 PX_SET_MP_PFN1(mp, i, pfn);
492 page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next;
496 PX_SET_MP_PFN1(mp, i, pfn);
517 px_dma_vapfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages)
521 caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as;
524 vaddr = mp->dmai_object.dmao_obj.virt_obj.v_addr + MMU_PAGE_SIZE;
529 PX_SET_MP_PFN1(mp, i, pfn);
530 DBG(DBG_DMA_BINDH, dip, "px_dma_vapfn: mp=%p pfnlst[%x]=%x\n",
531 mp, i, pfn);
543 * mp->dmai_ndvmapages - set to total # of dma pages
550 px_dma_pfn(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
552 uint32_t npages = mp->dmai_ndvmapages;
554 int i, ret, peer = PX_DMA_ISPTP(mp);
555 int peer32 = PX_DMA_ISPTP32(mp);
565 DBG(DBG_DMA_BINDH, dip, "px_dma_pfn: mp=%p pfn0=%x\n",
566 mp, PX_MP_PFN0(mp) - pfn_adj);
569 PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj);
573 if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (px_iopfn_t),
581 PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj); /* pfnlst[0] */
582 if ((ret = PX_DMA_ISPGPFN(mp) ? px_dma_pgpfn(px_p, mp, npages) :
583 px_dma_vapfn(px_p, mp, npages)) != DDI_SUCCESS)
588 px_iopfn_t pfn = PX_GET_MP_PFN1(mp, i);
591 NAMEINST(mp->dmai_rdip), PX_MP_PFN0(mp), pfn);
598 PX_SET_MP_PFN1(mp, i, pfn - pfn_adj);
602 px_dma_freepfn(mp);
612 * accesses the following mp-> members:
613 * mp->dmai_attr.dma_attr_count_max
614 * mp->dmai_attr.dma_attr_seg
615 * mp->dmai_roffset - start offset of 1st window
616 * mp->dmai_rflags (redzone)
617 * mp->dmai_ndvmapages (for 1 page fast path)
619 * sets the following mp-> members:
620 * mp->dmai_size - xfer size, != winsize if 1st/last win (not fixed)
621 * mp->dmai_winsize - window size (no redzone), n * page size (fixed)
622 * mp->dmai_nwin - # of DMA windows of entire object (fixed)
623 * mp->dmai_rflags - remove partial flag if nwin == 1 (fixed)
624 * mp->dmai_winlst - NULL, window objects not used for DVMA (fixed)
630 px_dvma_win(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
632 uint32_t redzone_sz = PX_HAS_REDZONE(mp) ? MMU_PAGE_SIZE : 0;
633 size_t obj_sz = mp->dmai_object.dmao_size;
637 if ((mp->dmai_ndvmapages == 1) && !redzone_sz) {
638 mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
639 mp->dmai_size = obj_sz;
640 mp->dmai_winsize = MMU_PAGE_SIZE;
641 mp->dmai_nwin = 1;
645 pg_off = mp->dmai_roffset;
649 uint64_t nocross = mp->dmai_attr.dma_attr_seg;
662 uint32_t count_max = mp->dmai_attr.dma_attr_count_max;
667 mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
668 mp->dmai_size = xfer_sz;
669 mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, MMU_PAGE_SIZE);
670 mp->dmai_nwin = 1;
680 mp->dmai_size = xfer_sz - pg_off; /* 1st window xferrable size */
681 mp->dmai_winsize = xfer_sz; /* redzone not in winsize */
682 mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz;
684 mp->dmai_winlst = NULL;
685 px_dump_dma_handle(DBG_DMA_MAP, px_p->px_dip, mp);
700 px_dvma_map_fast(px_mmu_t *mmu_p, ddi_dma_impl_t *mp)
704 io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
705 mp->dmai_attr.dma_attr_flags);
709 size_t npages = MMU_BTOP(mp->dmai_winsize);
713 ASSERT(MMU_PTOB(npages) == mp->dmai_winsize);
714 ASSERT(npages + PX_HAS_REDZONE(mp) <= clustsz);
736 PX_ADD_ATTR_EXTNS(attr, mp->dmai_bdf), (void *)mp, 0,
743 if (!PX_MAP_BUFZONE(mp))
748 ASSERT(PX_HAS_REDZONE(mp));
751 PX_ADD_ATTR_EXTNS(attr, mp->dmai_bdf), (void *)mp, npages - 1,
764 mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg);
765 mp->dmai_offset = 0;
766 mp->dmai_flags |= PX_DMAI_FLAGS_FASTTRACK;
767 PX_SAVE_MP_TTE(mp, attr); /* save TTE template for unmapping */
769 px_dvma_alloc_debug(mmu_p, (char *)mp->dmai_mapping,
770 mp->dmai_size, mp);
779 px_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, px_mmu_t *mmu_p)
781 uint_t npages = PX_DMA_WINNPGS(mp);
784 io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
785 mp->dmai_attr.dma_attr_flags);
787 dev_info_t *dip = mp->dmai_rdip;
796 if ((npages == 1) && !PX_HAS_REDZONE(mp) && PX_HAS_NOSYSLIMIT(mp)) {
799 mp->dmai_flags |= PX_DMAI_FLAGS_VMEMCACHE;
805 MMU_PTOB(npages + PX_HAS_REDZONE(mp)),
806 MAX(mp->dmai_attr.dma_attr_align, MMU_PAGE_SIZE),
808 mp->dmai_attr.dma_attr_seg + 1,
809 (void *)mp->dmai_attr.dma_attr_addr_lo,
810 (void *)(mp->dmai_attr.dma_attr_addr_hi + 1),
823 mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg);
824 mp->dmai_offset = 0;
825 PX_SAVE_MP_TTE(mp, attr); /* mp->dmai_tte = tte */
828 mp, dvma_pg, npages, 0)) != DDI_SUCCESS) {
829 if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) {
837 MMU_PTOB(npages + PX_HAS_REDZONE(mp)));
856 px_dvma_unmap(px_mmu_t *mmu_p, ddi_dma_impl_t *mp)
858 px_dvma_addr_t dvma_addr = (px_dvma_addr_t)mp->dmai_mapping;
862 if (mp->dmai_flags & PX_DMAI_FLAGS_FASTTRACK) {
874 if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) {
881 size_t npages = MMU_BTOP(mp->dmai_winsize) + PX_HAS_REDZONE(mp);
895 px_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
909 px_dma_freewin(ddi_dma_impl_t *mp)
911 px_dma_win_t *win_p = mp->dmai_winlst, *win2_p;
917 mp->dmai_nwin = 0;
918 mp->dmai_winlst = NULL;
952 px_dma_newwin(dev_info_t *dip, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp,
959 px_iopfn_t pfn = PX_GET_MP_PFN(mp, start_idx);
965 io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
966 mp->dmai_attr.dma_attr_flags);
980 pfn = PX_GET_MP_PFN1(mp, start_idx);
996 DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n",
1014 DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n",
1039 * mp->dmai_size - 1st window xferrable size
1040 * mp->dmai_offset - 0, which is the dma offset of the 1st window
1047 px_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, px_dma_win_t *win_p)
1050 size_t pg_offset = mp->dmai_roffset;
1056 DBG(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset);
1058 mp->dmai_size = win_p->win_size;
1059 mp->dmai_offset = 0;
1061 pg_offset += mp->dmai_object.dmao_size;
1065 DBG(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset);
1068 DBG(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset);
1077 ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size);
1085 * Dependency: mp->dmai_pfnlst points to an array of pfns
1098 * mp->dmai_attr.dma_attr_sgllen. If the transfer size is
1106 * mp->dmai_winlst - points to a link list of px_dma_win_t objects.
1111 * mp->dmai_pfnlst - NULL, the pfn list is freed to conserve memory.
1112 * mp->dmai_nwin - # of total DMA windows on mp->dmai_winlst.
1113 * mp->dmai_mapping - starting cookie address
1114 * mp->dmai_rflags - consistent, nosync, no redzone
1115 * mp->dmai_cookie - start of cookie table of the 1st DMA window
1126 px_dma_physwin(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
1128 uint_t npages = mp->dmai_ndvmapages;
1129 int ret, sgllen = mp->dmai_attr.dma_attr_sgllen;
1131 px_iopfn_t pfn = PX_GET_MP_PFN(mp, 0);
1134 px_dma_win_t **win_pp = (px_dma_win_t **)&mp->dmai_winlst;
1136 io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
1137 mp->dmai_attr.dma_attr_flags);
1140 ASSERT(PX_DMA_ISPTP(mp) || PX_DMA_ISBYPASS(mp));
1141 if (PX_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */
1142 ddi_dma_attr_t *dev_attr_p = PX_DEV_ATTR(mp);
1145 px_iopfn_t pfn_last = PX_DMA_ISPTP32(mp) ?
1167 count_max = mp->dmai_attr.dma_attr_count_max;
1168 pfn_lo = MMU_BTOP(mp->dmai_attr.dma_attr_addr_lo);
1169 pfn_hi = MMU_BTOP(mp->dmai_attr.dma_attr_addr_hi);
1173 DBG(DBG_BYPASS, mp->dmai_rdip,
1183 pfn = PX_GET_MP_PFN1(mp, i);
1204 DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
1206 if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no,
1220 DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
1222 if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no, win_pfn0_index,
1226 px_dma_adjust(dmareq, mp, mp->dmai_winlst);
1227 mp->dmai_nwin = win_no;
1228 mp->dmai_rflags |= DDI_DMA_CONSISTENT | DMP_NOSYNC;
1229 mp->dmai_rflags &= ~DDI_DMA_REDZONE;
1230 mp->dmai_flags |= PX_DMAI_FLAGS_NOSYNC;
1231 cookie0_p = (ddi_dma_cookie_t *)(PX_WINLST(mp) + 1);
1232 mp->dmai_cookie = PX_WINLST(mp)->win_ncookies > 1 ? cookie0_p + 1 : 0;
1233 mp->dmai_mapping = cookie0_p->dmac_laddress;
1235 px_dma_freepfn(mp);
1238 px_dma_freewin(mp);
1243 px_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
1307 ddi_dma_impl_t *mp)
1322 ptr->mp = mp;
1329 ptr->mp = mp;
1340 ddi_dma_impl_t *mp)
1355 ptr->mp = mp;
1384 DBG(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n",