Lines Matching defs:mr
65 static int hermon_mr_common_rereg(hermon_state_t *state, hermon_mrhdl_t mr,
68 static int hermon_mr_rereg_xlat_helper(hermon_state_t *state, hermon_mrhdl_t mr,
179 hermon_mrhdl_t mr;
229 mr = (hermon_mrhdl_t)rsrc->hr_addr;
230 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
237 mr->mr_accflag = 0;
239 mr->mr_accflag |= IBT_MR_WINDOW_BIND;
241 mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
243 mr->mr_accflag |= IBT_MR_REMOTE_READ;
245 mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
247 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
258 mr->mr_rkey = mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
317 mr->mr_mttrsrcp = mrhdl->mr_mttrsrcp;
318 mr->mr_logmttpgsz = mrhdl->mr_logmttpgsz;
319 mr->mr_bindinfo = mrhdl->mr_bindinfo;
320 mr->mr_mttrefcntp = mrhdl->mr_mttrefcntp;
322 bind = &mr->mr_bindinfo;
324 mtt = mr->mr_mttrsrcp;
330 (void) hermon_mtt_refcnt_inc(mr->mr_mttrefcntp);
338 pgsize_msk = (((uint64_t)1 << mr->mr_logmttpgsz) - 1);
340 (mr->mr_bindinfo.bi_addr & pgsize_msk));
349 mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
350 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
351 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
352 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
353 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
356 mpt_entry.entity_sz = mr->mr_logmttpgsz;
357 mpt_entry.mem_key = mr->mr_lkey;
388 mr->mr_mptrsrcp = mpt;
389 mr->mr_mttrsrcp = mtt;
390 mr->mr_mpt_type = HERMON_MPT_DMPT;
391 mr->mr_pdhdl = pd;
392 mr->mr_rsrcp = rsrc;
393 mr->mr_is_umem = mr_is_umem;
394 mr->mr_is_fmr = 0;
395 mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL;
396 mr->mr_umem_cbfunc = NULL;
397 mr->mr_umem_cbarg1 = NULL;
398 mr->mr_umem_cbarg2 = NULL;
399 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
400 mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey);
412 *mrhdl_new = mr;
420 (void) hermon_mtt_refcnt_dec(mr->mr_mttrefcntp);
448 hermon_mrhdl_t mr;
499 mr = (hermon_mrhdl_t)rsrc->hr_addr;
500 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
507 mr->mr_accflag = 0;
509 mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
511 mr->mr_accflag |= IBT_MR_REMOTE_READ;
513 mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
515 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
526 mr->mr_fmr_key = 1; /* ready for the next reload */
527 mr->mr_rkey = mr->mr_lkey = mpt->hr_indx;
553 mr->mr_logmttpgsz = mtt_pgsize_bits;
563 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
564 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
565 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
566 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
571 mpt_entry.entity_sz = mr->mr_logmttpgsz;
577 mpt_entry.mem_key = mr->mr_lkey;
610 mr->mr_mptrsrcp = mpt;
611 mr->mr_mttrsrcp = mtt;
613 mr->mr_mpt_type = HERMON_MPT_DMPT;
614 mr->mr_pdhdl = pd;
615 mr->mr_rsrcp = rsrc;
616 mr->mr_is_fmr = 1;
617 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
618 mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey);
619 mr->mr_mttaddr = mtt_addr;
620 (void) memcpy(&mr->mr_bindinfo, &bind, sizeof (hermon_bind_info_t));
630 *mrhdl = mr;
657 ibt_pmr_attr_t *mem_pattr_p, hermon_mrhdl_t mr, ibt_pmr_desc_t *mem_desc_p)
664 mutex_enter(&mr->mr_lock);
665 mpt = mr->mr_mptrsrcp;
678 status = hermon_mr_fast_mtt_write_fmr(state, mr->mr_mttrsrcp,
679 mem_pattr_p, mr->mr_logmttpgsz);
681 mutex_exit(&mr->mr_lock);
695 key = mpt->hr_indx | (mr->mr_fmr_key++ << HERMON_MEMKEY_SHIFT);
696 mr->mr_lkey = mr->mr_rkey = hermon_mr_key_swap(key);
716 mem_desc_p->pmd_lkey = mr->mr_lkey;
717 mem_desc_p->pmd_rkey = mr->mr_rkey;
722 mr->mr_bindinfo.bi_addr = mem_pattr_p->pmr_iova;
723 mr->mr_bindinfo.bi_flags = mem_pattr_p->pmr_flags & IBT_MR_NONCOHERENT;
725 mutex_exit(&mr->mr_lock);
751 hermon_mrhdl_t mr;
774 mr = *mrhdl;
775 mutex_enter(&mr->mr_lock);
776 mpt = mr->mr_mptrsrcp;
777 mtt = mr->mr_mttrsrcp;
778 mtt_refcnt = mr->mr_mttrefcntp;
779 rsrc = mr->mr_rsrcp;
780 pd = mr->mr_pdhdl;
781 bind = &mr->mr_bindinfo;
787 if (mr->mr_is_fmr) {
788 mutex_exit(&mr->mr_lock);
797 if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
801 IBTF_DPRINTF_L2("mr", "dereg: mr %p key %x",
802 mr, mr->mr_rkey);
811 mutex_exit(&mr->mr_lock);
825 if (mr->mr_mpt_type >= HERMON_MPT_DMPT) {
848 * 'mr' structure is needed, and we would have returned previously for
851 mutex_enter(&mr->mr_lock);
863 if (mr->mr_is_umem) {
865 (uint64_t)(uintptr_t)mr->mr_umemcookie,
870 ddi_umem_unlock(mr->mr_umemcookie);
872 ddi_umem_unlock(mr->mr_umemcookie);
873 mr->mr_umemcookie = NULL;
912 if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
913 mutex_exit(&mr->mr_lock);
918 mutex_exit(&mr->mr_lock);
946 hermon_mrhdl_t mr;
954 mr = *mrhdl;
955 mutex_enter(&mr->mr_lock);
956 mpt = mr->mr_mptrsrcp;
957 mtt = mr->mr_mttrsrcp;
958 rsrc = mr->mr_rsrcp;
959 pd = mr->mr_pdhdl;
960 mutex_exit(&mr->mr_lock);
987 hermon_mr_query(hermon_state_t *state, hermon_mrhdl_t mr,
996 mutex_enter(&mr->mr_lock);
1003 if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
1004 mutex_exit(&mr->mr_lock);
1009 mr->mr_lkey >> 8, &mpt_entry, sizeof (hermon_hw_dmpt_t),
1013 mutex_exit(&mr->mr_lock);
1017 /* Update the mr sw struct from the hw struct. */
1019 mr->mr_lkey = mr->mr_rkey = (lkey >> 8) | (lkey << 24);
1020 mr->mr_bindinfo.bi_addr = mpt_entry.start_addr;
1021 mr->mr_bindinfo.bi_len = mpt_entry.reg_win_len;
1022 mr->mr_accflag = (mr->mr_accflag & IBT_MR_RO_DISABLED) |
1028 mr->mr_mttaddr = ((uint64_t)mpt_entry.mtt_addr_h << 32) |
1030 mr->mr_logmttpgsz = mpt_entry.entity_sz;
1038 attr->mr_attr_flags = mr->mr_accflag;
1039 attr->mr_pd = (ibt_pd_hdl_t)mr->mr_pdhdl;
1042 attr->mr_lkey = (ibt_lkey_t)mr->mr_lkey;
1043 attr->mr_lbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr;
1044 attr->mr_lbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len;
1051 if ((mr->mr_accflag & IBT_MR_REMOTE_READ) ||
1052 (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ||
1053 (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) {
1054 attr->mr_rkey = (ibt_rkey_t)mr->mr_rkey;
1055 attr->mr_rbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr;
1056 attr->mr_rbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len;
1063 attr->mr_sync_required = (mr->mr_bindinfo.bi_flags &
1066 mutex_exit(&mr->mr_lock);
1076 hermon_mr_reregister(hermon_state_t *state, hermon_mrhdl_t mr,
1096 status = hermon_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op);
1106 hermon_mr_reregister_buf(hermon_state_t *state, hermon_mrhdl_t mr,
1135 status = hermon_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op);
1476 hermon_mrhdl_t mr;
1564 mr = (hermon_mrhdl_t)rsrc->hr_addr;
1565 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
1572 mr->mr_accflag = 0;
1574 mr->mr_accflag |= IBT_MR_WINDOW_BIND;
1576 mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
1578 mr->mr_accflag |= IBT_MR_REMOTE_READ;
1580 mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
1582 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
1594 mr->mr_rkey = mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
1646 bh = &mr->mr_bindinfo;
1660 mr->mr_logmttpgsz = mtt_pgsize_bits;
1675 mr->mr_mttrefcntp = mtt_refcnt;
1694 mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
1695 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
1696 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
1697 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
1698 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
1703 mpt_entry.entity_sz = mr->mr_logmttpgsz;
1704 mpt_entry.mem_key = mr->mr_lkey;
1715 bh->bi_addr = bh->bi_addr & ((1 << mr->mr_logmttpgsz) - 1);
1748 IBTF_DPRINTF_L2("mr", " reg: mr %p key %x",
1749 mr, hermon_mr_key_swap(mr->mr_rkey));
1757 mr->mr_mttaddr = mtt_addr;
1759 mr->mr_log2_pgsz = (mr->mr_logmttpgsz - HERMON_PAGESHIFT);
1760 mr->mr_mptrsrcp = mpt;
1761 mr->mr_mttrsrcp = mtt;
1762 mr->mr_pdhdl = pd;
1763 mr->mr_rsrcp = rsrc;
1764 mr->mr_is_umem = mr_is_umem;
1765 mr->mr_is_fmr = 0;
1766 mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL;
1767 mr->mr_umem_cbfunc = NULL;
1768 mr->mr_umem_cbarg1 = NULL;
1769 mr->mr_umem_cbarg2 = NULL;
1770 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
1771 mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey);
1772 mr->mr_mpt_type = mpt_type;
1784 *mrhdl = mr;
1835 hermon_mrhdl_t mr;
1882 mr = (hermon_mrhdl_t)rsrc->hr_addr;
1883 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
1884 bzero(mr, sizeof (*mr));
1891 mr->mr_accflag = 0;
1893 mr->mr_accflag |= IBT_MR_WINDOW_BIND;
1895 mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
1897 mr->mr_accflag |= IBT_MR_REMOTE_READ;
1899 mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
1901 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
1913 mr->mr_rkey = mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
1924 mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
1925 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
1926 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
1927 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
1928 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
1933 mpt_entry.entity_sz = mr->mr_logmttpgsz;
1934 mpt_entry.mem_key = mr->mr_lkey;
1980 mr->mr_mttaddr = 0;
1982 mr->mr_log2_pgsz = 0;
1983 mr->mr_mptrsrcp = mpt;
1984 mr->mr_mttrsrcp = NULL;
1985 mr->mr_pdhdl = pd;
1986 mr->mr_rsrcp = rsrc;
1987 mr->mr_is_umem = 0;
1988 mr->mr_is_fmr = 0;
1989 mr->mr_umemcookie = NULL;
1990 mr->mr_umem_cbfunc = NULL;
1991 mr->mr_umem_cbarg1 = NULL;
1992 mr->mr_umem_cbarg2 = NULL;
1993 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
1994 mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey);
1995 mr->mr_mpt_type = HERMON_MPT_DMPT;
1997 *mrhdl = mr;
2025 hermon_mrhdl_t mr;
2061 mr = (hermon_mrhdl_t)rsrc->hr_addr;
2062 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
2063 bzero(mr, sizeof (*mr));
2064 mr->mr_bindinfo.bi_type = HERMON_BINDHDL_LKEY;
2066 mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
2073 mr->mr_logmttpgsz = PAGESHIFT;
2088 mr->mr_mttrefcntp = mtt_refcnt;
2100 mpt_entry.entity_sz = mr->mr_logmttpgsz;
2101 mpt_entry.mem_key = mr->mr_lkey;
2141 mr->mr_accflag = IBT_MR_LOCAL_WRITE;
2142 mr->mr_mttaddr = mtt_addr;
2143 mr->mr_log2_pgsz = (mr->mr_logmttpgsz - HERMON_PAGESHIFT);
2144 mr->mr_mptrsrcp = mpt;
2145 mr->mr_mttrsrcp = mtt;
2146 mr->mr_pdhdl = pd;
2147 mr->mr_rsrcp = rsrc;
2148 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
2149 mr->mr_rkey = mr->mr_lkey;
2150 mr->mr_mpt_type = HERMON_MPT_DMPT;
2152 *mrhdl = mr;
2378 hermon_mr_common_rereg(hermon_state_t *state, hermon_mrhdl_t mr,
2398 if (mr->mr_is_umem) {
2403 mutex_enter(&mr->mr_lock);
2406 mpt = mr->mr_mptrsrcp;
2419 mutex_exit(&mr->mr_lock);
2440 mutex_exit(&mr->mr_lock);
2454 if (hermon_mr_deregister(state, &mr,
2470 mutex_exit(&mr->mr_lock);
2478 if (hermon_mr_deregister(state, &mr,
2493 pd_to_use = mr->mr_pdhdl;
2507 mutex_exit(&mr->mr_lock);
2515 if (hermon_mr_deregister(state, &mr,
2543 acc_flags_to_use = mr->mr_accflag;
2561 status = hermon_mr_rereg_xlat_helper(state, mr, bind, op,
2564 mutex_exit(&mr->mr_lock);
2569 if (hermon_mr_deregister(state, &mr, dereg_level,
2576 vaddr_to_use = mr->mr_bindinfo.bi_addr;
2577 len_to_use = mr->mr_bindinfo.bi_len;
2579 mtt_addr_to_use = mr->mr_mttaddr;
2580 vaddr_to_use = mr->mr_bindinfo.bi_addr;
2581 len_to_use = mr->mr_bindinfo.bi_len;
2591 mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
2595 mr->mr_rkey = mr->mr_lkey;
2597 mr->mr_rkey = 0;
2617 mpt_entry.entity_sz = mr->mr_logmttpgsz;
2618 mpt_entry.mem_key = mr->mr_lkey;
2635 mutex_exit(&mr->mr_lock);
2647 if (hermon_mr_deregister(state, &mr,
2661 hermon_pd_refcnt_dec(mr->mr_pdhdl);
2669 mr->mr_pdhdl = pd_to_use;
2670 mr->mr_accflag = acc_flags_to_use;
2671 mr->mr_is_umem = 0;
2672 mr->mr_is_fmr = 0;
2673 mr->mr_umemcookie = NULL;
2674 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
2675 mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey);
2678 *mrhdl_new = mr;
2679 mutex_exit(&mr->mr_lock);
2696 hermon_mr_rereg_xlat_helper(hermon_state_t *state, hermon_mrhdl_t mr,
2707 ASSERT(MUTEX_HELD(&mr->mr_lock));
2746 nummtt_in_currrsrc = mr->mr_mttrsrcp->hr_len >> HERMON_MTT_SIZE_SHIFT;
2754 * the mr to reflect the changes.
2760 * new MTT entries, and updating the mr to reflect the changes.
2769 swrc_old = (hermon_sw_refcnt_t *)mr->mr_mttrefcntp->hr_addr;
2780 if (HERMON_MR_REUSE_DMAHDL(mr, bind->bi_flags)) {
2781 mr->mr_bindinfo.bi_free_dmahdl = 0;
2782 hermon_mr_mem_unbind(state, &mr->mr_bindinfo);
2783 dmahdl = mr->mr_bindinfo.bi_dmahdl;
2786 hermon_mr_mem_unbind(state, &mr->mr_bindinfo);
2830 mtt = mr->mr_mttrsrcp;
2859 mr->mr_bindinfo = *bind;
2860 mr->mr_logmttpgsz = mtt_pgsize_bits;
2880 if (HERMON_MR_REUSE_DMAHDL(mr, bind->bi_flags)) {
2881 mr->mr_bindinfo.bi_free_dmahdl = 0;
2882 hermon_mr_mem_unbind(state, &mr->mr_bindinfo);
2883 dmahdl = mr->mr_bindinfo.bi_dmahdl;
2886 hermon_mr_mem_unbind(state, &mr->mr_bindinfo);
2995 mtt_refcnt = mr->mr_mttrefcntp;
3038 (void) hermon_mtt_refcnt_dec(mr->mr_mttrefcntp);
3041 hermon_rsrc_free(state, &mr->mr_mttrsrcp);
3045 mr->mr_bindinfo = *bind;
3046 mr->mr_logmttpgsz = mtt_pgsize_bits;
3047 mr->mr_mttrsrcp = mtt;
3048 mr->mr_mttrefcntp = mtt_refcnt;