Lines Matching refs:sfmmup
318 * The lock is hashed on the sfmmup since the case where we need to lock
343 #define TSB_HASH(sfmmup) \
345 &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)])
347 #define TSB_HASH(sfmmup) &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)]
475 #define sfmmu_load_tsb(sfmmup, vaddr, tte, szc) \
476 sfmmu_mod_tsb(sfmmup, vaddr, tte, szc)
477 #define sfmmu_unload_tsb(sfmmup, vaddr, szc) \
478 sfmmu_mod_tsb(sfmmup, vaddr, NULL, szc)
837 #define SFMMU_GET_TSBINFO(tsbinfop, sfmmup, tte_szc) { \
838 (tsbinfop) = (sfmmup)->sfmmu_tsb; \
840 sfmmu_hat_lock_held(sfmmup)); \
852 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, ismhat) \
856 sfmmu_unload_tsb(sfmmup, addr, ttesz); \
862 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); \
1459 sfmmu_t *sfmmup;
1465 sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP);
1466 sfmmup->sfmmu_as = as;
1467 sfmmup->sfmmu_flags = 0;
1468 sfmmup->sfmmu_tteflags = 0;
1469 sfmmup->sfmmu_rtteflags = 0;
1470 LOCK_INIT_CLEAR(&sfmmup->sfmmu_ctx_lock);
1473 ksfmmup = sfmmup;
1474 sfmmup->sfmmu_cext = 0;
1477 sfmmup->sfmmu_clrstart = 0;
1478 sfmmup->sfmmu_tsb = NULL;
1492 sfmmup->sfmmu_cext = 0;
1496 sfmmup->sfmmu_clrstart = get_color_start(as);
1503 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, size,
1504 TSB8K|TSB64K|TSB512K, 0, sfmmup);
1507 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb,
1509 TSB8K|TSB64K|TSB512K, 0, sfmmup);
1510 sfmmup->sfmmu_flags = HAT_SWAPPED | HAT_ALLCTX_INVALID;
1511 ASSERT(sfmmup->sfmmu_tsb != NULL);
1516 sfmmup->sfmmu_ctxs[i].cnum = cnum;
1517 sfmmup->sfmmu_ctxs[i].gnum = 0;
1521 sfmmup->sfmmu_ttecnt[i] = 0;
1522 sfmmup->sfmmu_scdrttecnt[i] = 0;
1523 sfmmup->sfmmu_ismttecnt[i] = 0;
1524 sfmmup->sfmmu_scdismttecnt[i] = 0;
1525 sfmmup->sfmmu_pgsz[i] = TTE8K;
1527 sfmmup->sfmmu_tsb0_4minflcnt = 0;
1528 sfmmup->sfmmu_iblk = NULL;
1529 sfmmup->sfmmu_ismhat = 0;
1530 sfmmup->sfmmu_scdhat = 0;
1531 sfmmup->sfmmu_ismblkpa = (uint64_t)-1;
1532 if (sfmmup == ksfmmup) {
1533 CPUSET_ALL(sfmmup->sfmmu_cpusran);
1535 CPUSET_ZERO(sfmmup->sfmmu_cpusran);
1537 sfmmup->sfmmu_free = 0;
1538 sfmmup->sfmmu_rmstat = 0;
1539 sfmmup->sfmmu_clrbin = sfmmup->sfmmu_clrstart;
1540 cv_init(&sfmmup->sfmmu_tsb_cv, NULL, CV_DEFAULT, NULL);
1541 sfmmup->sfmmu_srdp = NULL;
1542 SF_RGNMAP_ZERO(sfmmup->sfmmu_region_map);
1543 bzero(sfmmup->sfmmu_hmeregion_links, SFMMU_L1_HMERLINKS_SIZE);
1544 sfmmup->sfmmu_scdp = NULL;
1545 sfmmup->sfmmu_scd_link.next = NULL;
1546 sfmmup->sfmmu_scd_link.prev = NULL;
1547 return (sfmmup);
1865 hat_setup(struct hat *sfmmup, int allocflag)
1876 hatlockp = sfmmu_hat_enter(sfmmup);
1884 sfmmu_tsb_swapin(sfmmup, hatlockp);
1886 sfmmu_get_ctx(sfmmup);
1892 hatlockp = sfmmu_hat_enter(sfmmup);
1895 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id);
1915 hat_free_start(struct hat *sfmmup)
1917 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as));
1918 ASSERT(sfmmup != ksfmmup);
1920 sfmmup->sfmmu_free = 1;
1921 if (sfmmup->sfmmu_scdp != NULL) {
1922 sfmmu_leave_scd(sfmmup, 0);
1925 ASSERT(sfmmup->sfmmu_scdp == NULL);
1929 hat_free_end(struct hat *sfmmup)
1933 ASSERT(sfmmup->sfmmu_free == 1);
1934 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0);
1935 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0);
1936 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0);
1937 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0);
1938 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0);
1939 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0);
1941 if (sfmmup->sfmmu_rmstat) {
1942 hat_freestat(sfmmup->sfmmu_as, NULL);
1945 while (sfmmup->sfmmu_tsb != NULL) {
1946 struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next;
1947 sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb);
1948 sfmmup->sfmmu_tsb = next;
1951 if (sfmmup->sfmmu_srdp != NULL) {
1952 sfmmu_leave_srd(sfmmup);
1953 ASSERT(sfmmup->sfmmu_srdp == NULL);
1955 if (sfmmup->sfmmu_hmeregion_links[i] != NULL) {
1956 kmem_free(sfmmup->sfmmu_hmeregion_links[i],
1958 sfmmup->sfmmu_hmeregion_links[i] = NULL;
1962 sfmmu_free_sfmmu(sfmmup);
1966 ASSERT(sfmmup->sfmmu_hmeregion_links[i] == NULL);
1970 kmem_cache_free(sfmmuid_cache, sfmmup);
1989 hat_swapout(struct hat *sfmmup)
2020 ASSERT(sfmmup != KHATID);
2028 if ((hmeblkp->hblk_tag.htag_id == sfmmup) &&
2031 (void) sfmmu_hblk_unload(sfmmup, hmeblkp,
2054 hatlockp = sfmmu_hat_enter(sfmmup);
2056 sfmmu_invalidate_ctx(sfmmup);
2072 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
2077 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPPED);
2079 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL;
2673 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp,
2676 ASSERT(sfmmup == ksfmmup);
2677 (void) sfmmu_tteload_array(sfmmup, ttep, vaddr, &pp, flags,
2686 sfmmu_mod_tsb(sfmmu_t *sfmmup, caddr_t vaddr, tte_t *ttep, int ttesz)
2696 if (sfmmup == ksfmmup) { /* No support for 32/256M ksfmmu pages */
2709 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz);
2767 sfmmu_unload_tsb_range(sfmmu_t *sfmmup, caddr_t start, caddr_t end, int ttesz)
2786 if (sfmmup == ksfmmup) {
2799 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz);
2876 sfmmu_tteload_array(sfmmu_t *sfmmup, tte_t *ttep, caddr_t vaddr,
2893 hmebp = sfmmu_tteload_acquire_hashbucket(sfmmup, vaddr, size, rid);
2899 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags,
2906 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags,
2921 sfmmu_tteload_acquire_hashbucket(sfmmu_t *sfmmup, caddr_t vaddr, int size,
2926 void *htagid = sfmmutohtagid(sfmmup, rid);
2941 * hmeblk doesn't exists for the [sfmmup, vaddr & size] signature, a hmeblk is
2945 sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp,
2952 SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size));
2954 hblktag.htag_id = sfmmutohtagid(sfmmup, rid);
2981 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size,
3006 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp);
3038 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep,
3110 SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size));
3159 sfmmu_vac_conflict(sfmmup, vaddr, pp);
3221 if ((sfmmup == KHATID) &&
3235 atomic_inc_ulong(&sfmmup->sfmmu_ttecnt[size]);
3237 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
3248 myflt = (astosfmmu(curthread->t_procp->p_as) == sfmmup);
3250 sfmmup != ksfmmup) {
3253 if (!(sfmmup->sfmmu_tteflags & tteflag)) {
3254 hatlockp = sfmmu_hat_enter(sfmmup);
3255 sfmmup->sfmmu_tteflags |= tteflag;
3258 } else if (!(sfmmup->sfmmu_rtteflags & tteflag)) {
3259 hatlockp = sfmmu_hat_enter(sfmmup);
3260 sfmmup->sfmmu_rtteflags |= tteflag;
3288 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) {
3289 hatlockp = sfmmu_hat_enter(sfmmup);
3290 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG);
3308 sfmmu_ttesync(sfmmup, vaddr, &tteold, pp);
3318 sfmmup->sfmmu_srdp->srd_hmergnp[rid], hmeblkp, 1);
3322 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0);
3323 xt_sync(sfmmup->sfmmu_cpusran);
3335 hatlockp = sfmmu_hat_enter(sfmmup);
3340 scdp = sfmmup->sfmmu_scdp;
3343 sfmmu_load_tsb(sfmmup, vaddr, &sfhme->hme_tte,
3588 sfmmu_shadow_hcreate(sfmmu_t *sfmmup, caddr_t vaddr, int ttesz, uint_t flags)
3596 ASSERT(sfmmup != KHATID);
3601 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0);
3602 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0);
3611 hblktag.htag_id = sfmmup;
3616 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift);
3623 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size,
3668 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp,
3690 sfmmu_free_hblks(sfmmup, addr, endaddr, hashno);
3696 sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr,
3706 hblktag.htag_id = sfmmup;
3714 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
3726 sfmmu_shadow_hcleanup(sfmmup,
3933 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len)
3941 ASSERT(sfmmup != NULL);
3943 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as));
3946 hblktag.htag_id = sfmmup;
3960 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
3986 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
3993 "addr %p hat %p", (void *)addr, (void *)sfmmup);
4003 hat_unlock_region(struct hat *sfmmup, caddr_t addr, size_t len,
4020 hat_unlock(sfmmup, addr, len);
4024 ASSERT(sfmmup != NULL);
4025 ASSERT(sfmmup != ksfmmup);
4027 srdp = sfmmup->sfmmu_srdp;
4034 SFMMU_VALIDATE_HMERID(sfmmup, rid, addr, len);
4076 "addr %p hat %p", (void *)va, (void *)sfmmup);
4730 hat_probe(struct hat *sfmmup, caddr_t addr)
4735 ASSERT(sfmmup != NULL);
4737 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as));
4739 if (sfmmup == ksfmmup) {
4740 while ((pfn = sfmmu_vatopfn(addr, sfmmup, &tte))
4742 sfmmu_vatopfn_suspended(addr, sfmmup, &tte);
4745 pfn = sfmmu_uvatopfn(addr, sfmmup, NULL);
4755 hat_getpagesize(struct hat *sfmmup, caddr_t addr)
4759 if (sfmmup == ksfmmup) {
4760 if (sfmmu_vatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4764 if (sfmmu_uvatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4774 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr)
4778 if (sfmmup == ksfmmup) {
4779 if (sfmmu_vatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4783 if (sfmmu_uvatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4833 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr,
4846 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as));
4857 hblktag.htag_id = sfmmup;
4859 DEMAP_RANGE_INIT(sfmmup, &dmr);
4865 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
4877 ASSERT(sfmmup != ksfmmup);
4882 addr = sfmmu_hblk_chgattr(sfmmup,
4891 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
4901 if (sfmmup == ksfmmup)
4915 cpuset = sfmmup->sfmmu_cpusran;
4929 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
4963 check_exec = (sfmmup != ksfmmup) &&
4964 AS_TYPE_64BIT(sfmmup->sfmmu_as) &&
5026 sfmmu_ttesync(sfmmup, addr, &tte, pp);
5040 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
5163 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot)
5176 ASSERT(sfmmup->sfmmu_as != NULL);
5186 hblktag.htag_id = sfmmup;
5188 DEMAP_RANGE_INIT(sfmmup, &dmr);
5194 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
5206 ASSERT(sfmmup != ksfmmup);
5211 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp,
5220 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
5229 if (sfmmup == ksfmmup)
5243 cpuset = sfmmup->sfmmu_cpusran;
5257 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
5289 check_exec = (sfmmup != ksfmmup) &&
5290 AS_TYPE_64BIT(sfmmup->sfmmu_as) &&
5354 sfmmu_ttesync(sfmmup, addr, &tte, pp);
5368 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
5449 struct hat *sfmmup,
5471 if (sfmmup->sfmmu_free) {
5475 DEMAP_RANGE_INIT(sfmmup, dmrp);
5493 if (hmeblkp->hblk_tag.htag_id != sfmmup ||
5507 (void) sfmmu_hblk_unload(sfmmup, hmeblkp,
5542 cpuset = sfmmup->sfmmu_cpusran;
5563 cpuset = sfmmup->sfmmu_cpusran;
5576 if (!sfmmup->sfmmu_free)
5577 sfmmu_check_page_sizes(sfmmup, 0);
5586 #define ISSEGKMAP(sfmmup, addr) (sfmmup == ksfmmup && \
5592 struct hat *sfmmup,
5608 int issegkmap = ISSEGKMAP(sfmmup, addr);
5611 ASSERT(sfmmup->sfmmu_as != NULL);
5613 ASSERT((sfmmup == ksfmmup) || (flags & HAT_UNLOAD_OTHER) || \
5614 AS_LOCK_HELD(sfmmup->sfmmu_as));
5616 ASSERT(sfmmup != NULL);
5630 if (sfmmup != KHATID && (len >> TTE_PAGE_SHIFT(TTE4M)) > UHMEHASH_SZ) {
5631 hat_unload_large_virtual(sfmmup, addr, len, flags, callback);
5641 if (sfmmup->sfmmu_free) {
5645 DEMAP_RANGE_INIT(sfmmup, dmrp);
5649 hblktag.htag_id = sfmmup;
5667 if (sfmmup == KHATID) {
5682 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
5784 ASSERT(sfmmup != KHATID);
5802 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr,
5822 cpuset = sfmmup->sfmmu_cpusran;
5863 cpuset = sfmmup->sfmmu_cpusran;
5877 if (!sfmmup->sfmmu_free)
5878 sfmmu_check_page_sizes(sfmmup, 0);
5886 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags)
5888 hat_unload_callback(sfmmup, addr, len, flags, NULL);
5918 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
5932 ASSERT(sfmmup != NULL || hmeblkp->hblk_shared);
5933 ASSERT(sfmmup == NULL || !hmeblkp->hblk_shared);
6019 sfmmu_ttesync(sfmmup, addr, &tte, pp);
6065 ASSERT(sfmmup != NULL);
6067 sfmmu_tlb_demap(addr, sfmmup, hmeblkp,
6068 sfmmup->sfmmu_free, 0);
6183 if (ttecnt > 0 && sfmmup != NULL) {
6184 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt);
6195 hat_flush_range(struct hat *sfmmup, caddr_t va, size_t size)
6201 sz = hat_getpagesize(sfmmup, va);
6206 vtag_flushpage(va, (uint64_t)sfmmup);
6218 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag)
6227 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as));
6235 hblktag.htag_id = sfmmup;
6250 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
6262 ASSERT(sfmmup != ksfmmup);
6267 addr = sfmmu_hblk_sync(sfmmup, hmeblkp,
6276 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
6285 if (sfmmup == ksfmmup)
6297 cpuset = sfmmup->sfmmu_cpusran;
6302 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
6353 sfmmu_tlb_demap(addr, sfmmup,
6357 sfmmu_ttesync(sfmmup, addr, &tte, pp);
6375 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp)
6399 if (sfmmup != NULL && sfmmup->sfmmu_rmstat) {
6404 hat_setstat(sfmmup->sfmmu_as, vaddr, MMU_PAGESIZE, rm);
6612 sfmmu_t *sfmmup;
6663 sfmmup = hblktosfmmu(hmeblkp);
6664 ASSERT(sfmmup == ksfmmup);
6674 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
7151 sfmmu_t *sfmmup;
7172 sfmmup = hblktosfmmu(hmeblkp);
7208 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
7221 sfmmu_ttesync(sfmmup, addr, &tte, pp);
7222 atomic_dec_ulong(&sfmmup->sfmmu_ttecnt[ttesz]);
7242 if (sfmmup->sfmmu_ismhat) {
7251 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp,
7258 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
7259 cpuset = sfmmup->sfmmu_cpusran;
7495 sfmmu_t *sfmmup;
7512 sfmmup = hblktosfmmu(hmeblkp);
7530 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
7543 sfmmu_tlb_demap(addr, sfmmup, hmeblkp,
7545 cpuset = sfmmup->sfmmu_cpusran;
7549 sfmmu_ttesync(hmeblkp->hblk_shared ? NULL : sfmmup, addr,
7569 sfmmu_t *sfmmup;
7583 sfmmup = hblktosfmmu(hmeblkp);
7601 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
7613 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
7614 cpuset = sfmmup->sfmmu_cpusran;
7873 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup, tte_t *ttep)
7890 sfmmu_t *sv_sfmmup = sfmmup;
7900 ASSERT(sfmmup != ksfmmup);
7905 ism_blkp = sfmmup->sfmmu_iblk;
7907 sfmmu_ismhat_enter(sfmmup, 0);
7908 locked_hatid = sfmmup;
7915 sfmmup = ism_hatid = ism_map[i].imap_ismhat;
7927 hblktag.htag_id = sfmmup;
7933 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift);
7951 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt));
8301 ism_tsb_entries(sfmmu_t *sfmmup, int szc)
8303 ism_blk_t *ism_blkp = sfmmup->sfmmu_iblk;
8311 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
8312 scdp = sfmmup->sfmmu_scdp;
8319 rid < sfmmup->sfmmu_srdp->srd_next_ismrid);
8323 /* ISM is in sfmmup's SCD */
8333 sfmmup->sfmmu_ismttecnt[szc] = npgs;
8334 sfmmup->sfmmu_scdismttecnt[szc] = npgs_scd;
8456 hat_share(struct hat *sfmmup, caddr_t addr,
8478 ASSERT(ism_hatid != NULL && sfmmup != NULL);
8505 sfmmu_ismhat_enter(sfmmup, 0);
8510 if (sfmmup->sfmmu_iblk == NULL) {
8511 sfmmup->sfmmu_iblk = new_iblk;
8515 sfmmup->sfmmu_ismblkpa = va_to_pa((caddr_t)new_iblk);
8524 ism_blkp = sfmmup->sfmmu_iblk;
8550 ism_blkp = sfmmup->sfmmu_iblk;
8574 ism_ment->iment_hat = sfmmup;
8598 * After calling hat_join_region, sfmmup may join a new SCD or
8600 * shrink the sfmmup's private tsb size, i.e., pass shrink to
8603 old_scdp = sfmmup->sfmmu_scdp;
8605 rcookie = hat_join_region(sfmmup, addr, len, (void *)ism_hatid, 0,
8611 * Update our counters for this sfmmup's ism mappings.
8615 (void) ism_tsb_entries(sfmmup, i);
8630 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_ISM)) {
8631 hatlockp = sfmmu_hat_enter(sfmmup);
8632 SFMMU_FLAGS_SET(sfmmup, HAT_256M_ISM);
8637 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_ISM)) {
8638 hatlockp = sfmmu_hat_enter(sfmmup);
8639 SFMMU_FLAGS_SET(sfmmup, HAT_32M_ISM);
8654 hatlockp = sfmmu_hat_enter(sfmmup);
8655 sfmmu_sync_mmustate(sfmmup);
8659 sfmmu_ismhat_exit(sfmmup, 0);
8670 if (sfmmup->sfmmu_scdp != NULL && old_scdp != sfmmup->sfmmu_scdp) {
8671 sfmmu_check_page_sizes(sfmmup, 0);
8673 sfmmu_check_page_sizes(sfmmup, 1);
8684 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc)
8700 ASSERT(sfmmup != NULL);
8701 ASSERT(sfmmup != ksfmmup);
8703 ASSERT(sfmmup->sfmmu_as != NULL);
8713 sfmmu_ismhat_enter(sfmmup, 0);
8728 ism_blkp = sfmmup->sfmmu_iblk;
8749 * After hat_leave_region, the sfmmup may leave SCD,
8753 old_scdp = sfmmup->sfmmu_scdp;
8758 hat_leave_region(sfmmup, (void *)((uint64_t)ism_rid),
8767 hatlockp = sfmmu_hat_enter(sfmmup);
8768 sfmmu_invalidate_ctx(sfmmup);
8815 if (!sfmmup->sfmmu_free) {
8816 hatlockp = sfmmu_hat_enter(sfmmup);
8817 for (tsbinfo = sfmmup->sfmmu_tsb; tsbinfo != NULL;
8835 * Update our counters for this sfmmup's ism mappings.
8839 (void) ism_tsb_entries(sfmmup, i);
8842 sfmmu_ismhat_exit(sfmmup, 0);
8855 if (!sfmmup->sfmmu_free) {
8856 if (found && old_scdp != NULL && sfmmup->sfmmu_scdp == NULL) {
8857 sfmmu_check_page_sizes(sfmmup, 1);
8859 sfmmu_check_page_sizes(sfmmup, 0);
9462 sfmmu_t *sfmmup;
9505 sfmmup = hblktosfmmu(hmeblkp);
9511 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
9523 } else if (sfmmup->sfmmu_ismhat) {
9529 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp,
9532 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp,
9546 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
9557 } else if (sfmmup->sfmmu_ismhat) {
9563 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp,
9566 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1);
9610 sfmmu_get_ctx(sfmmu_t *sfmmup)
9616 ASSERT(sfmmu_hat_lock_held(sfmmup));
9617 ASSERT(sfmmup != ksfmmup);
9619 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ALLCTX_INVALID)) {
9620 sfmmu_setup_tsbinfo(sfmmup);
9621 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ALLCTX_INVALID);
9641 if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) {
9642 mmu_set_ctx_page_sizes(sfmmup);
9655 if (sfmmu_alloc_ctx(sfmmup, 1, CPU, SFMMU_PRIVATE) &&
9656 sfmmup->sfmmu_scdp != NULL) {
9657 sf_scd_t *scdp = sfmmup->sfmmu_scdp;
9664 sfmmu_load_mmustate(sfmmup);
9763 sfmmu_sync_mmustate(sfmmu_t *sfmmup)
9768 ASSERT(sfmmup != ksfmmup);
9769 ASSERT(sfmmu_hat_lock_held(sfmmup));
9774 * We check whether the pass'ed-in sfmmup is the same as the
9778 if ((sfmmup == curthread->t_procp->p_as->a_hat) &&
9781 cnum = sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum;
9794 sfmmu_load_mmustate(sfmmup);
9801 * or when sfmmup is not the same as the curproc.
9803 sfmmu_invalidate_ctx(sfmmup);
9826 sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc,
9834 ASSERT(sfmmup != ksfmmup);
9835 ASSERT(sfmmup->sfmmu_ismhat == 0);
9836 ASSERT(sfmmu_hat_lock_held(sfmmup));
9839 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_BUSY))
9847 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb;
9853 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
9862 SFMMU_FLAGS_SET(sfmmup, HAT_BUSY);
9878 tte_sz_mask, flags, sfmmup) &&
9883 tte_sz_mask, flags, sfmmup))) {
9884 (void) sfmmu_hat_enter(sfmmup);
9887 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY);
9890 (void) sfmmu_hat_enter(sfmmup);
9898 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
9900 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY);
9903 (void) sfmmu_hat_enter(sfmmup);
9909 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb;
9934 sfmmu_invalidate_ctx(sfmmup);
9943 ASSERT(curthread->t_procp->p_as->a_hat == sfmmup);
9954 ASSERT(sfmmup->sfmmu_ctxs[i].cnum == INVALID_CONTEXT);
9963 sfmmup->sfmmu_tsb = new_tsbinfo;
9973 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY);
9988 (void) sfmmu_hat_enter(sfmmup);
9999 sfmmu_reprog_pgsz_arr(sfmmu_t *sfmmup, uint8_t *tmp_pgsz)
10004 hatlockp = sfmmu_hat_enter(sfmmup);
10008 sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i];
10012 sfmmu_invalidate_ctx(sfmmup);
10084 sfmmu_check_page_sizes(sfmmu_t *sfmmup, int growing)
10095 if (sfmmup == ksfmmup || sfmmup->sfmmu_ismhat != NULL)
10098 if (!SFMMU_LGPGS_INUSE(sfmmup) &&
10099 sfmmup->sfmmu_ttecnt[TTE8K] <= tsb_rss_factor)
10103 ttecnt[i] = sfmmup->sfmmu_ttecnt[i] +
10104 sfmmup->sfmmu_ismttecnt[i];
10109 mmu_check_page_sizes(sfmmup, ttecnt);
10129 tte8k_cnt += sfmmup->sfmmu_tsb0_4minflcnt;
10146 if (sfmmup->sfmmu_flags & HAT_4MTEXT_FLAG) {
10160 sfmmu_size_tsb(sfmmup, growing, tte8k_cnt, tte4m_cnt, sectsb_thresh);
10164 sfmmu_size_tsb(sfmmu_t *sfmmup, int growing, uint64_t tte8k_cnt,
10172 hatlockp = sfmmu_hat_enter(sfmmup);
10174 tsbinfop = sfmmup->sfmmu_tsb;
10189 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc,
10192 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc,
10195 tsbinfop = sfmmup->sfmmu_tsb;
10220 int allocflags = SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)?
10234 allocflags, sfmmup)) &&
10237 tsb_bits, allocflags, sfmmup)) &&
10239 tsb_bits, allocflags, sfmmup)) {
10243 hatlockp = sfmmu_hat_enter(sfmmup);
10245 sfmmu_invalidate_ctx(sfmmup);
10247 if (sfmmup->sfmmu_tsb->tsb_next == NULL) {
10248 sfmmup->sfmmu_tsb->tsb_next = newtsb;
10283 (void) sfmmu_replace_tsb(sfmmup, tsbinfop,
10287 (void) sfmmu_replace_tsb(sfmmup, tsbinfop,
10301 sfmmu_free_sfmmu(sfmmu_t *sfmmup)
10309 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0);
10310 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0);
10311 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0);
10312 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0);
10313 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0);
10314 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0);
10315 ASSERT(SF_RGNMAP_ISNULL(sfmmup));
10317 sfmmup->sfmmu_free = 0;
10318 sfmmup->sfmmu_ismhat = 0;
10320 blkp = sfmmup->sfmmu_iblk;
10321 sfmmup->sfmmu_iblk = NULL;
10807 sfmmu_hat_enter(sfmmu_t *sfmmup)
10811 if (sfmmup != ksfmmup) {
10812 hatlockp = TSB_HASH(sfmmup);
10820 sfmmu_hat_tryenter(sfmmu_t *sfmmup)
10824 if (sfmmup != ksfmmup) {
10825 hatlockp = TSB_HASH(sfmmup);
10857 sfmmu_hat_lock_held(sfmmu_t *sfmmup)
10859 ASSERT(sfmmup != ksfmmup);
10860 return (MUTEX_HELD(HATLOCK_MUTEXP(TSB_HASH(sfmmup))));
10875 sfmmu_ismhat_enter(sfmmu_t *sfmmup, int hatlock_held)
10881 hatlockp = sfmmu_hat_enter(sfmmup);
10882 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY))
10883 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp));
10884 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY);
10890 sfmmu_ismhat_exit(sfmmu_t *sfmmup, int hatlock_held)
10895 hatlockp = sfmmu_hat_enter(sfmmup);
10896 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
10897 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY);
10898 cv_broadcast(&sfmmup->sfmmu_tsb_cv);
10929 sfmmu_hblk_alloc(sfmmu_t *sfmmup, caddr_t vaddr,
10947 SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size));
10997 if (sfmmup != KHATID && !SFMMU_IS_SHMERID_VALID(rid)) {
11000 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr,
11004 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr,
11019 ASSERT(sfmmup != KHATID);
11021 srdp = sfmmup->sfmmu_srdp;
11072 sleep = ((sfmmup == KHATID) ? KM_NOSLEEP : KM_SLEEP);
11094 if (size == TTE8K && sfmmup != KHATID) {
11159 forcefree = (sfmmup == KHATID) ? 1 : 0;
11164 ASSERT(sfmmup != KHATID);
11452 sfmmu_t *sfmmup;
11455 sfmmup = hblktosfmmu(hmeblkp);
11456 if (hmeblkp->hblk_shared || sfmmup->sfmmu_ismhat) {
11459 DEMAP_RANGE_INIT(sfmmup, &dmr);
11460 (void) sfmmu_hblk_unload(sfmmup, hmeblkp,
11556 sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp)
11562 ASSERT(sfmmup != ksfmmup);
11563 ASSERT(sfmmu_hat_lock_held(sfmmup));
11565 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPIN)) {
11566 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp));
11569 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
11570 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPIN);
11575 ASSERT(sfmmup->sfmmu_tsb != NULL);
11581 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; tsbinfop = next) {
11584 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, tsbinfop->tsb_szc,
11594 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN);
11595 cv_broadcast(&sfmmup->sfmmu_tsb_cv);
11613 tsbinfop = sfmmup->sfmmu_tsb;
11622 hatlockp = sfmmu_hat_enter(sfmmup);
11629 tsbinfop = sfmmup->sfmmu_tsb;
11630 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, TSB_MIN_SZCODE,
11635 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN);
11636 cv_broadcast(&sfmmup->sfmmu_tsb_cv);
11695 sfmmu_t *sfmmup, *shsfmmup;
11706 sfmmup = astosfmmu(curthread->t_procp->p_as);
11713 ASSERT(sfmmup != ksfmmup && ctxtype != KCONTEXT);
11714 ASSERT(sfmmup->sfmmu_ismhat == 0);
11715 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED) ||
11739 hatlockp = sfmmu_hat_enter(sfmmup);
11742 ASSERT(sfmmup == tsbmp->usfmmup);
11743 if (((tsbmp->uhat_tteflags ^ sfmmup->sfmmu_tteflags) &
11745 ((tsbmp->uhat_rtteflags ^ sfmmup->sfmmu_rtteflags) &
11747 tsbmp->uhat_tteflags = sfmmup->sfmmu_tteflags;
11748 tsbmp->uhat_rtteflags = sfmmup->sfmmu_rtteflags;
11751 if (sfmmup->sfmmu_srdp != NULL) {
11752 ulong_t *sm = sfmmup->sfmmu_hmeregion_map.bitmap;
11760 sfmmup->sfmmu_srdp,
11797 hatlockp = sfmmu_hat_enter(sfmmup);
11799 if ((scdp = sfmmup->sfmmu_scdp) != NULL) {
11814 if (sfmmup->sfmmu_scdp == scdp &&
11820 hatlockp = sfmmu_hat_enter(sfmmup);
11826 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL;
11829 cv_wait(&sfmmup->sfmmu_tsb_cv,
11838 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) {
11839 cv_wait(&sfmmup->sfmmu_tsb_cv,
11845 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) {
11851 sfmmu_finish_join_scd(sfmmup);
11852 SFMMU_FLAGS_CLEAR(sfmmup, HAT_JOIN_SCD);
11861 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
11862 sfmmu_tsb_swapin(sfmmup, hatlockp);
11866 sfmmu_get_ctx(sfmmup);
11881 sfmmu_tsb_chk_reloc(sfmmu_t *sfmmup, hatlock_t *hatlockp)
11885 ASSERT(sfmmu_hat_lock_held(sfmmup));
11887 for (tp = sfmmup->sfmmu_tsb; tp != NULL; tp = tp->tsb_next) {
11889 cv_wait(&sfmmup->sfmmu_tsb_cv,
11947 sfmmu_t *sfmmup;
11963 sfmmup = rgnp->rgn_sfmmu_head;
11964 while (sfmmup != NULL) {
11966 hatlockp = sfmmu_hat_enter(sfmmup);
11977 if (!sfmmup->sfmmu_scdhat && sfmmup->sfmmu_scdp != NULL &&
11978 !SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) {
11979 scdp = sfmmup->sfmmu_scdp;
11988 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
11991 cpuset = sfmmup->sfmmu_cpusran;
11994 SFMMU_XCALL_STATS(sfmmup);
11996 (uint64_t)addr, (uint64_t)sfmmup);
11997 vtag_flushpage(addr, (uint64_t)sfmmup);
12006 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 0, 0);
12008 sfmmup = rlink->next;
12023 find_ism_rid(sfmmu_t *sfmmup, sfmmu_t *ism_sfmmup, caddr_t va, uint_t *ism_rid)
12031 ASSERT(sfmmu_hat_lock_held(sfmmup));
12033 ism_blkp = sfmmup->sfmmu_iblk;
12068 sfmmu_t *sfmmup;
12093 sfmmup = ment->iment_hat;
12107 if (!sfmmup->sfmmu_scdhat && sfmmup->sfmmu_scdp != NULL &&
12108 !SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD) &&
12109 !SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) {
12110 if (!find_ism_rid(sfmmup, ism_sfmmup, va,
12116 scdp = sfmmup->sfmmu_scdp;
12123 SFMMU_UNLOAD_TSB(va, sfmmup, hmeblkp, 1);
12125 cpuset = sfmmup->sfmmu_cpusran;
12128 SFMMU_XCALL_STATS(sfmmup);
12130 (uint64_t)sfmmup);
12131 vtag_flushpage(va, (uint64_t)sfmmup);
12143 SFMMU_XCALL_STATS(sfmmup);
12159 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp,
12195 hatlockp = sfmmu_hat_enter(sfmmup);
12202 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
12204 cpuset = sfmmup->sfmmu_cpusran;
12208 SFMMU_XCALL_STATS(sfmmup);
12211 (uint64_t)sfmmup);
12213 vtag_flushpage(addr, (uint64_t)sfmmup);
12230 cpuset = sfmmup->sfmmu_cpusran;
12234 SFMMU_XCALL_STATS(sfmmup);
12248 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp,
12266 hatlockp = sfmmu_hat_enter(sfmmup);
12267 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
12271 cpuset = sfmmup->sfmmu_cpusran;
12275 SFMMU_XCALL_STATS(sfmmup);
12276 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, (uint64_t)sfmmup);
12278 vtag_flushpage(addr, (uint64_t)sfmmup);
12299 sfmmu_t *sfmmup = dmrp->dmr_sfmmup;
12332 hatlockp = sfmmu_hat_enter(sfmmup);
12333 sfmmu_unload_tsb_range(sfmmup, addr, eaddr, TTE8K);
12341 if (sfmmup->sfmmu_free == 0) {
12354 * the low 6 bits of sfmmup. This is doable since pgcnt
12357 ASSERT(!((uint64_t)sfmmup & SFMMU_PGCNT_MASK));
12358 sfmmu_pgcnt = (uint64_t)sfmmup |
12370 hatlockp = sfmmu_hat_enter(sfmmup);
12373 cpuset = sfmmup->sfmmu_cpusran;
12377 SFMMU_XCALL_STATS(sfmmup);
12383 vtag_flushpage(addr, (uint64_t)sfmmup);
12418 sfmmu_invalidate_ctx(sfmmu_t *sfmmup)
12428 ASSERT(sfmmu_hat_lock_held(sfmmup));
12429 ASSERT(sfmmup != ksfmmup);
12438 currcnum = sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum;
12442 lock_set(&sfmmup->sfmmu_ctx_lock); /* acquire PP lock */
12446 cnum = sfmmup->sfmmu_ctxs[i].cnum;
12451 sfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT;
12454 lock_clear(&sfmmup->sfmmu_ctx_lock); /* release PP lock */
12458 cpuset = sfmmup->sfmmu_cpusran;
12462 SFMMU_XCALL_STATS(sfmmup);
12464 (uint64_t)sfmmup, INVALID_CONTEXT);
12482 SFMMU_FLAGS_SET(sfmmup, HAT_ALLCTX_INVALID);
12490 ASSERT(sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum == INVALID_CONTEXT);
12546 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu;
12554 * If tsb is a shared TSB with TSB_SHAREDCTX set, sfmmup must
12556 * If tsb is not shared, sfmmup is a private hat, then set
12559 hatlockp = sfmmu_hat_enter(sfmmup);
12563 sfmmu_tsb_inv_ctx(sfmmup);
12568 /* sfmmup is a shared hat */
12569 ASSERT(sfmmup->sfmmu_scdhat);
12570 scdp = sfmmup->sfmmu_scdp;
12574 sfmmup = scdp->scd_sf_list;
12575 while (sfmmup != NULL) {
12576 hatlockp = sfmmu_hat_enter(sfmmup);
12582 sfmmu_invalidate_ctx(sfmmup);
12584 sfmmup = sfmmup->sfmmu_scd_link.next;
12593 sfmmu_tsb_inv_ctx(sfmmu_t *sfmmup)
12597 ASSERT(sfmmu_hat_lock_held(sfmmup));
12629 sfmmu_invalidate_ctx(sfmmup);
12639 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu;
12644 hatlockp = sfmmu_hat_enter(sfmmup);
12666 cv_broadcast(&sfmmup->sfmmu_tsb_cv);
12679 uint_t flags, sfmmu_t *sfmmup)
12687 tsb_szc, flags, sfmmup)) != 0) {
12803 int tsbcode, uint_t flags, sfmmu_t *sfmmup)
12829 tsbinfo->tsb_sfmmu = sfmmup;
13602 hat_join_srd(struct hat *sfmmup, vnode_t *evp)
13608 ASSERT(sfmmup != ksfmmup);
13609 ASSERT(sfmmup->sfmmu_srdp == NULL);
13623 sfmmup->sfmmu_srdp = srdp;
13645 sfmmup->sfmmu_srdp = srdp;
13654 sfmmup->sfmmu_srdp = newsrdp;
13661 sfmmu_leave_srd(sfmmu_t *sfmmup)
13664 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
13674 ASSERT(sfmmup != ksfmmup);
13677 ASSERT(sfmmup->sfmmu_scdp == NULL);
13678 ASSERT(sfmmup->sfmmu_free == 1);
13680 sfmmup->sfmmu_srdp = NULL;
13809 hat_join_region(struct hat *sfmmup,
13819 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
13840 ASSERT(sfmmup != ksfmmup);
13841 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as));
13906 (sfmmup == astosfmmu(curthread->t_procp->p_as));
13908 sfmmu_link_to_hmeregion(sfmmup, rgnp);
13933 if (tteflag && !(sfmmup->sfmmu_rtteflags & tteflag)) {
13934 hatlockp = sfmmu_hat_enter(sfmmup);
13935 sfmmup->sfmmu_rtteflags |= tteflag;
13938 hatlockp = sfmmu_hat_enter(sfmmup);
13945 sfmmup->sfmmu_tsb0_4minflcnt +=
13951 atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc],
13957 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) {
13958 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG);
13969 sfmmu_check_page_sizes(sfmmup, 1);
13970 hatlockp = sfmmu_hat_enter(sfmmup);
13971 SF_RGNMAP_ADD(sfmmup->sfmmu_hmeregion_map, rid);
13980 (sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum
13985 ASSERT(sfmmup == tsbmp->usfmmup);
13998 hatlockp = sfmmu_hat_enter(sfmmup);
13999 SF_RGNMAP_ADD(sfmmup->sfmmu_ismregion_map, rid);
14005 sfmmu_find_scd(sfmmup);
14103 hat_leave_region(struct hat *sfmmup, hat_region_cookie_t rcookie, uint_t flags)
14105 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14121 ASSERT(sfmmup != ksfmmup);
14126 ASSERT(!sfmmup->sfmmu_free || sfmmup->sfmmu_scdp == NULL);
14144 ASSERT(AS_LOCK_HELD(sfmmup->sfmmu_as));
14146 if (sfmmup->sfmmu_free) {
14151 ASSERT(sfmmup->sfmmu_scdp == NULL);
14153 SF_RGNMAP_DEL(sfmmup->sfmmu_ismregion_map, rid);
14157 ASSERT(sfmmup->sfmmu_ttecnt[r_pgszc] >= rttecnt);
14159 atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc],
14162 SF_RGNMAP_DEL(sfmmup->sfmmu_hmeregion_map, rid);
14165 hatlockp = sfmmu_hat_enter(sfmmup);
14167 SF_RGNMAP_DEL(sfmmup->sfmmu_ismregion_map, rid);
14168 scdp = sfmmup->sfmmu_scdp;
14171 sfmmu_leave_scd(sfmmup, r_type);
14172 ASSERT(sfmmu_hat_lock_held(sfmmup));
14183 hatlockp = sfmmu_hat_enter(sfmmup);
14185 SF_RGNMAP_DEL(sfmmup->sfmmu_hmeregion_map, rid);
14194 scdp = sfmmup->sfmmu_scdp;
14197 sfmmu_leave_scd(sfmmup, r_type);
14198 ASSERT(sfmmu_hat_lock_held(sfmmup));
14200 sfmmu_invalidate_ctx(sfmmup);
14205 sfmmu_unload_tsb_range(sfmmup, r_saddr,
14219 ASSERT(sfmmup->sfmmu_tsb0_4minflcnt >=
14221 sfmmup->sfmmu_tsb0_4minflcnt -= rttecnt;
14226 ASSERT(sfmmup->sfmmu_ttecnt[r_pgszc] >= rttecnt);
14227 atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc], -rttecnt);
14230 if (scdp != NULL && sfmmup->sfmmu_scdp == NULL) {
14231 /* sfmmup left the scd, grow private tsb */
14232 sfmmu_check_page_sizes(sfmmup, 1);
14234 sfmmu_check_page_sizes(sfmmup, 0);
14239 sfmmu_unlink_from_hmeregion(sfmmup, rgnp);
14305 hat_dup_region(struct hat *sfmmup, hat_region_cookie_t rcookie)
14307 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14314 ASSERT(sfmmup != ksfmmup);
14331 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 1, 0);
14341 hrlink->prev = sfmmup;
14347 rgnp->rgn_sfmmu_head = sfmmup;
14352 atomic_add_long(&sfmmup->sfmmu_ttecnt[rgnp->rgn_pgszc], rttecnt);
14355 sfmmup->sfmmu_tsb0_4minflcnt +=
14362 SF_RGNMAP_ADD(sfmmup->sfmmu_hmeregion_map, rid);
14413 check_scd_sfmmu_list(sfmmu_t **headp, sfmmu_t *sfmmup, int onlist)
14416 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14420 if (sp == sfmmup) {
14425 "list 0x%p", (void *)sfmmup,
14432 (void *)sfmmup, (void *)*headp);
14438 #define check_scd_sfmmu_list(headp, sfmmup, onlist)
14445 sfmmu_from_scd_list(sfmmu_t **headp, sfmmu_t *sfmmup)
14447 ASSERT(sfmmup->sfmmu_srdp != NULL);
14448 check_scd_sfmmu_list(headp, sfmmup, 1);
14449 if (sfmmup->sfmmu_scd_link.prev != NULL) {
14450 ASSERT(*headp != sfmmup);
14451 sfmmup->sfmmu_scd_link.prev->sfmmu_scd_link.next =
14452 sfmmup->sfmmu_scd_link.next;
14454 ASSERT(*headp == sfmmup);
14455 *headp = sfmmup->sfmmu_scd_link.next;
14457 if (sfmmup->sfmmu_scd_link.next != NULL) {
14458 sfmmup->sfmmu_scd_link.next->sfmmu_scd_link.prev =
14459 sfmmup->sfmmu_scd_link.prev;
14468 sfmmu_to_scd_list(sfmmu_t **headp, sfmmu_t *sfmmup)
14470 check_scd_sfmmu_list(headp, sfmmup, 0);
14471 sfmmup->sfmmu_scd_link.prev = NULL;
14472 sfmmup->sfmmu_scd_link.next = *headp;
14474 (*headp)->sfmmu_scd_link.prev = sfmmup;
14475 *headp = sfmmup;
14644 sfmmu_link_to_hmeregion(sfmmu_t *sfmmup, sf_region_t *rgnp)
14655 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 1, 1);
14666 rgnp->rgn_sfmmu_head = sfmmup;
14674 hrlink->prev = sfmmup;
14680 rgnp->rgn_sfmmu_head = sfmmup;
14689 sfmmu_unlink_from_hmeregion(sfmmu_t *sfmmup, sf_region_t *rgnp)
14698 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 0, 0);
14701 if (rgnp->rgn_sfmmu_head == sfmmup) {
14718 ASSERT(nrlink->prev == sfmmup);
14730 ASSERT(prlink->next == sfmmup);
14746 ASSERT(nrlink->prev == sfmmup);
14924 sfmmu_join_scd(sf_scd_t *scdp, sfmmu_t *sfmmup)
14927 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14934 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as));
14936 if ((old_scdp = sfmmup->sfmmu_scdp) != NULL) {
14940 sfmmu_from_scd_list(&old_scdp->scd_sf_list, sfmmup);
14943 * sfmmup leaves the old scd. Update sfmmu_ttecnt to
14948 ASSERT(sfmmup->sfmmu_scdrttecnt[i] ==
14950 atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
14951 sfmmup->sfmmu_scdrttecnt[i]);
14959 sfmmu_to_scd_list(&scdp->scd_sf_list, sfmmup);
14963 hatlockp = sfmmu_hat_enter(sfmmup);
14969 SFMMU_FLAGS_SET(sfmmup, HAT_JOIN_SCD);
14971 sfmmu_invalidate_ctx(sfmmup);
14972 sfmmup->sfmmu_scdp = scdp;
14975 * Copy scd_rttecnt into sfmmup's sfmmu_scdrttecnt, and update
14979 sfmmup->sfmmu_scdrttecnt[i] = scdp->scd_rttecnt[i];
14980 ASSERT(sfmmup->sfmmu_ttecnt[i] >= scdp->scd_rttecnt[i]);
14981 atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
14982 -sfmmup->sfmmu_scdrttecnt[i]);
14986 sfmmup->sfmmu_tsb0_4minflcnt +=
14989 ASSERT(sfmmup->sfmmu_tsb0_4minflcnt >=
14991 sfmmup->sfmmu_tsb0_4minflcnt -= scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt;
15007 sfmmu_finish_join_scd(sfmmu_t *sfmmup)
15011 ASSERT(sfmmu_hat_lock_held(sfmmup));
15012 ASSERT(sfmmup->sfmmu_scdp != NULL);
15013 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD));
15014 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
15015 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ALLCTX_INVALID));
15017 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL;
15029 sfmmu_ism_hatflags(sfmmup, 1);
15039 sfmmu_find_scd(sfmmu_t *sfmmup)
15041 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
15046 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as));
15052 &sfmmup->sfmmu_region_map, ret);
15056 sfmmu_join_scd(scdp, sfmmup);
15068 &sfmmup->sfmmu_region_map, ret);
15080 if ((new_scdp = sfmmu_alloc_scd(srdp, &sfmmup->sfmmu_region_map)) ==
15101 sfmmu_join_scd(new_scdp, sfmmup);
15112 sfmmu_leave_scd(sfmmu_t *sfmmup, uchar_t r_type)
15114 sf_scd_t *scdp = sfmmup->sfmmu_scdp;
15115 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
15116 hatlock_t *hatlockp = TSB_HASH(sfmmup);
15122 if (sfmmup->sfmmu_free) {
15128 sfmmu_from_scd_list(&scdp->scd_sf_list, sfmmup);
15135 ASSERT(sfmmup->sfmmu_scdrttecnt[i] ==
15137 atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
15138 sfmmup->sfmmu_scdrttecnt[i]);
15139 sfmmup->sfmmu_scdrttecnt[i] = 0;
15141 sfmmup->sfmmu_scdp = NULL;
15148 SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
15150 ASSERT(!sfmmup->sfmmu_free);
15151 ASSERT(sfmmu_hat_lock_held(sfmmup));
15152 ASSERT(AS_LOCK_HELD(sfmmup->sfmmu_as));
15158 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY) &&
15159 sfmmup->sfmmu_scdp != NULL) {
15160 cv_wait(&sfmmup->sfmmu_tsb_cv,
15164 if (sfmmup->sfmmu_scdp == NULL) {
15168 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY);
15171 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) {
15172 SFMMU_FLAGS_CLEAR(sfmmup, HAT_JOIN_SCD);
15183 sfmmu_invalidate_ctx(sfmmup);
15187 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
15188 sfmmu_ism_hatflags(sfmmup, 0);
15192 * are in SCD before this sfmmup leaves the SCD.
15195 ASSERT(sfmmup->sfmmu_scdrttecnt[i] ==
15197 atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
15198 sfmmup->sfmmu_scdrttecnt[i]);
15199 sfmmup->sfmmu_scdrttecnt[i] = 0;
15201 sfmmup->sfmmu_ismttecnt[i] += sfmmup->sfmmu_scdismttecnt[i];
15202 sfmmup->sfmmu_scdismttecnt[i] = 0;
15205 sfmmup->sfmmu_tsb0_4minflcnt += scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt;
15208 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY);
15210 sfmmup->sfmmu_scdp = NULL;
15223 sfmmu_from_scd_list(&scdp->scd_sf_list, sfmmup);
15228 hatlockp = sfmmu_hat_enter(sfmmup);
15305 sfmmu_ism_hatflags(sfmmu_t *sfmmup, int addflag)
15307 sf_scd_t *scdp = sfmmup->sfmmu_scdp;
15312 ASSERT(sfmmup->sfmmu_iblk != NULL);
15318 ASSERT(sfmmu_hat_lock_held(sfmmup));
15320 ism_blkp = sfmmup->sfmmu_iblk;