Lines Matching defs:pVM
59 * @param pVM Pointer to the VM.
66 VMMDECL(int) pgmPhysHandlerRedirectToHC(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
68 NOREF(pVM); NOREF(uErrorCode); NOREF(pRegFrame); NOREF(pvFault); NOREF(GCPhysFault); NOREF(pvUser);
78 * @param pVM Pointer to the VM.
85 VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
90 PVMCPU pVCpu = VMMGetCpu(pVM);
107 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
127 rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
151 * @param pVM Pointer to the VM.
153 void pgmPhysInvalidRamRangeTlbs(PVM pVM)
155 pgmLock(pVM);
158 pVM->pgm.s.apRamRangesTlbR3[i] = NIL_RTR3PTR;
159 pVM->pgm.s.apRamRangesTlbR0[i] = NIL_RTR0PTR;
160 pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
162 pgmUnlock(pVM);
182 PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys)
184 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
186 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
192 pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
209 PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys)
211 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
214 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
220 pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
240 PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys)
242 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
244 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
250 pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
268 int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
270 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
272 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
278 pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
299 int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
301 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
303 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
309 pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
346 * @param pVM Pointer to the VM.
349 VMMDECL(bool) PGMPhysIsGCPhysValid(PVM pVM, RTGCPHYS GCPhys)
351 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
362 * @param pVM Pointer to the VM.
365 VMMDECL(bool) PGMPhysIsGCPhysNormal(PVM pVM, RTGCPHYS GCPhys)
367 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
382 * @param pVM Pointer to the VM.
386 VMMDECL(int) PGMPhysGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
388 pgmLock(pVM);
390 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
393 pgmUnlock(pVM);
401 * @param pVM Pointer to the VM.
403 void pgmPhysInvalidatePageMapTLB(PVM pVM)
405 pgmLock(pVM);
406 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushes);
409 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
411 pVM->pgm.s.PhysTlbHC.aEntries[i].GCPhys = NIL_RTGCPHYS;
412 pVM->pgm.s.PhysTlbHC.aEntries[i].pPage = 0;
413 pVM->pgm.s.PhysTlbHC.aEntries[i].pMap = 0;
414 pVM->pgm.s.PhysTlbHC.aEntries[i].pv = 0;
419 pgmUnlock(pVM);
426 * @param pVM Pointer to the VM.
429 void pgmPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys)
431 PGM_LOCK_ASSERT_OWNER(pVM);
433 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushEntry);
437 pVM->pgm.s.PhysTlbHC.aEntries[idx].GCPhys = NIL_RTGCPHYS;
438 pVM->pgm.s.PhysTlbHC.aEntries[idx].pPage = 0;
439 pVM->pgm.s.PhysTlbHC.aEntries[idx].pMap = 0;
440 pVM->pgm.s.PhysTlbHC.aEntries[idx].pv = 0;
443 PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
462 * @param pVM Pointer to the VM.
467 static int pgmPhysEnsureHandyPage(PVM pVM)
469 AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
475 if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
477 if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
484 if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
486 if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
490 pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) ));
492 int rc = PGMR3PhysAllocateHandyPages(pVM);
494 int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES, 0);
501 if (!pVM->pgm.s.cHandyPages)
506 Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
507 Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY));
510 REMR3NotifyFF(pVM);
513 VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
516 AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
517 && pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
518 ("%u\n", pVM->pgm.s.cHandyPages),
523 if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
524 VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
526 if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
528 Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
529 VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
549 * @param pVM Pointer to the VM.
564 int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
571 PGM_LOCK_ASSERT_OWNER(pVM);
579 if ( PGMIsUsingLargePages(pVM)
585 int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
589 rc = pgmPhysAllocLargePage(pVM, GCPhys);
594 PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT);
605 int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /*fFlushTLBs*/, &fFlushTLBs);
611 int rc2 = pgmPhysEnsureHandyPage(pVM);
615 PGM_INVL_ALL_VCPU_TLBS(pVM);
620 PGM_LOCK_ASSERT_OWNER(pVM);
624 uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
625 AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
626 Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_RTHCPHYS);
627 Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
628 Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
629 Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
637 const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
638 pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
644 pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
648 GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
649 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageReplaceShared));
650 pVM->pgm.s.cSharedPages--;
653 rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
658 Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
659 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
660 pVM->pgm.s.cZeroPages--;
666 pVM->pgm.s.cPrivatePages++;
667 PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
668 PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
669 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
670 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT);
671 pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
679 rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc);
683 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
689 PGM_INVL_ALL_VCPU_TLBS(pVM);
705 * @param pVM Pointer to the VM.
711 int pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys)
719 PGM_LOCK_ASSERT_OWNER(pVM);
720 Assert(PGMIsUsingLargePages(pVM));
723 int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
742 rc = pgmPhysGetPageEx(pVM, GCPhys, &pSubPage);
756 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
757 PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT);
765 rc = PGMR3PhysAllocateLargeHandyPage(pVM, GCPhysBase);
767 rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE, GCPhysBase);
772 pVM->pgm.s.cLargePages++;
779 PGMSetLargePageUsage(pVM, false);
794 * @param pVM Pointer to the VM.
798 int pgmPhysRecheckLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
800 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
814 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
821 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
835 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
839 PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE);
840 pVM->pgm.s.cLargePagesDisabled--;
855 * @param pVM Pointer to the VM.
860 void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage)
863 PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
864 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
865 Assert(pVM->pgm.s.cMonitoredPages > 0);
866 pVM->pgm.s.cMonitoredPages--;
867 pVM->pgm.s.cWrittenToPages++;
879 * @param pVM Pointer to the VM.
885 int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
887 PGM_LOCK_ASSERT_OWNER(pVM);
891 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
907 return pgmPhysAllocPage(pVM, pPage, GCPhys);
923 * @param pVM Pointer to the VM.
931 int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
936 PGM_LOCK_ASSERT_OWNER(pVM);
945 return pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
952 PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
955 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
960 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
965 pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
967 pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
971 int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
973 pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
976 int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
1002 * @param pVM Pointer to the VM.
1011 static int pgmPhysPageMapCommon(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
1013 PGM_LOCK_ASSERT_OWNER(pVM);
1022 Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
1023 pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
1038 AssertLogRelMsgReturn((uint8_t)(idMmio2 - 1U) < RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)),
1040 RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)), PGM_PAGE_GET_TYPE(pPage), GCPhys,
1043 PPGMMMIO2RANGE pMmio2Range = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[idMmio2 - 1];
1061 AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage),
1063 *ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1078 PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
1081 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
1087 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
1092 pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1096 pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
1101 int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
1103 pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1106 int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
1138 * @param pVM Pointer to the VM.
1147 int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1149 int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1154 int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
1173 * @param pVM Pointer to the VM.
1183 int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1187 return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
1202 * @param pVM Pointer to the VM.
1211 int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
1214 return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
1228 int pgmPhysPageLoadIntoTlb(PVM pVM, RTGCPHYS GCPhys)
1230 PGM_LOCK_ASSERT_OWNER(pVM);
1236 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1239 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1243 return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
1254 * @param pVM Pointer to the VM.
1259 int pgmPhysPageLoadIntoTlbWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1261 PGM_LOCK_ASSERT_OWNER(pVM);
1262 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1268 PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
1274 int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
1283 AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage));
1285 pTlbe->pv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1311 * @param pVM Pointer to the VM.
1319 int pgmPhysGCPhys2CCPtrInternalDepr(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1323 PGM_LOCK_ASSERT_OWNER(pVM);
1324 pVM->pgm.s.cDeprecatedPageLocks++;
1331 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1343 rc = pgmRZDynMapHCPageInlined(VMMGetCpu(pVM),
1352 rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1365 * @param pVM Pointer to the VM.
1370 DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
1380 pVM->pgm.s.cWriteLockedPages++;
1398 * @param pVM Pointer to the VM.
1403 DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
1413 pVM->pgm.s.cReadLockedPages++;
1440 * @param pVM Pointer to the VM.
1449 int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1453 PGM_LOCK_ASSERT_OWNER(pVM);
1460 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1472 PVMCPU pVCpu = VMMGetCpu(pVM);
1485 rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1488 pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
1504 * @param pVM Pointer to the VM.
1513 int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock)
1516 PGM_LOCK_ASSERT_OWNER(pVM);
1524 PVMCPU pVCpu = VMMGetCpu(pVM);
1537 int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1540 pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
1563 * @param pVM Pointer to the VM.
1580 VMMDECL(int) PGMPhysGCPhys2CCPtr(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1582 int rc = pgmLock(pVM);
1590 rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1594 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1599 PVMCPU pVCpu = VMMGetCpu(pVM);
1622 rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
1632 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1636 rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1644 pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
1650 pgmUnlock(pVM);
1667 * @param pVM Pointer to the VM.
1683 VMMDECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
1685 int rc = pgmLock(pVM);
1693 rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1700 PVMCPU pVCpu = VMMGetCpu(pVM);
1723 rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
1735 pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
1741 pgmUnlock(pVM);
1775 VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1779 rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1809 VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1813 rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1824 * @param pVM Pointer to the VM.
1827 VMMDECL(void) PGMPhysReleasePageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
1831 Assert(pLock->pVCpu == VMMGetCpu(pVM));
1844 pgmLock(pVM);
1853 Assert(pVM->pgm.s.cWriteLockedPages > 0);
1854 pVM->pgm.s.cWriteLockedPages--;
1861 PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
1862 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
1863 Assert(pVM->pgm.s.cMonitoredPages > 0);
1864 pVM->pgm.s.cMonitoredPages--;
1865 pVM->pgm.s.cWrittenToPages++;
1876 Assert(pVM->pgm.s.cReadLockedPages > 0);
1877 pVM->pgm.s.cReadLockedPages--;
1888 pgmUnlock(pVM);
1899 * @param pVM Pointer to the VM.
1904 void pgmPhysReleaseInternalPageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
1906 PGM_LOCK_ASSERT_OWNER(pVM);
1907 PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */
1922 * @param pVM Pointer to the VM.
1928 int pgmPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
1932 VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
1937 NOREF(pVM); NOREF(pR3Ptr);
1940 pgmLock(pVM);
1944 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
1946 rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
1948 pgmUnlock(pVM);
1967 * @param pVM Pointer to the VM.
1975 int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
1980 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
1982 rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
1988 int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
2026 PVM pVM = pVCpu->CTX_SUFF(pVM);
2030 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys | ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), pHCPhys);
2044 * @param pVM Pointer to the VM.
2051 static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE *pCache, RTGCPHYS GCPhys, uint8_t *pbR3)
2055 Assert(VM_IS_EMT(pVM));
2077 * @param pVM Pointer to the VM.
2083 static int pgmPhysReadHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb)
2095 int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck);
2115 pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2127 PGM_LOCK_ASSERT_OWNER(pVM);
2129 pgmUnlock(pVM);
2130 rc = pfnHandler(pVM, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, pvUser);
2131 pgmLock(pVM);
2133 pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2143 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2156 int rc2 = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iPage);
2174 rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, /*pVirt->CTX_SUFF(pvUser)*/ NULL);
2185 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2195 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2210 * @param pVM Pointer to the VM.
2215 VMMDECL(int) PGMPhysRead(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
2220 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysRead));
2221 STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysReadBytes), cbRead);
2223 pgmLock(pVM);
2228 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2252 int rc = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
2255 pgmUnlock(pVM);
2266 int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
2270 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2283 pgmUnlock(pVM);
2318 pgmUnlock(pVM);
2330 * @param pVM Pointer to the VM.
2336 static int pgmPhysWriteHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite)
2352 PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
2371 rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2380 PGM_LOCK_ASSERT_OWNER(pVM);
2382 pgmUnlock(pVM);
2383 rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2384 pgmLock(pVM);
2386 pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
2406 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2428 rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pCur, &iPage);
2444 rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2455 rc = pCur->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, /*pCur->CTX_SUFF(pvUser)*/ NULL);
2468 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2489 rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2515 rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iVirtPage);
2524 pVirtPhys = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers,
2549 pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2557 pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,
2602 PGM_LOCK_ASSERT_OWNER(pVM);
2604 pgmUnlock(pVM);
2605 rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2606 pgmLock(pVM);
2608 pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2619 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2640 rc = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, /*pCur->CTX_SUFF(pvUser)*/ NULL);
2649 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2673 PGM_LOCK_ASSERT_OWNER(pVM);
2675 pgmUnlock(pVM);
2676 rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2677 pgmLock(pVM);
2679 pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2693 int rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, /*pCur->CTX_SUFF(pvUser)*/ NULL);
2706 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2718 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2745 * @param pVM Pointer to the VM.
2750 VMMDECL(int) PGMPhysWrite(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
2752 AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()!\n"));
2756 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWrite));
2757 STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
2759 pgmLock(pVM);
2764 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2788 int rc = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
2791 pgmUnlock(pVM);
2802 int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
2807 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2818 pgmUnlock(pVM);
2849 pgmUnlock(pVM);
2859 * @param pVM Pointer to the VM.
2864 VMMDECL(int) PGMPhysSimpleReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
2875 int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2884 PGMPhysReleasePageMappingLock(pVM, &Lock);
2890 PGMPhysReleasePageMappingLock(pVM, &Lock);
2901 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2909 PGMPhysReleasePageMappingLock(pVM, &Lock);
2915 PGMPhysReleasePageMappingLock(pVM, &Lock);
2931 * @param pVM Pointer to the VM.
2936 VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
2949 int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2958 PGMPhysReleasePageMappingLock(pVM, &Lock);
2964 PGMPhysReleasePageMappingLock(pVM, &Lock);
2975 rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2983 PGMPhysReleasePageMappingLock(pVM, &Lock);
2989 PGMPhysReleasePageMappingLock(pVM, &Lock);
3012 PVM pVM = pVCpu->CTX_SUFF(pVM);
3021 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleRead));
3022 STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
3027 pgmLock(pVM);
3035 pgmUnlock(pVM);
3044 PGMPhysReleasePageMappingLock(pVM, &Lock);
3045 pgmUnlock(pVM);
3051 PGMPhysReleasePageMappingLock(pVM, &Lock);
3065 pgmUnlock(pVM);
3073 PGMPhysReleasePageMappingLock(pVM, &Lock);
3074 pgmUnlock(pVM);
3080 PGMPhysReleasePageMappingLock(pVM, &Lock);
3103 PVM pVM = pVCpu->CTX_SUFF(pVM);
3112 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWrite));
3113 STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
3127 PGMPhysReleasePageMappingLock(pVM, &Lock);
3133 PGMPhysReleasePageMappingLock(pVM, &Lock);
3152 PGMPhysReleasePageMappingLock(pVM, &Lock);
3158 PGMPhysReleasePageMappingLock(pVM, &Lock);
3183 PVM pVM = pVCpu->CTX_SUFF(pVM);
3205 PGMPhysReleasePageMappingLock(pVM, &Lock);
3212 PGMPhysReleasePageMappingLock(pVM, &Lock);
3232 PGMPhysReleasePageMappingLock(pVM, &Lock);
3239 PGMPhysReleasePageMappingLock(pVM, &Lock);
3267 PVM pVM = pVCpu->CTX_SUFF(pVM);
3295 return PGMPhysRead(pVM, GCPhys, pvDst, cb);
3319 rc = PGMPhysRead(pVM, GCPhys, pvDst, cbRead);
3324 return PGMPhysRead(pVM, GCPhys, pvDst, cb);
3355 PVM pVM = pVCpu->CTX_SUFF(pVM);
3387 return PGMPhysWrite(pVM, GCPhys, pvSrc, cb);
3415 rc = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite);
3420 rc = PGMPhysWrite(pVM, GCPhys, pvSrc, cb);
3456 PVM pVM = pVCpu->CTX_SUFF(pVM);
3486 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &PgMpLck);
3492 PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3532 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc1, &PgMpLck);
3537 PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3548 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc2, &PgMpLck);
3553 PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3639 PVM pVM = pVCpu->CTX_SUFF(pVM);
3665 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &Lock);
3672 PGMPhysReleasePageMappingLock(pVM, &Lock);
3714 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc, &Lock);
3721 PGMPhysReleasePageMappingLock(pVM, &Lock);
3733 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc, &Lock);
3738 PGMPhysReleasePageMappingLock(pVM, &Lock);
3833 PVM pVM = pVCpu->CTX_SUFF(pVM);
3863 rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, &pvDst, &Lock);
3870 PGMPhysReleasePageMappingLock(pVM, &Lock);
3916 rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys1, &pvDst, &Lock);
3923 PGMPhysReleasePageMappingLock(pVM, &Lock);
3935 rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys2, &pvDst, &Lock);
3940 PGMPhysReleasePageMappingLock(pVM, &Lock);
4014 * @param pVM Pointer to the VM.
4017 VMMDECL(PGMPAGETYPE) PGMPhysGetPageType(PVM pVM, RTGCPHYS GCPhys)
4019 pgmLock(pVM);
4020 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4022 pgmUnlock(pVM);
4042 * @param pVM Pointer to the cross context VM structure.
4055 VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers,
4060 pgmLock(pVM);
4064 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4102 rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4122 rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
4127 pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
4129 pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
4143 pgmUnlock(pVM);
4160 * @param pVM Pointer to the VM.
4170 VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVM pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers)
4172 pgmLock(pVM);
4173 PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4177 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4202 pgmUnlock(pVM);