Lines Matching defs:pGMM
596 PGMM pGMM;
627 PGMM pGMM;
645 PGMM pGMM;
662 * @param pGMM The name of the pGMM variable.
666 #define GMM_GET_VALID_INSTANCE(pGMM, rc) \
668 (pGMM) = g_pGMM; \
669 AssertPtrReturn((pGMM), (rc)); \
670 AssertMsgReturn((pGMM)->u32Magic == GMM_MAGIC, ("%p - %#x\n", (pGMM), (pGMM)->u32Magic), (rc)); \
678 * @param pGMM The name of the pGMM variable.
680 #define GMM_GET_VALID_INSTANCE_VOID(pGMM) \
682 (pGMM) = g_pGMM; \
683 AssertPtrReturnVoid((pGMM)); \
684 AssertMsgReturnVoid((pGMM)->u32Magic == GMM_MAGIC, ("%p - %#x\n", (pGMM), (pGMM)->u32Magic)); \
694 * @param pGMM The name of the pGMM variable.
697 # define GMM_CHECK_SANITY_UPON_ENTERING(pGMM) (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)
699 # define GMM_CHECK_SANITY_UPON_ENTERING(pGMM) (true)
708 * @param pGMM The name of the pGMM variable.
711 # define GMM_CHECK_SANITY_UPON_LEAVING(pGMM) (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)
713 # define GMM_CHECK_SANITY_UPON_LEAVING(pGMM) (true)
722 * @param pGMM The name of the pGMM variable.
725 # define GMM_CHECK_SANITY_IN_LOOPS(pGMM) (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)
727 # define GMM_CHECK_SANITY_IN_LOOPS(pGMM) (true)
735 static bool gmmR0CleanupVMScanChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
738 DECLINLINE(void) gmmR0SelectSetAndLinkChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
740 static uint32_t gmmR0SanityCheck(PGMM pGMM, const char *pszFunction, unsigned uLineNo);
742 static bool gmmR0FreeChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem);
743 DECLINLINE(void) gmmR0FreePrivatePage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage);
744 DECLINLINE(void) gmmR0FreeSharedPage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage);
745 static int gmmR0UnmapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
747 static void gmmR0SharedModuleCleanup(PGMM pGMM, PGVM pGVM);
749 static uint32_t gmmR0StrictPageChecksum(PGMM pGMM, PGVM pGVM, uint32_t idPage);
770 PGMM pGMM = (PGMM)RTMemAllocZ(sizeof(*pGMM));
771 if (!pGMM)
774 pGMM->u32Magic = GMM_MAGIC;
775 for (unsigned i = 0; i < RT_ELEMENTS(pGMM->ChunkTLB.aEntries); i++)
776 pGMM->ChunkTLB.aEntries[i].idChunk = NIL_GMM_CHUNKID;
777 RTListInit(&pGMM->ChunkList);
778 ASMBitSet(&pGMM->bmChunkId[0], NIL_GMM_CHUNKID);
781 int rc = RTCritSectInit(&pGMM->GiantCritSect);
783 int rc = RTSemFastMutexCreate(&pGMM->hMtx);
788 for (iMtx = 0; iMtx < RT_ELEMENTS(pGMM->aChunkMtx); iMtx++)
790 rc = RTSemFastMutexCreate(&pGMM->aChunkMtx[iMtx].hMtx);
808 pGMM->fLegacyAllocationMode = pGMM->fBoundMemoryMode = true;
813 pGMM->fLegacyAllocationMode = false;
817 pGMM->fBoundMemoryMode = true;
819 pGMM->fBoundMemoryMode = false;
822 pGMM->fLegacyAllocationMode = true;
823 pGMM->fBoundMemoryMode = true;
830 pGMM->cMaxPages = UINT32_MAX; /** @todo IPRT function for query ram size and such. */
832 g_pGMM = pGMM;
833 LogFlow(("GMMInit: pGMM=%p fLegacyAllocationMode=%RTbool fBoundMemoryMode=%RTbool\n", pGMM, pGMM->fLegacyAllocationMode, pGMM->fBoundMemoryMode));
841 RTSemFastMutexDestroy(pGMM->aChunkMtx[iMtx].hMtx);
843 RTCritSectDelete(&pGMM->GiantCritSect);
845 RTSemFastMutexDestroy(pGMM->hMtx);
849 pGMM->u32Magic = 0;
850 RTMemFree(pGMM);
866 PGMM pGMM = g_pGMM;
867 if (!VALID_PTR(pGMM))
869 if (pGMM->u32Magic != GMM_MAGIC)
871 SUPR0Printf("GMMR0Term: u32Magic=%#x\n", pGMM->u32Magic);
880 pGMM->u32Magic = ~GMM_MAGIC;
882 RTCritSectDelete(&pGMM->GiantCritSect);
884 RTSemFastMutexDestroy(pGMM->hMtx);
885 pGMM->hMtx = NIL_RTSEMFASTMUTEX;
889 RTAvlU32Destroy(&pGMM->pChunks, gmmR0TermDestroyChunk, pGMM);
892 for (unsigned iMtx = 0; iMtx < RT_ELEMENTS(pGMM->aChunkMtx); iMtx++)
894 Assert(pGMM->aChunkMtx[iMtx].cUsers == 0);
895 RTSemFastMutexDestroy(pGMM->aChunkMtx[iMtx].hMtx);
896 pGMM->aChunkMtx[iMtx].hMtx = NIL_RTSEMFASTMUTEX;
900 RTMemFree(pGMM);
962 * @param pGMM Pointer to the GMM instance.
964 static int gmmR0MutexAcquire(PGMM pGMM)
966 ASMAtomicIncU32(&pGMM->cMtxContenders);
968 int rc = RTCritSectEnter(&pGMM->GiantCritSect);
970 int rc = RTSemFastMutexRequest(pGMM->hMtx);
972 ASMAtomicDecU32(&pGMM->cMtxContenders);
975 pGMM->hMtxOwner = RTThreadNativeSelf();
985 * @param pGMM Pointer to the GMM instance.
987 static int gmmR0MutexRelease(PGMM pGMM)
990 pGMM->hMtxOwner = NIL_RTNATIVETHREAD;
993 int rc = RTCritSectLeave(&pGMM->GiantCritSect);
995 int rc = RTSemFastMutexRelease(pGMM->hMtx);
1007 * @param pGMM Pointer to the GMM instance.
1011 static bool gmmR0MutexYield(PGMM pGMM, uint64_t *puLockNanoTS)
1016 if (ASMAtomicReadU32(&pGMM->cMtxContenders) == 0)
1030 pGMM->hMtxOwner = NIL_RTNATIVETHREAD;
1032 ASMAtomicIncU32(&pGMM->cMtxContenders);
1034 int rc1 = RTCritSectLeave(&pGMM->GiantCritSect); AssertRC(rc1);
1036 int rc1 = RTSemFastMutexRelease(pGMM->hMtx); AssertRC(rc1);
1042 int rc2 = RTCritSectEnter(&pGMM->GiantCritSect); AssertRC(rc2);
1044 int rc2 = RTSemFastMutexRequest(pGMM->hMtx); AssertRC(rc2);
1047 ASMAtomicDecU32(&pGMM->cMtxContenders);
1049 pGMM->hMtxOwner = RTThreadNativeSelf();
1066 * @param pGMM Pointer to the GMM instance.
1070 static int gmmR0ChunkMutexAcquire(PGMMR0CHUNKMTXSTATE pMtxState, PGMM pGMM, PGMMCHUNK pChunk, uint32_t fFlags)
1073 Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
1075 pMtxState->pGMM = pGMM;
1081 Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
1085 iChunkMtx = pGMM->iNextChunkMtx++;
1086 iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
1089 if (pGMM->aChunkMtx[iChunkMtx].cUsers)
1091 iChunkMtx = pGMM->iNextChunkMtx++;
1092 iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
1093 if (pGMM->aChunkMtx[iChunkMtx].cUsers)
1095 iChunkMtx = pGMM->iNextChunkMtx++;
1096 iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
1097 if (pGMM->aChunkMtx[iChunkMtx].cUsers)
1099 iChunkMtx = pGMM->iNextChunkMtx++;
1100 iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
1107 AssertCompile(RT_ELEMENTS(pGMM->aChunkMtx) < UINT8_MAX);
1109 ASMAtomicIncU32(&pGMM->aChunkMtx[iChunkMtx].cUsers);
1118 gmmR0MutexRelease(pGMM);
1124 int rc = RTSemFastMutexRequest(pGMM->aChunkMtx[iChunkMtx].hMtx);
1134 * @param pGMM Pointer to the GMM instance.
1143 PGMM pGMM = pMtxState->pGMM;
1148 int rc = RTSemFastMutexRelease(pGMM->aChunkMtx[pMtxState->iChunkMtx].hMtx);
1151 rc = gmmR0MutexAcquire(pGMM);
1153 Assert((pMtxState->fFlags != GMMR0CHUNK_MTX_DROP_GIANT) == (pGMM->hMtxOwner == RTThreadNativeSelf()));
1159 if ( ASMAtomicDecU32(&pGMM->aChunkMtx[pMtxState->iChunkMtx].cUsers) == 0
1167 rc = gmmR0MutexAcquire(pGMM);
1170 if (pGMM->aChunkMtx[pMtxState->iChunkMtx].cUsers == 0)
1172 rc = gmmR0MutexRelease(pGMM);
1177 pMtxState->pGMM = NULL;
1196 Assert(pMtxState->pGMM->hMtxOwner == RTThreadNativeSelf());
1200 return gmmR0MutexRelease(pMtxState->pGMM);
1229 PGMM pGMM;
1230 GMM_GET_VALID_INSTANCE_VOID(pGMM);
1236 gmmR0SharedModuleCleanup(pGMM, pGVM);
1239 gmmR0MutexAcquire(pGMM);
1241 GMM_CHECK_SANITY_UPON_ENTERING(pGMM);
1257 Assert(pGMM->cRegisteredVMs);
1258 pGMM->cRegisteredVMs--;
1274 RTListForEachReverse(&pGMM->ChunkList, pChunk, GMMCHUNK, ListNode)
1276 uint32_t const cFreeChunksOld = pGMM->cFreedChunks;
1277 if ( ( !pGMM->fBoundMemoryMode
1279 && gmmR0CleanupVMScanChunk(pGMM, pGVM, pChunk))
1289 gmmR0MutexYield(pGMM, &uLockNanoTS);
1293 if (pGMM->cFreedChunks != cFreeChunksOld)
1304 pGMM->cAllocatedPages -= cPrivatePages;
1309 PGMMCHUNKFREESET pPrivateSet = pGMM->fBoundMemoryMode ? &pGVM->gmm.s.Private : &pGMM->PrivateX;
1319 if ( !pGMM->fBoundMemoryMode
1323 if (gmmR0FreeChunk(pGMM, pGVM, pChunk, true /*fRelaxedSem*/))
1339 fRedoFromStart = gmmR0MutexYield(pGMM, &uLockNanoTS)
1353 Assert(pGMM->cSharedPages >= pGVM->gmm.s.Stats.cSharedPages);
1355 pGMM->cLeftBehindSharedPages += pGVM->gmm.s.Stats.cSharedPages;
1361 Assert(pGMM->cBalloonedPages >= pGVM->gmm.s.Stats.cBalloonedPages);
1362 pGMM->cBalloonedPages -= pGVM->gmm.s.Stats.cBalloonedPages;
1367 pGMM->cReservedPages -= pGVM->gmm.s.Stats.Reserved.cBasePages
1385 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
1386 gmmR0MutexRelease(pGMM);
1399 * @param pGMM Pointer to the GMM instance.
1403 static bool gmmR0CleanupVMScanChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
1405 Assert(!pGMM->fBoundMemoryMode || pChunk->hGVM == pGVM->hSelf);
1452 gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
1485 gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
1493 gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
1563 PGMM pGMM;
1564 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
1576 gmmR0MutexAcquire(pGMM);
1577 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
1599 pGMM->cReservedPages += cBasePages + cFixedPages + cShadowPages;
1600 pGMM->cRegisteredVMs++;
1605 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
1609 gmmR0MutexRelease(pGMM);
1660 PGMM pGMM;
1661 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
1671 gmmR0MutexAcquire(pGMM);
1672 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
1687 pGMM->cReservedPages -= pGVM->gmm.s.Stats.Reserved.cBasePages
1690 pGMM->cReservedPages += cBasePages + cFixedPages + cShadowPages;
1699 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
1703 gmmR0MutexRelease(pGMM);
1736 * @param pGMM Pointer to the GMM instance.
1742 static uint32_t gmmR0SanityCheckSet(PGMM pGMM, PGMMCHUNKFREESET pSet, const char *pszSetName,
1775 * @param pGMM Pointer to the GMM instance.
1779 static uint32_t gmmR0SanityCheck(PGMM pGMM, const char *pszFunction, unsigned uLineNo)
1783 cErrors += gmmR0SanityCheckSet(pGMM, &pGMM->PrivateX, "private", pszFunction, uLineNo);
1784 cErrors += gmmR0SanityCheckSet(pGMM, &pGMM->Shared, "shared", pszFunction, uLineNo);
1798 * @param pGMM Pointer to the GMM instance.
1802 static PGMMCHUNK gmmR0GetChunkSlow(PGMM pGMM, uint32_t idChunk, PGMMCHUNKTLBE pTlbe)
1804 PGMMCHUNK pChunk = (PGMMCHUNK)RTAvlU32Get(&pGMM->pChunks, idChunk);
1818 * @param pGMM Pointer to the GMM instance.
1821 DECLINLINE(PGMMCHUNK) gmmR0GetChunk(PGMM pGMM, uint32_t idChunk)
1826 PGMMCHUNKTLBE pTlbe = &pGMM->ChunkTLB.aEntries[GMM_CHUNKTLB_IDX(idChunk)];
1829 return gmmR0GetChunkSlow(pGMM, idChunk, pTlbe);
1840 * @param pGMM Pointer to the GMM instance.
1843 DECLINLINE(PGMMPAGE) gmmR0GetPage(PGMM pGMM, uint32_t idPage)
1845 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
1856 * @param pGMM Pointer to the GMM instance.
1859 DECLINLINE(RTHCPHYS) gmmR0GetPageHCPhys(PGMM pGMM, uint32_t idPage)
1861 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
1955 DECLINLINE(void) gmmR0SelectSetAndLinkChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
1958 if (pGMM->fBoundMemoryMode)
1961 pSet = &pGMM->Shared;
1963 pSet = &pGMM->PrivateX;
1971 * @param pGMM Pointer to the GMM instance.
1974 static void gmmR0FreeChunkId(PGMM pGMM, uint32_t idChunk)
1977 AssertMsg(ASMBitTest(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk));
1978 ASMAtomicBitClear(&pGMM->bmChunkId[0], idChunk);
1986 * @param pGMM Pointer to the GMM instance.
1988 static uint32_t gmmR0AllocateChunkId(PGMM pGMM)
1996 int32_t idChunk = ++pGMM->idChunkPrev;
2010 idChunk = ASMBitNextClear(&pGMM->bmChunkId[0], GMM_CHUNKID_LAST + 1, idChunk - 1);
2013 AssertMsgReturn(!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk), NIL_GMM_CHUNKID);
2014 return pGMM->idChunkPrev = idChunk;
2022 idChunk = ASMBitFirstClear(&pGMM->bmChunkId[0], GMM_CHUNKID_LAST + 1);
2024 AssertMsgReturn(!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk), NIL_GMM_CHUNKID);
2026 return pGMM->idChunkPrev = idChunk;
2081 * @param pGMM Pointer to the GMM instance data.
2109 * @param pGMM Pointer to the GMM instance.
2121 static int gmmR0RegisterChunk(PGMM pGMM, PGMMCHUNKFREESET pSet, RTR0MEMOBJ MemObj, uint16_t hGVM, uint16_t fChunkFlags,
2124 Assert(pGMM->hMtxOwner != RTThreadNativeSelf());
2125 Assert(hGVM != NIL_GVM_HANDLE || pGMM->fBoundMemoryMode);
2154 rc = gmmR0MutexAcquire(pGMM);
2157 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
2159 pChunk->Core.Key = gmmR0AllocateChunkId(pGMM);
2162 && RTAvlU32Insert(&pGMM->pChunks, &pChunk->Core))
2164 pGMM->cChunks++;
2165 RTListAppend(&pGMM->ChunkList, &pChunk->ListNode);
2167 LogFlow(("gmmR0RegisterChunk: pChunk=%p id=%#x cChunks=%d\n", pChunk, pChunk->Core.Key, pGMM->cChunks));
2171 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
2180 gmmR0MutexRelease(pGMM);
2198 * @param pGMM Pointer to the GMM instance data.
2206 static int gmmR0AllocateChunkNew(PGMM pGMM, PGVM pGVM, PGMMCHUNKFREESET pSet, uint32_t cPages,
2209 gmmR0MutexRelease(pGMM);
2219 rc = gmmR0RegisterChunk(pGMM, pSet, hMemObj, pGVM->hSelf, 0 /*fChunkFlags*/, &pChunk);
2230 int rc2 = gmmR0MutexAcquire(pGMM);
2352 * @param pGMM Pointer to the GMM instance data.
2359 static uint32_t gmmR0AllocatePagesAssociatedWithVM(PGMM pGMM, PGVM pGVM, PGMMCHUNKFREESET pSet,
2367 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pGVM->gmm.s.idLastChunkHint);
2471 static bool gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLotsFree(PGMM pGMM)
2476 if (pGMM->PrivateX.cFreePages >= GMM_CHUNK_NUM_PAGES * 16)
2493 * @param pGMM Pointer to the GMM instance data.
2502 static int gmmR0AllocatePagesNew(PGMM pGMM, PGVM pGVM, uint32_t cPages, PGMMPAGEDESC paPages, GMMACCOUNT enmAccount)
2504 Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
2509 if (RT_UNLIKELY(pGMM->cAllocatedPages + cPages > pGMM->cMaxPages))
2548 if ( pGMM->fLegacyAllocationMode
2551 Assert(pGMM->fBoundMemoryMode);
2568 pGMM->cAllocatedPages += cPages;
2574 if (pGMM->fLegacyAllocationMode)
2585 if (pGMM->fBoundMemoryMode)
2590 rc = gmmR0AllocateChunkNew(pGMM, pGVM, &pGVM->gmm.s.Private, cPages, paPages, &iPage);
2601 iPage = gmmR0AllocatePagesAssociatedWithVM(pGMM, pGVM, &pGMM->PrivateX, iPage, cPages, paPages);
2609 iPage = gmmR0AllocatePagesFromSameNode(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
2615 iPage = gmmR0AllocatePagesFromEmptyChunksOnSameNode(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
2619 iPage = gmmR0AllocatePagesFromEmptyChunksOnSameNode(&pGMM->Shared, pGVM, iPage, cPages, paPages);
2624 && gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLotsFree(pGMM))
2626 iPage = gmmR0AllocatePagesFromSameNode(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
2628 iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
2637 rc = gmmR0AllocateChunkNew(pGMM, pGVM, &pGMM->PrivateX, cPages, paPages, &iPage);
2642 && pGMM->PrivateX.cFreePages + pGMM->Shared.cFreePages >= cPages - iPage)
2644 iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
2646 iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->Shared, pGVM, iPage, cPages, paPages);
2662 pGMM->cAllocatedPages -= cPages - iPage;
2675 PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
2680 gmmR0FreePrivatePage(pGMM, pGVM, idPage, pPage);
2738 PGMM pGMM;
2739 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
2775 gmmR0MutexAcquire(pGMM);
2776 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
2791 PGMMPAGE pPage = gmmR0GetPage(pGMM, paPages[iPage].idPage);
2833 PGMMPAGE pPage = gmmR0GetPage(pGMM, paPages[iPage].idSharedPage);
2847 gmmR0FreeSharedPage(pGMM, pGVM, paPages[iPage].idSharedPage, pPage);
2850 Assert(pGMM->cDuplicatePages);
2851 pGMM->cDuplicatePages--;
2887 rc = gmmR0AllocatePagesNew(pGMM, pGVM, cPagesToAlloc, paPages, GMMACCOUNT_BASE);
2892 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
2896 gmmR0MutexRelease(pGMM);
2932 PGMM pGMM;
2933 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
2956 gmmR0MutexAcquire(pGMM);
2957 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
2964 rc = gmmR0AllocatePagesNew(pGMM, pGVM, cPages, paPages, enmAccount);
2967 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
2971 gmmR0MutexRelease(pGMM);
3031 PGMM pGMM;
3032 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3039 if (pGMM->fLegacyAllocationMode)
3045 gmmR0MutexAcquire(pGMM);
3046 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
3054 gmmR0MutexRelease(pGMM);
3065 gmmR0MutexRelease(pGMM);
3071 PGMMCHUNKFREESET pSet = pGMM->fBoundMemoryMode ? &pGVM->gmm.s.Private : &pGMM->PrivateX;
3073 rc = gmmR0RegisterChunk(pGMM, pSet, hMemObj, pGVM->hSelf, GMM_CHUNK_FLAGS_LARGE_PAGE, &pChunk);
3097 pGMM->cAllocatedPages += cPages;
3100 gmmR0MutexRelease(pGMM);
3108 gmmR0MutexRelease(pGMM);
3132 PGMM pGMM;
3133 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3140 if (pGMM->fLegacyAllocationMode)
3143 gmmR0MutexAcquire(pGMM);
3144 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
3151 gmmR0MutexRelease(pGMM);
3155 PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
3159 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
3165 gmmR0FreeChunk(pGMM, NULL, pChunk, false /*fRelaxedSem*/); /** @todo this can be relaxed too! */
3170 pGMM->cAllocatedPages -= cPages;
3178 gmmR0MutexRelease(pGMM);
3210 * @param pGMM Pointer to the GMM instance.
3217 static bool gmmR0FreeChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem)
3222 gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
3229 && !pGMM->fLegacyAllocationMode
3231 gmmR0UnmapChunkLocked(pGMM, pGVM, pChunk);
3261 PAVLU32NODECORE pCore = RTAvlU32Remove(&pGMM->pChunks, pChunk->Core.Key);
3264 PGMMCHUNKTLBE pTlbe = &pGMM->ChunkTLB.aEntries[GMM_CHUNKTLB_IDX(pChunk->Core.Key)];
3271 Assert(pGMM->cChunks > 0);
3272 pGMM->cChunks--;
3277 gmmR0FreeChunkId(pGMM, pChunk->Core.Key);
3280 pGMM->cFreedChunks++;
3284 gmmR0MutexRelease(pGMM);
3295 gmmR0MutexAcquire(pGMM);
3305 * @param pGMM Pointer to the GMM instance data.
3311 static void gmmR0FreePageWorker(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, uint32_t idPage, PGMMPAGE pPage)
3335 gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
3356 && !pGMM->fLegacyAllocationMode))
3357 gmmR0FreeChunk(pGMM, NULL, pChunk, false);
3365 * @param pGMM Pointer to the GMM instance.
3370 DECLINLINE(void) gmmR0FreeSharedPage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage)
3372 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
3376 Assert(pGMM->cSharedPages > 0);
3377 Assert(pGMM->cAllocatedPages > 0);
3381 pGMM->cAllocatedPages--;
3382 pGMM->cSharedPages--;
3383 gmmR0FreePageWorker(pGMM, pGVM, pChunk, idPage, pPage);
3390 * @param pGMM Pointer to the GMM instance.
3395 DECLINLINE(void) gmmR0FreePrivatePage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage)
3397 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
3401 Assert(pGMM->cAllocatedPages > 0);
3404 pGMM->cAllocatedPages--;
3405 gmmR0FreePageWorker(pGMM, pGVM, pChunk, idPage, pPage);
3415 * @param pGMM Pointer to the GMM instance data.
3421 static int gmmR0FreePages(PGMM pGMM, PGVM pGVM, uint32_t cPages, PGMMFREEPAGEDESC paPages, GMMACCOUNT enmAccount)
3464 PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
3473 gmmR0FreePrivatePage(pGMM, pGVM, idPage, pPage);
3490 uint32_t uChecksum = gmmR0StrictPageChecksum(pGMM, pGVM, idPage);
3498 gmmR0FreeSharedPage(pGMM, pGVM, idPage, pPage);
3501 Assert(pGMM->cDuplicatePages);
3502 pGMM->cDuplicatePages--;
3563 PGMM pGMM;
3564 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3582 gmmR0MutexAcquire(pGMM);
3583 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
3585 rc = gmmR0FreePages(pGMM, pGVM, cPages, paPages, enmAccount);
3586 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
3590 gmmR0MutexRelease(pGMM);
3655 PGMM pGMM;
3656 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3665 gmmR0MutexAcquire(pGMM);
3666 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
3678 pGMM->cBalloonedPages += cBalloonedPages;
3687 cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages,
3694 cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages));
3715 Assert(pGMM->cBalloonedPages >= cBalloonedPages);
3716 pGMM->cBalloonedPages -= cBalloonedPages;
3722 cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages, pGVM->gmm.s.Stats.cReqDeflatePages));
3731 cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages));
3744 Assert(pGMM->cBalloonedPages >= pGVM->gmm.s.Stats.cBalloonedPages);
3746 pGMM->cBalloonedPages -= pGVM->gmm.s.Stats.cBalloonedPages;
3755 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
3760 gmmR0MutexRelease(pGMM);
3809 PGMM pGMM;
3810 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3811 pReq->cAllocPages = pGMM->cAllocatedPages;
3812 pReq->cFreePages = (pGMM->cChunks << (GMM_CHUNK_SHIFT- PAGE_SHIFT)) - pGMM->cAllocatedPages;
3813 pReq->cBalloonedPages = pGMM->cBalloonedPages;
3814 pReq->cMaxPages = pGMM->cMaxPages;
3815 pReq->cSharedPages = pGMM->cDuplicatePages;
3816 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
3843 PGMM pGMM;
3844 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3853 gmmR0MutexAcquire(pGMM);
3854 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
3864 gmmR0MutexRelease(pGMM);
3876 * @param pGMM Pointer to the GMM instance data.
3880 static int gmmR0UnmapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
3882 Assert(!pGMM->fLegacyAllocationMode);
3920 * @param pGMM Pointer to the GMM instance data.
3924 static int gmmR0UnmapChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem)
3926 if (!pGMM->fLegacyAllocationMode)
3932 int rc = gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk,
3936 rc = gmmR0UnmapChunkLocked(pGMM, pGVM, pChunk);
3954 * @param pGMM Pointer to the GMM instance data.
3961 static int gmmR0MapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, PRTR3PTR ppvR3)
3966 if (pGMM->fLegacyAllocationMode)
4045 * @param pGMM Pointer to the GMM instance data.
4054 static int gmmR0MapChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem, PRTR3PTR ppvR3)
4061 int rc = gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk,
4065 rc = gmmR0MapChunkLocked(pGMM, pGVM, pChunk, ppvR3);
4079 * @param pGMM Pointer to the GMM instance.
4084 static bool gmmR0IsChunkMapped(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, PRTR3PTR ppvR3)
4087 gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
4128 PGMM pGMM;
4129 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
4157 gmmR0MutexAcquire(pGMM);
4158 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
4163 pMap = gmmR0GetChunk(pGMM, idChunkMap);
4165 rc = gmmR0MapChunk(pGMM, pGVM, pMap, true /*fRelaxedSem*/, ppvR3);
4178 PGMMCHUNK pUnmap = gmmR0GetChunk(pGMM, idChunkUnmap);
4180 rc = gmmR0UnmapChunk(pGMM, pGVM, pUnmap, true /*fRelaxedSem*/);
4188 gmmR0UnmapChunk(pGMM, pGVM, pMap, false /*fRelaxedSem*/);
4191 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
4195 gmmR0MutexRelease(pGMM);
4238 PGMM pGMM;
4239 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
4248 if (!pGMM->fLegacyAllocationMode)
4262 rc = gmmR0RegisterChunk(pGMM, &pGVM->gmm.s.Private, MemObj, pGVM->hSelf, 0 /*fChunkFlags*/, NULL);
4264 gmmR0MutexRelease(pGMM);
4285 static uint32_t gmmR0StrictPageChecksum(PGMM pGMM, PGVM pGVM, uint32_t idPage)
4287 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
4291 if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
4317 * @param pGMM The GMM instance data.
4324 static PGMMSHAREDMODULE gmmR0ShModFindGlobal(PGMM pGMM, uint32_t uHash, uint32_t cbModule, VBOXOSFAMILY enmGuestOS,
4328 for (PGMMSHAREDMODULE pGblMod = (PGMMSHAREDMODULE)RTAvllU32Get(&pGMM->pGlobalSharedModuleTree, uHash);
4367 * @param pGMM The GMM instance data.
4377 static int gmmR0ShModNewGlobal(PGMM pGMM, uint32_t uHash, uint32_t cbModule, VBOXOSFAMILY enmGuestOS,
4382 if (pGMM->cShareableModules >= GMM_MAX_SHARED_GLOBAL_MODULES)
4412 bool fInsert = RTAvllU32Insert(&pGMM->pGlobalSharedModuleTree, &pGblMod->Core);
4414 pGMM->cShareableModules++;
4424 * @param pGMM The GMM instance data.
4427 static void gmmR0ShModDeleteGlobal(PGMM pGMM, PGMMSHAREDMODULE pGblMod)
4430 Assert(pGMM->cShareableModules > 0 && pGMM->cShareableModules <= GMM_MAX_SHARED_GLOBAL_MODULES);
4432 void *pvTest = RTAvllU32RemoveNode(&pGMM->pGlobalSharedModuleTree, &pGblMod->Core);
4434 pGMM->cShareableModules--;
4475 static void gmmR0ShModDeletePerVM(PGMM pGMM, PGVM pGVM, PGMMSHAREDMODULEPERVM pRecVM, bool fRemove)
4500 gmmR0ShModDeleteGlobal(pGMM, pGblMod);
4531 PGMM pGMM;
4532 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
4569 gmmR0MutexAcquire(pGMM);
4570 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
4587 PGMMSHAREDMODULE pGblMod = gmmR0ShModFindGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
4592 rc = gmmR0ShModNewGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
4600 gmmR0ShModDeletePerVM(pGMM, pGVM, pRecVM, true /*fRemove*/);
4617 PGMMSHAREDMODULE pGblMod = gmmR0ShModFindGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
4637 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
4642 gmmR0MutexRelease(pGMM);
4695 PGMM pGMM;
4696 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
4714 gmmR0MutexAcquire(pGMM);
4715 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
4726 gmmR0ShModDeletePerVM(pGMM, pGVM, pRecVM, true /*fRemove*/);
4731 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
4736 gmmR0MutexRelease(pGMM);
4771 * @param pGMM Pointer to the GMM instance.
4775 DECLINLINE(void) gmmR0UseSharedPage(PGMM pGMM, PGVM pGVM, PGMMPAGE pPage)
4777 Assert(pGMM->cSharedPages > 0);
4778 Assert(pGMM->cAllocatedPages > 0);
4780 pGMM->cDuplicatePages++;
4791 * @param pGMM Pointer to the GMM instance.
4797 DECLINLINE(void) gmmR0ConvertToSharedPage(PGMM pGMM, PGVM pGVM, RTHCPHYS HCPhys, uint32_t idPage, PGMMPAGE pPage,
4800 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
4808 pGMM->cSharedPages++;
4817 pPageDesc->u32StrictChecksum = gmmR0StrictPageChecksum(pGMM, pGVM, idPage);
4826 static int gmmR0SharedModuleCheckPageFirstTime(PGMM pGMM, PGVM pGVM, PGMMSHAREDMODULE pModule,
4833 PGMMPAGE pPage = gmmR0GetPage(pGMM, pPageDesc->idPage);
4840 gmmR0ConvertToSharedPage(pGMM, pGVM, pPageDesc->HCPhys, pPageDesc->idPage, pPage, pPageDesc);
4861 * @param pGMM Pointer to the GMM instance data.
4872 PGMM pGMM;
4873 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
4909 return gmmR0SharedModuleCheckPageFirstTime(pGMM, pGVM, pModule, idxRegion, idxPage, pPageDesc, pGlobalRegion);
4922 PGMMPAGE pPage = gmmR0GetPage(pGMM, pGlobalRegion->paidPages[idxPage]);
4934 return gmmR0SharedModuleCheckPageFirstTime(pGMM, pGVM, pModule, idxRegion, idxPage, pPageDesc, pGlobalRegion);
4942 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pPageDesc->idPage >> GMM_CHUNKID_SHIFT);
4947 AssertMsgReturn(gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk),
4955 pChunk = gmmR0GetChunk(pGMM, pGlobalRegion->paidPages[idxPage] >> GMM_CHUNKID_SHIFT);
4962 if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
4965 rc = gmmR0MapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/, (PRTR3PTR)&pbChunk);
4992 rc = gmmR0FreePages(pGMM, pGVM, 1, &PageDesc, GMMACCOUNT_BASE);
4995 gmmR0UseSharedPage(pGMM, pGVM, pPage);
5016 gmmR0ShModDeletePerVM(((GMMR0SHMODPERVMDTORARGS *)pvArgs)->pGMM,
5030 * @param pGMM The GMM handle.
5033 static void gmmR0SharedModuleCleanup(PGMM pGMM, PGVM pGVM)
5035 gmmR0MutexAcquire(pGMM);
5036 GMM_CHECK_SANITY_UPON_ENTERING(pGMM);
5040 Args.pGMM = pGMM;
5046 gmmR0MutexRelease(pGMM);
5064 PGMM pGMM;
5065 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
5074 gmmR0MutexAcquire(pGMM);
5075 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
5080 Args.pGMM = pGMM;
5085 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
5090 gmmR0MutexRelease(pGMM);
5132 PGMM pGMM;
5133 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
5138 gmmR0MutexAcquire(pGMM);
5139 if (!GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
5158 PGMM pGMM;
5159 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
5161 gmmR0MutexRelease(pGMM);
5180 PGMM pGMM;
5181 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
5191 gmmR0MutexAcquire(pGMM);
5193 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
5206 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
5212 gmmR0MutexRelease(pGMM);
5235 PGMM pGMM = pArgs->pGMM;
5239 if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
5241 int rc = gmmR0MapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/, (PRTR3PTR)&pbChunk);
5261 gmmR0UnmapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/);
5284 PGMM pGMM;
5285 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
5295 rc = gmmR0MutexAcquire(pGMM);
5296 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
5299 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pReq->idPage >> GMM_CHUNKID_SHIFT);
5302 if (gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
5305 PGMMPAGE pPage = gmmR0GetPage(pGMM, pReq->idPage);
5310 Args.pGMM = pGMM;
5313 RTAvlU32DoWithAll(&pGMM->pChunks, true /* fFromLeft */, gmmR0FindDupPageInChunk, &Args);
5332 gmmR0MutexRelease(pGMM);
5359 PGMM pGMM;
5360 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
5376 rc = gmmR0MutexAcquire(pGMM);
5383 pStats->cMaxPages = pGMM->cMaxPages;
5384 pStats->cReservedPages = pGMM->cReservedPages;
5385 pStats->cOverCommittedPages = pGMM->cOverCommittedPages;
5386 pStats->cAllocatedPages = pGMM->cAllocatedPages;
5387 pStats->cSharedPages = pGMM->cSharedPages;
5388 pStats->cDuplicatePages = pGMM->cDuplicatePages;
5389 pStats->cLeftBehindSharedPages = pGMM->cLeftBehindSharedPages;
5390 pStats->cBalloonedPages = pGMM->cBalloonedPages;
5391 pStats->cChunks = pGMM->cChunks;
5392 pStats->cFreedChunks = pGMM->cFreedChunks;
5393 pStats->cShareableModules = pGMM->cShareableModules;
5404 gmmR0MutexRelease(pGMM);