Lines Matching defs:pHeap
44 #define ASSERT_OFFPREV(pHeap, pChunk) \
46 Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
48 || (uint8_t *)(pChunk) == (pHeap)->CTX_SUFF(pbHeap), \
49 ("pChunk=%p pvHyperHeap=%p\n", (pChunk), (pHeap)->CTX_SUFF(pbHeap))); \
52 #define ASSERT_OFFNEXT(pHeap, pChunk) \
54 ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
57 #define ASSERT_OFFHEAP(pHeap, pChunk) \
59 AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
60 ("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
61 Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
65 #define ASSERT_OFFSTAT(pHeap, pChunk) \
73 AssertMsg(uPtr - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) < (pHeap)->offPageAligned, \
74 ("%p - %p < %RX32\n", uPtr, (pHeap)->CTX_SUFF(pbHeap), (pHeap)->offPageAligned)); \
78 #define ASSERT_OFFSTAT(pHeap, pChunk) \
83 #define ASSERT_CHUNK(pHeap, pChunk) \
84 do { ASSERT_OFFNEXT(pHeap, pChunk); \
85 ASSERT_OFFPREV(pHeap, pChunk); \
86 ASSERT_OFFHEAP(pHeap, pChunk); \
87 ASSERT_OFFSTAT(pHeap, pChunk); \
89 #define ASSERT_CHUNK_USED(pHeap, pChunk) \
90 do { ASSERT_OFFNEXT(pHeap, pChunk); \
91 ASSERT_OFFPREV(pHeap, pChunk); \
95 #define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
97 ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
100 || (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeHead, \
101 ("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap),\
102 (pHeap)->offFreeHead)); \
105 #define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
107 ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
110 || (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeTail, \
111 ("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap), \
112 (pHeap)->offFreeTail)); \
115 #define ASSERT_FREE_CB(pHeap, pChunk) \
122 ASSERT_LE((pChunk)->cb, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
125 #define ASSERT_CHUNK_FREE(pHeap, pChunk) \
126 do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
128 ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
129 ASSERT_FREE_OFFPREV(pHeap, pChunk); \
130 ASSERT_FREE_CB(pHeap, pChunk); \
137 static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
138 static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
140 static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
145 static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
147 static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
160 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
163 if (!PDMCritSectIsInitialized(&pHeap->Lock))
166 Assert(PDMCritSectIsInitialized(&pHeap->Lock));
168 int rc = PDMCritSectEnter(&pHeap->Lock, VERR_SEM_BUSY);
185 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
188 if (!PDMCritSectIsInitialized(&pHeap->Lock))
191 Assert(PDMCritSectIsInitialized(&pHeap->Lock));
192 PDMCritSectLeave(&pHeap->Lock);
255 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
257 PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
270 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
276 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
298 void *pv = mmHyperAllocPages(pHeap, cbAligned);
371 * @param pHeap The heap.
376 static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
380 mmHyperHeapCheck(pHeap);
390 if (pHeap->offFreeHead == NIL_OFFSET)
400 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeHead);
403 ASSERT_CHUNK_FREE(pHeap, pFree);
431 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)pHeap->CTX_SUFF(pbHeap);
439 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
440 pHeap->cbFree -= offAlign;
452 ASSERT_CHUNK(pHeap, pNext);
462 ASSERT_CHUNK_FREE(pHeap, pNext);
465 pHeap->offFreeTail += offAlign;
471 ASSERT_CHUNK_FREE(pHeap, pPrev);
474 pHeap->offFreeHead += offAlign;
475 pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
477 ASSERT_CHUNK_FREE(pHeap, pFree);
498 ASSERT_CHUNK(pHeap, pNext);
508 ASSERT_CHUNK_FREE(pHeap, pNext);
511 pHeap->offFreeTail += off;
517 ASSERT_CHUNK_FREE(pHeap, pPrev);
520 pHeap->offFreeHead += off;
521 pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
523 ASSERT_CHUNK_FREE(pHeap, pNew);
532 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
533 pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
534 pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
536 ASSERT_CHUNK(pHeap, &pFree->core);
552 ASSERT_CHUNK_FREE(pHeap, pPrev);
556 pHeap->offFreeHead += pFree->offNext;
559 ASSERT_CHUNK_FREE(pHeap, pNext);
565 pHeap->offFreeTail += pFree->offPrev;
568 ASSERT_CHUNK_FREE(pHeap, pPrev);
572 pHeap->offFreeHead = NIL_OFFSET;
573 pHeap->offFreeTail = NIL_OFFSET;
577 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
578 pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
579 pHeap->cbFree -= pFree->cb;
582 ASSERT_CHUNK(pHeap, &pFree->core);
598 : (uint32_t *)(pHeap->CTX_SUFF(pbHeap) + pHeap->cbHeap);
604 mmHyperHeapCheck(pHeap);
616 * @param pHeap The heap.
620 static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
625 mmHyperHeapCheck(pHeap);
631 if (pHeap->offFreeHead == NIL_OFFSET)
645 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail);
646 ASSERT_CHUNK_FREE(pHeap, pFree);
663 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
664 pHeap->cbFree -= cb;
665 ASSERT_CHUNK_FREE(pHeap, pFree);
674 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
675 pHeap->cbFree -= pFree->cb;
695 pHeap->offFreeTail += pFree->offPrev;
700 pHeap->offFreeTail = NIL_OFFSET;
701 pHeap->offFreeHead = NIL_OFFSET;
705 pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
709 mmHyperHeapCheck(pHeap);
721 * @param pHeap The heap.
724 static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
727 PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
731 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
739 RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
744 mmR3HyperStatRegisterOne(pHeap->pVMR3, pStat);
830 PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
831 AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
833 ("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
836 AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
837 ("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
839 Assert(pHeap == pVM->mm.s.CTX_SUFF(pHyperHeap));
841 /* Some more verifications using additional info from pHeap. */
842 AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)pHeap->CTX_SUFF(pbHeap),
846 AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
850 AssertMsgReturn( (uintptr_t)pv - (uintptr_t)pHeap->CTX_SUFF(pbHeap) <= pHeap->offPageAligned,
851 ("Invalid pointer %p! (heap: %p-%p)\n", pv, pHeap->CTX_SUFF(pbHeap),
852 (char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned),
856 mmHyperHeapCheck(pHeap);
863 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
875 unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
877 if (pHeap->aDelayedFrees[i].offChunk)
879 PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
882 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pCur - sizeof(*pCur);
887 pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
895 if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
897 PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
898 rc = mmHyperFree(pHeap, pChunkFree);
900 pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
901 pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
902 pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
908 int rc = mmHyperFree(pHeap, pChunk);
952 * @param pHeap The heap.
955 static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
957 Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
968 if (pHeap->offFreeTail != NIL_OFFSET)
973 ASSERT_CHUNK(pHeap, &pRight->core);
982 ASSERT_CHUNK(pHeap, &pRight->core);
986 pRight = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail); /** @todo this can't be correct! 'pLeft = .. ; else' I think */
989 ASSERT_CHUNK_FREE(pHeap, pRight);
993 ASSERT_CHUNK_FREE(pHeap, pLeft);
1011 pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1020 pHeap->offFreeTail = pHeap->offFreeHead;
1039 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
1040 pHeap->cbFree -= pLeft->cb;
1059 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1076 ASSERT_CHUNK(pHeap, pNext);
1090 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1092 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1093 pHeap->cbFree -= pRight->cb;
1101 pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap)) - sizeof(MMHYPERCHUNK);
1102 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1103 pHeap->cbFree += pFree->cb;
1104 ASSERT_CHUNK_FREE(pHeap, pFree);
1107 mmHyperHeapCheck(pHeap);
1117 * @param pHeap Pointer to the heap.
1120 static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1130 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1135 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1142 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1147 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1157 static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1160 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1164 ASSERT_CHUNK_USED(pHeap, &pCur->core);
1166 ASSERT_CHUNK_FREE(pHeap, pCur);
1172 uint32_t off = (uint8_t *)pCur - pHeap->CTX_SUFF(pbHeap);
1174 && off < pHeap->offPageAligned)
1178 : pHeap->cbHeap - off;
1184 mmHyperHeapDumpOne(pHeap, pCur);
1192 mmHyperHeapDumpOne(pHeap, pCur);
1235 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
1236 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1239 mmHyperHeapDumpOne(pHeap, pCur);