Lines Matching refs:patm
24 #include <VBox/vmm/patm.h>
108 #define patmPatchHCPtr2PatchGCPtr(pVM, pHC) (pVM->patm.s.pPatchMemGC + (pHC - pVM->patm.s.pPatchMemHC))
109 #define patmPatchGCPtr2PatchHCPtr(pVM, pGC) (pVM->patm.s.pPatchMemHC + (pGC - pVM->patm.s.pPatchMemGC))
167 pVM->patm.s.cbPatchMem = PATCH_MEMORY_SIZE;
170 rc = MMR3HyperAllocOnceNoRel(pVM, PATCH_MEMORY_SIZE + PAGE_SIZE + PATM_STACK_TOTAL_SIZE + PAGE_SIZE + PATM_STAT_MEMSIZE, PAGE_SIZE, MM_TAG_PATM, (void **)&pVM->patm.s.pPatchMemHC);
176 pVM->patm.s.pPatchMemGC = MMHyperR3ToRC(pVM, pVM->patm.s.pPatchMemHC);
179 pVM->patm.s.pGCStackHC = (RTRCPTR *)(pVM->patm.s.pPatchMemHC + PATCH_MEMORY_SIZE + PAGE_SIZE);
180 pVM->patm.s.pGCStackGC = MMHyperR3ToRC(pVM, pVM->patm.s.pGCStackHC);
192 pVM->patm.s.pGCStateHC = (PPATMGCSTATE)((uint8_t *)pVM->patm.s.pGCStackHC + PATM_STACK_TOTAL_SIZE);
193 pVM->patm.s.pGCStateGC = MMHyperR3ToRC(pVM, pVM->patm.s.pGCStateHC);
196 pVM->patm.s.pStatsHC = (PSTAMRATIOU32)((uint8_t *)pVM->patm.s.pGCStateHC + PAGE_SIZE);
197 pVM->patm.s.pStatsGC = MMHyperR3ToRC(pVM, pVM->patm.s.pStatsHC);
200 rc = MMHyperAlloc(pVM, sizeof(*pVM->patm.s.PatchLookupTreeHC), 0, MM_TAG_PATM, (void **)&pVM->patm.s.PatchLookupTreeHC);
202 pVM->patm.s.PatchLookupTreeGC = MMHyperR3ToRC(pVM, pVM->patm.s.PatchLookupTreeHC);
223 rc = SSMR3RegisterInternal(pVM, "PATM", 0, PATM_SAVED_STATE_VERSION, sizeof(pVM->patm.s) + PATCH_MEMORY_SIZE + PAGE_SIZE + PATM_STACK_TOTAL_SIZE + PAGE_SIZE,
243 STAM_REG(pVM, &pVM->patm.s.StatNrOpcodeRead, STAMTYPE_COUNTER, "/PATM/OpcodeBytesRead", STAMUNIT_OCCURENCES, "The number of opcode bytes read by the recompiler.");
244 STAM_REG(pVM, &pVM->patm.s.StatPATMMemoryUsed,STAMTYPE_COUNTER, "/PATM/MemoryUsed", STAMUNIT_OCCURENCES, "The amount of hypervisor heap used for patches.");
245 STAM_REG(pVM, &pVM->patm.s.StatDisabled, STAMTYPE_COUNTER, "/PATM/Patch/Disabled", STAMUNIT_OCCURENCES, "Number of times patches were disabled.");
246 STAM_REG(pVM, &pVM->patm.s.StatEnabled, STAMTYPE_COUNTER, "/PATM/Patch/Enabled", STAMUNIT_OCCURENCES, "Number of times patches were enabled.");
247 STAM_REG(pVM, &pVM->patm.s.StatDirty, STAMTYPE_COUNTER, "/PATM/Patch/Dirty", STAMUNIT_OCCURENCES, "Number of times patches were marked dirty.");
248 STAM_REG(pVM, &pVM->patm.s.StatUnusable, STAMTYPE_COUNTER, "/PATM/Patch/Unusable", STAMUNIT_OCCURENCES, "Number of unusable patches (conflicts).");
249 STAM_REG(pVM, &pVM->patm.s.StatInstalled, STAMTYPE_COUNTER, "/PATM/Patch/Installed", STAMUNIT_OCCURENCES, "Number of installed patches.");
250 STAM_REG(pVM, &pVM->patm.s.StatInt3Callable, STAMTYPE_COUNTER, "/PATM/Patch/Int3Callable", STAMUNIT_OCCURENCES, "Number of cli patches turned into int3 patches.");
252 STAM_REG(pVM, &pVM->patm.s.StatInt3BlockRun, STAMTYPE_COUNTER, "/PATM/Patch/Run/Int3", STAMUNIT_OCCURENCES, "Number of times an int3 block patch was executed.");
253 STAMR3RegisterF(pVM, &pVM->patm.s.pGCStateHC->uPatchCalls, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, NULL, "/PATM/Patch/Run/Normal");
255 STAM_REG(pVM, &pVM->patm.s.StatInstalledFunctionPatches, STAMTYPE_COUNTER, "/PATM/Patch/Installed/Function", STAMUNIT_OCCURENCES, "Number of installed function duplication patches.");
256 STAM_REG(pVM, &pVM->patm.s.StatInstalledTrampoline, STAMTYPE_COUNTER, "/PATM/Patch/Installed/Trampoline", STAMUNIT_OCCURENCES, "Number of installed trampoline patches.");
257 STAM_REG(pVM, &pVM->patm.s.StatInstalledJump, STAMTYPE_COUNTER, "/PATM/Patch/Installed/Jump", STAMUNIT_OCCURENCES, "Number of installed jump patches.");
259 STAM_REG(pVM, &pVM->patm.s.StatOverwritten, STAMTYPE_COUNTER, "/PATM/Patch/Overwritten", STAMUNIT_OCCURENCES, "Number of overwritten patches.");
260 STAM_REG(pVM, &pVM->patm.s.StatFixedConflicts,STAMTYPE_COUNTER, "/PATM/Patch/ConflictFixed", STAMUNIT_OCCURENCES, "Number of fixed conflicts.");
261 STAM_REG(pVM, &pVM->patm.s.StatFlushed, STAMTYPE_COUNTER, "/PATM/Patch/Flushed", STAMUNIT_OCCURENCES, "Number of flushes of pages with patch jumps.");
262 STAM_REG(pVM, &pVM->patm.s.StatMonitored, STAMTYPE_COUNTER, "/PATM/Patch/Monitored", STAMUNIT_OCCURENCES, "Number of patches in monitored patch pages.");
263 STAM_REG(pVM, &pVM->patm.s.StatPageBoundaryCrossed, STAMTYPE_COUNTER, "/PATM/Patch/BoundaryCross", STAMUNIT_OCCURENCES, "Number of refused patches due to patch jump crossing page boundary.");
265 STAM_REG(pVM, &pVM->patm.s.StatHandleTrap, STAMTYPE_PROFILE, "/PATM/HandleTrap", STAMUNIT_TICKS_PER_CALL, "Profiling of PATMR3HandleTrap");
266 STAM_REG(pVM, &pVM->patm.s.StatPushTrap, STAMTYPE_COUNTER, "/PATM/HandleTrap/PushWP", STAMUNIT_OCCURENCES, "Number of traps due to monitored stack pages.");
268 STAM_REG(pVM, &pVM->patm.s.StatSwitchBack, STAMTYPE_COUNTER, "/PATM/SwitchBack", STAMUNIT_OCCURENCES, "Switch back to original guest code when IF=1 & executing PATM instructions");
269 STAM_REG(pVM, &pVM->patm.s.StatSwitchBackFail,STAMTYPE_COUNTER, "/PATM/SwitchBackFail", STAMUNIT_OCCURENCES, "Failed switch back to original guest code when IF=1 & executing PATM instructions");
271 STAM_REG(pVM, &pVM->patm.s.StatDuplicateREQFailed, STAMTYPE_COUNTER, "/PATM/Function/DupREQ/Failed", STAMUNIT_OCCURENCES, "Nr of failed PATMR3DuplicateFunctionRequest calls");
272 STAM_REG(pVM, &pVM->patm.s.StatDuplicateREQSuccess, STAMTYPE_COUNTER, "/PATM/Function/DupREQ/Success", STAMUNIT_OCCURENCES, "Nr of successful PATMR3DuplicateFunctionRequest calls");
273 STAM_REG(pVM, &pVM->patm.s.StatDuplicateUseExisting,STAMTYPE_COUNTER, "/PATM/Function/DupREQ/UseExist", STAMUNIT_OCCURENCES, "Nr of successful PATMR3DuplicateFunctionRequest calls when using an existing patch");
275 STAM_REG(pVM, &pVM->patm.s.StatFunctionLookupInsert, STAMTYPE_COUNTER, "/PATM/Function/Lookup/Insert", STAMUNIT_OCCURENCES, "Nr of successful function address insertions");
276 STAM_REG(pVM, &pVM->patm.s.StatFunctionLookupReplace, STAMTYPE_COUNTER, "/PATM/Function/Lookup/Replace", STAMUNIT_OCCURENCES, "Nr of successful function address replacements");
277 STAM_REG(pVM, &pVM->patm.s.StatU32FunctionMaxSlotsUsed, STAMTYPE_U32_RESET,"/PATM/Function/Lookup/MaxSlots", STAMUNIT_OCCURENCES, "Maximum nr of lookup slots used in all call patches");
279 STAM_REG(pVM, &pVM->patm.s.StatFunctionFound, STAMTYPE_COUNTER, "/PATM/Function/Found", STAMUNIT_OCCURENCES, "Nr of successful function patch lookups in GC");
280 STAM_REG(pVM, &pVM->patm.s.StatFunctionNotFound, STAMTYPE_COUNTER, "/PATM/Function/NotFound", STAMUNIT_OCCURENCES, "Nr of failed function patch lookups in GC");
282 STAM_REG(pVM, &pVM->patm.s.StatPatchWrite, STAMTYPE_PROFILE, "/PATM/Write/Handle", STAMUNIT_TICKS_PER_CALL, "Profiling of PATMR3PatchWrite");
283 STAM_REG(pVM, &pVM->patm.s.StatPatchWriteDetect, STAMTYPE_PROFILE, "/PATM/Write/Detect", STAMUNIT_TICKS_PER_CALL, "Profiling of PATMIsWriteToPatchPage");
284 STAM_REG(pVM, &pVM->patm.s.StatPatchWriteInterpreted, STAMTYPE_COUNTER, "/PATM/Write/Interpreted/Success", STAMUNIT_OCCURENCES, "Nr of interpreted patch writes.");
285 STAM_REG(pVM, &pVM->patm.s.StatPatchWriteInterpretedFailed, STAMTYPE_COUNTER, "/PATM/Write/Interpreted/Failed", STAMUNIT_OCCURENCES, "Nr of failed interpreted patch writes.");
287 STAM_REG(pVM, &pVM->patm.s.StatPatchRefreshSuccess, STAMTYPE_COUNTER, "/PATM/Refresh/Success", STAMUNIT_OCCURENCES, "Successful patch refreshes");
288 STAM_REG(pVM, &pVM->patm.s.StatPatchRefreshFailed, STAMTYPE_COUNTER, "/PATM/Refresh/Failure", STAMUNIT_OCCURENCES, "Failed patch refreshes");
290 STAM_REG(pVM, &pVM->patm.s.StatPatchPageInserted, STAMTYPE_COUNTER, "/PATM/Page/Inserted", STAMUNIT_OCCURENCES, "Nr of inserted guest pages that were patched");
291 STAM_REG(pVM, &pVM->patm.s.StatPatchPageRemoved, STAMTYPE_COUNTER, "/PATM/Page/Removed", STAMUNIT_OCCURENCES, "Nr of removed guest pages that were patched");
293 STAM_REG(pVM, &pVM->patm.s.StatInstrDirty, STAMTYPE_COUNTER, "/PATM/Instr/Dirty/Detected", STAMUNIT_OCCURENCES, "Number of times instructions were marked dirty.");
294 STAM_REG(pVM, &pVM->patm.s.StatInstrDirtyGood, STAMTYPE_COUNTER, "/PATM/Instr/Dirty/Corrected", STAMUNIT_OCCURENCES, "Number of times instructions were marked dirty and corrected later on.");
295 STAM_REG(pVM, &pVM->patm.s.StatInstrDirtyBad, STAMTYPE_COUNTER, "/PATM/Instr/Dirty/Failed", STAMUNIT_OCCURENCES, "Number of times instructions were marked dirty and we were not able to correct them.");
297 STAM_REG(pVM, &pVM->patm.s.StatSysEnter, STAMTYPE_COUNTER, "/PATM/Emul/SysEnter", STAMUNIT_OCCURENCES, "Number of times sysenter was emulated.");
298 STAM_REG(pVM, &pVM->patm.s.StatSysExit, STAMTYPE_COUNTER, "/PATM/Emul/SysExit" , STAMUNIT_OCCURENCES, "Number of times sysexit was emulated.");
299 STAM_REG(pVM, &pVM->patm.s.StatEmulIret, STAMTYPE_COUNTER, "/PATM/Emul/Iret/Success", STAMUNIT_OCCURENCES, "Number of times iret was emulated.");
300 STAM_REG(pVM, &pVM->patm.s.StatEmulIretFailed, STAMTYPE_COUNTER, "/PATM/Emul/Iret/Failed", STAMUNIT_OCCURENCES, "Number of times iret was emulated.");
302 STAM_REG(pVM, &pVM->patm.s.StatGenRet, STAMTYPE_COUNTER, "/PATM/Gen/Ret" , STAMUNIT_OCCURENCES, "Number of generated ret instructions.");
303 STAM_REG(pVM, &pVM->patm.s.StatGenRetReused, STAMTYPE_COUNTER, "/PATM/Gen/RetReused" , STAMUNIT_OCCURENCES, "Number of reused ret instructions.");
304 STAM_REG(pVM, &pVM->patm.s.StatGenCall, STAMTYPE_COUNTER, "/PATM/Gen/Call", STAMUNIT_OCCURENCES, "Number of generated call instructions.");
305 STAM_REG(pVM, &pVM->patm.s.StatGenJump, STAMTYPE_COUNTER, "/PATM/Gen/Jmp" , STAMUNIT_OCCURENCES, "Number of generated indirect jump instructions.");
306 STAM_REG(pVM, &pVM->patm.s.StatGenPopf, STAMTYPE_COUNTER, "/PATM/Gen/Popf" , STAMUNIT_OCCURENCES, "Number of generated popf instructions.");
308 STAM_REG(pVM, &pVM->patm.s.StatCheckPendingIRQ, STAMTYPE_COUNTER, "/PATM/GC/CheckIRQ" , STAMUNIT_OCCURENCES, "Number of traps that ask to check for pending irqs.");
344 int rc = PGMMapSetPage(pVM, pVM->patm.s.pGCStateGC, PAGE_SIZE, X86_PTE_P | X86_PTE_A | X86_PTE_D | X86_PTE_RW);
347 rc = PGMMapSetPage(pVM, pVM->patm.s.pGCStackGC, PATM_STACK_TOTAL_SIZE, X86_PTE_P | X86_PTE_A | X86_PTE_D | X86_PTE_RW);
350 rc = PGMMapSetPage(pVM, pVM->patm.s.pStatsGC, PATM_STAT_MEMSIZE, X86_PTE_P | X86_PTE_A | X86_PTE_D | X86_PTE_RW);
356 rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_PatchHlpBegin", &pVM->patm.s.pbPatchHelpersRC);
358 pVM->patm.s.pbPatchHelpersR3 = (uint8_t *)MMHyperRCToR3(pVM, pVM->patm.s.pbPatchHelpersRC);
359 AssertLogRelReturn(pVM->patm.s.pbPatchHelpersR3 != NULL, VERR_INTERNAL_ERROR_3);
365 pVM->patm.s.cbPatchHelpers = RCPtrEnd - pVM->patm.s.pbPatchHelpersRC;
366 AssertLogRelMsgReturn(pVM->patm.s.cbPatchHelpers < _128K,
367 ("%RRv-%RRv => %#x\n", pVM->patm.s.pbPatchHelpersRC, RCPtrEnd, pVM->patm.s.cbPatchHelpers),
384 AssertRelease(!(RT_OFFSETOF(VM, patm.s) & 31));
385 AssertRelease(sizeof(pVM->patm.s) <= sizeof(pVM->patm.padding));
390 pVM->patm.s.offVM = RT_OFFSETOF(VM, patm);
398 Assert(pVM->patm.s.pGCStateHC);
399 memset(pVM->patm.s.pGCStateHC, 0, PAGE_SIZE);
400 AssertReleaseMsg(pVM->patm.s.pGCStateGC, ("Impossible! MMHyperHC2GC(%p) failed!\n", pVM->patm.s.pGCStateGC));
402 Log(("Patch memory allocated at %p - %RRv\n", pVM->patm.s.pPatchMemHC, pVM->patm.s.pPatchMemGC));
403 pVM->patm.s.pGCStateHC->uVMFlags = X86_EFL_IF;
405 Assert(pVM->patm.s.pGCStackHC);
406 memset(pVM->patm.s.pGCStackHC, 0, PAGE_SIZE);
407 AssertReleaseMsg(pVM->patm.s.pGCStackGC, ("Impossible! MMHyperHC2GC(%p) failed!\n", pVM->patm.s.pGCStackGC));
408 pVM->patm.s.pGCStateHC->Psp = PATM_STACK_SIZE;
409 pVM->patm.s.pGCStateHC->fPIF = 1; /* PATM Interrupt Flag */
411 Assert(pVM->patm.s.pStatsHC);
412 memset(pVM->patm.s.pStatsHC, 0, PATM_STAT_MEMSIZE);
413 AssertReleaseMsg(pVM->patm.s.pStatsGC, ("Impossible! MMHyperHC2GC(%p) failed!\n", pVM->patm.s.pStatsGC));
415 Assert(pVM->patm.s.pPatchMemHC);
416 Assert(pVM->patm.s.pPatchMemGC == MMHyperR3ToRC(pVM, pVM->patm.s.pPatchMemHC));
417 memset(pVM->patm.s.pPatchMemHC, 0, PATCH_MEMORY_SIZE);
418 AssertReleaseMsg(pVM->patm.s.pPatchMemGC, ("Impossible! MMHyperHC2GC(%p) failed!\n", pVM->patm.s.pPatchMemHC));
421 pVM->patm.s.pCPUMCtxGC = VM_RC_ADDR(pVM, CPUMQueryGuestCtxPtr(VMMGetCpu(pVM)));
423 Assert(pVM->patm.s.PatchLookupTreeHC);
424 Assert(pVM->patm.s.PatchLookupTreeGC == MMHyperR3ToRC(pVM, pVM->patm.s.PatchLookupTreeHC));
429 Assert(!pVM->patm.s.PatchLookupTreeHC->PatchTree);
430 Assert(!pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr);
431 Assert(!pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage);
432 pVM->patm.s.offPatchMem = 16; /* don't start with zero here */
433 pVM->patm.s.uCurrentPatchIdx = 1; /* Index zero is a dummy */
434 pVM->patm.s.pvFaultMonitor = 0;
435 pVM->patm.s.deltaReloc = 0;
438 pVM->patm.s.pPatchedInstrGCLowest = ~0;
439 pVM->patm.s.pPatchedInstrGCHighest = 0;
441 pVM->patm.s.PatchLookupTreeHC->PatchTree = 0;
442 pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr = 0;
443 pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage = 0;
445 pVM->patm.s.pfnSysEnterPatchGC = 0;
446 pVM->patm.s.pfnSysEnterGC = 0;
448 pVM->patm.s.fOutOfMemory = false;
450 pVM->patm.s.pfnHelperCallGC = 0;
455 rc = MMHyperAlloc(pVM, sizeof(PATMPATCHREC), 0, MM_TAG_PATM_PATCH, (void **)&pVM->patm.s.pGlobalPatchRec);
461 pVM->patm.s.pGlobalPatchRec->patch.flags = PATMFL_GLOBAL_FUNCTIONS;
462 pVM->patm.s.pGlobalPatchRec->patch.uState = PATCH_ENABLED;
463 pVM->patm.s.pGlobalPatchRec->patch.pPatchBlockOffset = pVM->patm.s.offPatchMem;
465 rc = patmPatchGenGlobalFunctions(pVM, &pVM->patm.s.pGlobalPatchRec->patch);
469 pVM->patm.s.offPatchMem += pVM->patm.s.pGlobalPatchRec->patch.uCurPatchOffset;
471 pVM->patm.s.offPatchMem = RT_ALIGN_32(pVM->patm.s.offPatchMem, 8);
493 RTRCPTR GCPtrNew = MMHyperR3ToRC(pVM, pVM->patm.s.pGCStateHC);
494 Assert((RTRCINTPTR)(GCPtrNew - pVM->patm.s.pGCStateGC) == offDelta);
496 Log(("PATMR3Relocate from %RRv to %RRv - delta %08X\n", pVM->patm.s.pGCStateGC, GCPtrNew, offDelta));
502 pVM->patm.s.pCPUMCtxGC += offDelta;
504 pVM->patm.s.deltaReloc = offDelta;
505 RTAvloU32DoWithAll(&pVM->patm.s.PatchLookupTreeHC->PatchTree, true, patmR3RelocatePatches, (void *)pVM);
507 pVM->patm.s.pGCStateGC = GCPtrNew;
508 pVM->patm.s.pPatchMemGC = MMHyperR3ToRC(pVM, pVM->patm.s.pPatchMemHC);
509 pVM->patm.s.pGCStackGC = MMHyperR3ToRC(pVM, pVM->patm.s.pGCStackHC);
510 pVM->patm.s.pStatsGC = MMHyperR3ToRC(pVM, pVM->patm.s.pStatsHC);
511 pVM->patm.s.PatchLookupTreeGC = MMHyperR3ToRC(pVM, pVM->patm.s.PatchLookupTreeHC);
513 if (pVM->patm.s.pfnSysEnterPatchGC)
514 pVM->patm.s.pfnSysEnterPatchGC += offDelta;
522 pVM->patm.s.pfnHelperCallGC += offDelta;
523 pVM->patm.s.pfnHelperRetGC += offDelta;
524 pVM->patm.s.pfnHelperIretGC += offDelta;
525 pVM->patm.s.pfnHelperJumpGC += offDelta;
527 pVM->patm.s.pbPatchHelpersRC += offDelta;
529 patmR3RelocatePatches(&pVM->patm.s.pGlobalPatchRec->Core, (void *)pVM);
570 PPATMPATCHREC pPatchRec = (PPATMPATCHREC)RTAvloU32RemoveBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTree, 0, true);
576 Assert(!pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage);
577 Assert(!pVM->patm.s.PatchLookupTreeHC->PatchTree);
578 pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr = 0;
579 pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage = 0;
768 delta = (RTRCINTPTR)pVM->patm.s.deltaReloc;
981 pVM->patm.s.pvFaultMonitor = GCPtr;
1034 return pVM->patm.s.pGCStateHC;
1084 RTRCUINTPTR offPatch = (RTRCUINTPTR)pAddrGC - (RTRCUINTPTR)pVM->patm.s.pPatchMemGC;
1085 if (offPatch >= pVM->patm.s.cbPatchMem)
1087 offPatch = (RTRCUINTPTR)pAddrGC - (RTRCUINTPTR)pVM->patm.s.pbPatchHelpersRC;
1088 if (offPatch >= pVM->patm.s.cbPatchHelpers)
1090 return pVM->patm.s.pbPatchHelpersR3 + offPatch;
1092 return pVM->patm.s.pPatchMemHC + offPatch;
1113 offset = (RTRCUINTPTR)pGCPtr - (RTRCUINTPTR)pVM->patm.s.pPatchMemGC;
1114 if (offset < pVM->patm.s.cbPatchMem)
1120 return pVM->patm.s.pPatchMemHC + offset;
1209 pPatchHC = pVM->patm.s.pPatchMemHC + (pPatchGC - pVM->patm.s.pPatchMemGC);
1294 uint32_t PatchOffset = pPatchInstrHC - pVM->patm.s.pPatchMemHC; /* Offset in memory reserved for PATM. */
1356 uint32_t PatchOffset = pPatchInstrGC - pVM->patm.s.pPatchMemGC; /* Offset in memory reserved for PATM. */
2269 delta = pVM->patm.s.pPatchMemGC - (uintptr_t)pVM->patm.s.pPatchMemHC;
2824 orgOffsetPatchMem = pVM->patm.s.offPatchMem;
2855 STAM_COUNTER_INC(&pVM->patm.s.StatPageBoundaryCrossed);
2868 pPatch->pPatchBlockOffset = pVM->patm.s.offPatchMem;
2925 pVM->patm.s.offPatchMem += pPatch->cbPatchBlockSize;
2927 pVM->patm.s.offPatchMem = RT_ALIGN_32(pVM->patm.s.offPatchMem, 8);
2934 fInserted = RTAvloU32Insert(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, &pPatchRec->CoreOffset);
2982 STAM_COUNTER_INC(&pVM->patm.s.StatInt3Callable);
3010 RTAvloU32Remove(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, pPatchRec->CoreOffset.Key);
3027 pVM->patm.s.offPatchMem = orgOffsetPatchMem;
3078 PPATMPATCHREC pJmpPatch = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pJmpInstrGC);
3085 pJmpPatch = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pJmpInstrGC);
3092 orgOffsetPatchMem = pVM->patm.s.offPatchMem;
3094 pPatch->pPatchBlockOffset = pVM->patm.s.offPatchMem;
3126 pVM->patm.s.offPatchMem += pPatch->cbPatchBlockSize;
3128 pVM->patm.s.offPatchMem = RT_ALIGN_32(pVM->patm.s.offPatchMem, 8);
3146 fInserted = RTAvloU32Insert(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, &pPatchRec->CoreOffset);
3158 pVM->patm.s.offPatchMem = orgOffsetPatchMem;
3181 orgOffsetPatchMem = pVM->patm.s.offPatchMem;
3183 pPatch->pPatchBlockOffset = pVM->patm.s.offPatchMem;
3201 pVM->patm.s.offPatchMem += pPatch->cbPatchBlockSize;
3203 pVM->patm.s.offPatchMem = RT_ALIGN_32(pVM->patm.s.offPatchMem, 8);
3221 fInserted = RTAvloU32Insert(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, &pPatchRec->CoreOffset);
3237 pVM->patm.s.offPatchMem = orgOffsetPatchMem;
3256 pRec = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pInstrGC);
3283 orgOffsetPatchMem = pVM->patm.s.offPatchMem;
3286 if (pVM->patm.s.ulCallDepth > PATM_MAX_CALL_DEPTH)
3292 pVM->patm.s.ulCallDepth++;
3301 pPatch->pPatchBlockOffset = pVM->patm.s.offPatchMem;
3326 pVM->patm.s.offPatchMem += pPatch->cbPatchBlockSize;
3328 pVM->patm.s.offPatchMem = RT_ALIGN_32(pVM->patm.s.offPatchMem, 8);
3337 fInserted = RTAvloU32Insert(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, &pPatchRec->CoreOffset);
3366 pVM->patm.s.ulCallDepth--;
3367 STAM_COUNTER_INC(&pVM->patm.s.StatInstalledFunctionPatches);
3372 RTAvloU32Remove(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, pPatchRec->CoreOffset.Key);
3389 pVM->patm.s.offPatchMem = orgOffsetPatchMem;
3391 pVM->patm.s.ulCallDepth--;
3417 orgOffsetPatchMem = pVM->patm.s.offPatchMem;
3423 PPATMPATCHPAGE pPatchPage = (PPATMPATCHPAGE)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage, (RTRCPTR)pPage);
3440 uint32_t offsetPatch = pPatchTargetGC - pVM->patm.s.pPatchMemGC;
3475 pPatch->pPatchBlockOffset = pVM->patm.s.offPatchMem;
3498 fInserted = RTAvloU32Insert(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, &pPatchRec->CoreOffset);
3511 pVM->patm.s.offPatchMem += pPatch->cbPatchBlockSize;
3513 pVM->patm.s.offPatchMem = RT_ALIGN_32(pVM->patm.s.offPatchMem, 8);
3528 STAM_COUNTER_INC(&pVM->patm.s.StatInstalledTrampoline);
3533 RTAvloU32Remove(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, pPatchRec->CoreOffset.Key);
3550 pVM->patm.s.offPatchMem = orgOffsetPatchMem;
3581 PPATMPATCHPAGE pPatchPage = (PPATMPATCHPAGE)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage, (RTRCPTR)pPage);
3598 STAM_COUNTER_INC(&pVM->patm.s.StatDuplicateUseExisting);
3625 pCtx->eax = pCtx->eax - (RTRCUINTPTR)pVM->patm.s.pPatchMemGC; /* make it relative */
3631 STAM_COUNTER_INC(&pVM->patm.s.StatDuplicateREQFailed);
3638 STAM_COUNTER_INC(&pVM->patm.s.StatDuplicateREQSuccess);
3679 pPatchFunction = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pTargetGC);
3704 pPatchFunction = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pTargetGC);
3762 Assert(pVM->patm.s.mmio.pCachedData);
3763 if (!pVM->patm.s.mmio.pCachedData)
3775 pVM->patm.s.mmio.pCachedData) != VINF_SUCCESS)
3790 &pVM->patm.s.mmio.pCachedData, sizeof(RTRCPTR));
3798 pVM->patm.s.mmio.pCachedData = 0;
3799 pVM->patm.s.mmio.GCPhys = 0;
3829 AssertReturn(pVM->patm.s.mmio.pCachedData, VERR_INVALID_PARAMETER);
3857 *(RTRCPTR *)&pInstrHC[cpu.cbInstr - sizeof(RTRCPTR)] = pVM->patm.s.mmio.pCachedData;
3864 pVM->patm.s.mmio.pCachedData = 0;
3865 pVM->patm.s.mmio.GCPhys = 0;
4018 STAM_COUNTER_INC(&pVM->patm.s.StatPageBoundaryCrossed);
4068 STAM_COUNTER_INC(&pVM->patm.s.StatInstalledJump);
4155 if (pVM->patm.s.fOutOfMemory == true)
4209 offset = pInstrGC - pVM->patm.s.pPatchMemGC;
4210 pvPatchCoreOffset = RTAvloU32GetBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, offset, false);
4223 pPatchRec = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pInstrGC);
4263 STAM_COUNTER_INC(&pVM->patm.s.StatOverwritten);
4321 fInserted = RTAvloU32Insert(&pVM->patm.s.PatchLookupTreeHC->PatchTree, &pPatchRec->Core);
4337 PPATMPATCHREC pPatchNear = (PPATMPATCHREC)RTAvloU32GetBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTree, (pInstrGC + SIZEOF_NEARJUMP32 - 1), false);
4381 if (pVM->patm.s.uCurrentPatchIdx < PATM_STAT_MAX_COUNTERS)
4383 pPatchRec->patch.uPatchIdx = pVM->patm.s.uCurrentPatchIdx++;
4431 pVM->patm.s.uSysEnterPatchIdx = pPatchRec->patch.uPatchIdx;
4536 pVM->patm.s.uCurrentPatchIdx--;
4544 if (pPatchRec->patch.pInstrGCLowest < pVM->patm.s.pPatchedInstrGCLowest)
4545 pVM->patm.s.pPatchedInstrGCLowest = pPatchRec->patch.pInstrGCLowest;
4546 if (pPatchRec->patch.pInstrGCHighest > pVM->patm.s.pPatchedInstrGCHighest)
4547 pVM->patm.s.pPatchedInstrGCHighest = pPatchRec->patch.pInstrGCHighest;
4550 Log(("Global lowest %RRv highest %RRv\n", pVM->patm.s.pPatchedInstrGCLowest, pVM->patm.s.pPatchedInstrGCHighest));
4552 STAM_COUNTER_ADD(&pVM->patm.s.StatInstalled, 1);
4553 STAM_COUNTER_ADD(&pVM->patm.s.StatPATMMemoryUsed, pPatchRec->patch.cbPatchBlockSize);
4572 STAMR3RegisterF(pVM, &pVM->patm.s.pStatsHC[pPatchRec->patch.uPatchIdx], STAMTYPE_RATIO_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_GOOD_BAD, PATMPatchType(pVM, &pPatchRec->patch),
4645 pPatchPage = (PPATMPATCHPAGE)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage, pPage);
4692 fInserted = RTAvloU32Insert(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage, &pPatchPage->Core);
4694 pVM->patm.s.cPageRecords++;
4696 STAM_COUNTER_INC(&pVM->patm.s.StatPatchPageInserted);
4769 pPatchPage = (PPATMPATCHPAGE)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage, pPage);
4803 STAM_COUNTER_INC(&pVM->patm.s.StatPatchPageRemoved);
4804 pPatchNode = (PPATMPATCHPAGE)RTAvloU32Remove(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage, pPage);
4812 pVM->patm.s.cPageRecords--;
4903 if ( GCPtr < pVM->patm.s.pPatchedInstrGCLowest
4904 || GCPtr > pVM->patm.s.pPatchedInstrGCHighest
4908 STAM_PROFILE_ADV_START(&pVM->patm.s.StatPatchWrite, a);
4916 PPATMPATCHPAGE pPatchPage = (PPATMPATCHPAGE)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage, (RTRCPTR)pPage);
4977 uint32_t PatchOffset = pPatchInstrGC - pVM->patm.s.pPatchMemGC; /* Offset in memory reserved for PATM. */
5006 STAM_COUNTER_INC(&pVM->patm.s.StatInstrDirty);
5022 pPatchPage = (PPATMPATCHPAGE)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage, (RTRCPTR)pPage);
5054 STAM_PROFILE_ADV_STOP(&pVM->patm.s.StatPatchWrite, a);
5074 PPATMPATCHPAGE pPatchPage = (PPATMPATCHPAGE)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPage, addr);
5090 STAM_COUNTER_INC(&pVM->patm.s.StatFlushed);
5106 pPatchRec = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pInstrGC);
5131 if ( pInstrGC < pVM->patm.s.pPatchedInstrGCLowest
5132 || pInstrGC > pVM->patm.s.pPatchedInstrGCHighest)
5135 pPatchRec = (PPATMPATCHREC)RTAvloU32GetBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pInstrGC, false);
5149 STAM_COUNTER_ADD(&pVM->patm.s.StatNrOpcodeRead, 1);
5172 if ( GCPtrInstr < pVM->patm.s.pPatchedInstrGCLowest
5173 || GCPtrInstr > pVM->patm.s.pPatchedInstrGCHighest)
5183 PPATMPATCHREC pPatchRec = (PPATMPATCHREC)RTAvloU32GetBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTree,
5208 STAM_COUNTER_ADD(&pVM->patm.s.StatNrOpcodeRead, 1);
5232 pPatchRec = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pInstrGC);
5292 STAM_COUNTER_INC(&pVM->patm.s.StatOverwritten);
5328 STAM_COUNTER_INC(&pVM->patm.s.StatOverwritten);
5353 STAM_COUNTER_ADD(&pVM->patm.s.StatDisabled, 1);
5413 STAM_COUNTER_INC(&pVM->patm.s.StatFixedConflicts);
5437 STAM_COUNTER_INC(&pVM->patm.s.StatInt3Callable);
5458 STAM_COUNTER_INC(&pVM->patm.s.StatUnusable);
5479 pPatchRec = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pInstrGC);
5503 STAM_COUNTER_INC(&pVM->patm.s.StatOverwritten);
5557 STAM_COUNTER_INC(&pVM->patm.s.StatOverwritten);
5574 STAM_COUNTER_ADD(&pVM->patm.s.StatEnabled, 1);
5613 pNode = RTAvloU32Remove(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, pPatchRec->patch.pPatchBlockOffset);
5644 RTAvloU32Remove(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pPatchRec->Core.Key);
5667 uint8_t *pPatchInstr = (uint8_t *)(pVM->patm.s.pPatchMemHC + pPatch2GuestRec->Core.Key);
5679 int32_t displ = pPatchTargetGC - (pVM->patm.s.pPatchMemGC + pPatch2GuestRec->Core.Key + SIZEOF_NEARJUMP32);
5735 RTAvloU32Remove(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pPatchRec->Core.Key);
5770 pNewPatchRec = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pInstrGC);
5778 bool fInserted = RTAvloU32Insert(&pVM->patm.s.PatchLookupTreeHC->PatchTree, &pNewPatchRec->Core);
5782 STAM_COUNTER_INC(&pVM->patm.s.StatPatchRefreshSuccess);
5830 bool fInserted = RTAvloU32Insert(&pVM->patm.s.PatchLookupTreeHC->PatchTree, &pPatchRec->Core);
5837 STAM_COUNTER_INC(&pVM->patm.s.StatPatchRefreshFailed);
5853 PPATMPATCHREC pPatchRec = (PPATMPATCHREC)RTAvloU32GetBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pInstrGC, false);
5923 pPatchRec = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pInstrGC);
5955 STAM_COUNTER_INC(&pVM->patm.s.StatDirty);
5973 CTXSUFF(pVM->patm.s.pGCState)->Psp = PATM_STACK_SIZE;
5990 PRECPATCHTOGUEST pPatchToGuestRec = (PRECPATCHTOGUEST)RTAvlU32GetBestFit(&pPatch->Patch2GuestAddrTree, pPatchGC - pVM->patm.s.pPatchMemGC, false);
6012 return pVM->patm.s.pPatchMemGC + pGuestToPatchRec->PatchOffset;
6027 PPATMPATCHREC pPatchRec = (PPATMPATCHREC)RTAvloU32GetBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pInstrGC, false);
6047 return pVM->patm.s.pPatchMemGC + pGuestToPatchRec->PatchOffset;
6068 pvPatchCoreOffset = RTAvloU32GetBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, pPatchGC - pVM->patm.s.pPatchMemGC, false);
6071 Log(("PATMR3PatchToGCPtr failed for %RRv offset %x\n", pPatchGC, pPatchGC - pVM->patm.s.pPatchMemGC));
6092 if (pVM->patm.s.pGCStateHC->GCPtrInhibitInterrupts == pPrivInstrGC)
6134 pPatchRec = (PPATMPATCHREC)RTAvloU32Get(&pVM->patm.s.PatchLookupTreeHC->PatchTree, pAddrGC);
6213 pRec = (PRECPATCHTOGUEST)RTAvlU32GetBestFit(&pPatch->patch.Patch2GuestAddrTree, pCurPatchInstrGC - pVM->patm.s.pPatchMemGC, true);
6218 pCurPatchInstrGC = pRec->Core.Key + pVM->patm.s.pPatchMemGC;
6297 if (RTAvlU32Get(&pPatch->patch.Patch2GuestAddrTree, pCurPatchInstrGC - pVM->patm.s.pPatchMemGC) == NULL)
6299 pRec = (PRECPATCHTOGUEST)RTAvlU32GetBestFit(&pPatch->patch.Patch2GuestAddrTree, pCurPatchInstrGC - pVM->patm.s.pPatchMemGC, true);
6302 unsigned cbFiller = pRec->Core.Key + pVM->patm.s.pPatchMemGC - pCurPatchInstrGC;
6342 STAM_COUNTER_INC(&pVM->patm.s.StatInstrDirtyGood);
6346 STAM_COUNTER_INC(&pVM->patm.s.StatInstrDirtyBad);
6353 if ( pVM->patm.s.fOutOfMemory == false
6393 STAM_PROFILE_ADV_START(&pVM->patm.s.StatHandleTrap, a);
6397 offset = pEip - pVM->patm.s.pPatchMemGC;
6398 pvPatchCoreOffset = RTAvloU32GetBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTreeByPatchAddr, offset, false);
6411 pVM->patm.s.pGCStateHC->fPIF = 1;
6421 pVM->patm.s.pGCStateHC->fPIF = 1;
6443 AssertReleaseMsg(pVM->patm.s.pGCStateHC->fPIF == 0, ("PATMR3HandleTrap: Unable to find translation record for %RRv (PIF=0)\n", pEip));
6446 if (pVM->patm.s.pGCStateHC->fPIF == 0)
6460 STAM_COUNTER_INC(&pVM->patm.s.StatPushTrap);
6471 CTXSUFF(pVM->patm.s.pGCState)->Psp = PATM_STACK_SIZE;
6473 pVM->patm.s.pGCStateHC->fPIF = 1;
6479 STAM_PROFILE_ADV_STOP(&pVM->patm.s.StatHandleTrap, a);
6494 STAM_PROFILE_ADV_STOP(&pVM->patm.s.StatHandleTrap, a);
6506 CTXSUFF(pVM->patm.s.pGCState)->Psp = PATM_STACK_SIZE;
6510 pVM->patm.s.pGCStateHC->fPIF = 1;
6514 STAM_PROFILE_ADV_STOP(&pVM->patm.s.StatHandleTrap, a);
6524 AssertLogRelMsg(pVM->patm.s.pGCStateHC->fPIF == 1,
6530 AssertLogRelMsg(pVM->patm.s.pGCStateHC->fPIF == 1,
6538 STAM_PROFILE_ADV_STOP(&pVM->patm.s.StatHandleTrap, a);
6547 Assert(pVM->patm.s.pGCStateHC->fPIF == 1);
6559 CTXSUFF(pVM->patm.s.pGCState)->Psp = PATM_STACK_SIZE;
6565 STAM_PROFILE_ADV_STOP(&pVM->patm.s.StatHandleTrap, a);
6595 Log(("Expected return address %RRv found address %RRv Psp=%x\n", pVM->patm.s.pGCStackHC[(pVM->patm.s.pGCStateHC->Psp+PATM_STACK_SIZE)/sizeof(RTRCPTR)], retaddr, pVM->patm.s.pGCStateHC->Psp));
6604 CTXSUFF(pVM->patm.s.pGCState)->Psp = PATM_STACK_SIZE;
6606 if (pVM->patm.s.pGCStateHC->GCPtrInhibitInterrupts == pNewEip)
6634 pVM->patm.s.pGCStateHC->GCPtrInhibitInterrupts = 0;
6644 STAM_PROFILE_ADV_STOP(&pVM->patm.s.StatHandleTrap, a);
6656 STAM_PROFILE_ADV_STOP(&pVM->patm.s.StatHandleTrap, a);
6661 STAM_PROFILE_ADV_STOP(&pVM->patm.s.StatHandleTrap, a);
6676 RTRCPTR addr = pVM->patm.s.pvFaultMonitor;
6682 PPATMPATCHREC pPatchRec = (PPATMPATCHREC)RTAvloU32GetBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTree, addr, false);
6685 STAM_COUNTER_INC(&pVM->patm.s.StatMonitored);
6699 pPatchRec = (PPATMPATCHREC)RTAvloU32GetBestFit(&pVM->patm.s.PatchLookupTreeHC->PatchTree, addr, true);
6706 STAM_COUNTER_INC(&pVM->patm.s.StatMonitored);
6714 pVM->patm.s.pvFaultMonitor = 0;
6786 pVM->patm.s.pStatsHC[pPatch->uPatchIdx].u32A = 0;
6787 pVM->patm.s.pStatsHC[pPatch->uPatchIdx].u32B = 0;
6808 pVM->patm.s.pStatsHC[pPatch->uPatchIdx].u32A, pVM->patm.s.pStatsHC[pPatch->uPatchIdx].u32B);
6821 return pVM->patm.s.pStatsGC + sizeof(STAMRATIOU32) * pPatch->uPatchIdx + RT_OFFSETOF(STAMRATIOU32, u32A);
6850 RTAvloU32DoWithAll(&pVM->patm.s.PatchLookupTreeHC->PatchTree, true, DisableAllPatches, pVM);
6879 RTAvloU32DoWithAll(&pVM->patm.s.PatchLookupTreeHC->PatchTree, true, EnableAllPatches, pVM);