Lines Matching refs:pVCpu
49 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
51 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
53 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
107 int rc = hmR0SvmCheckExitDueToEventDelivery(pVCpu, pCtx, pSvmTransient); \
123 #define HMSVM_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
128 #define HMSVM_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
129 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
131 pVCpu->hm.s.idEnteredCpu, RTMpCpuId()));
257 * @param pVCpu Pointer to the VMCPU.
261 typedef int FNSVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
266 static void hmR0SvmSetMsrPermission(PVMCPU pVCpu, unsigned uMsr, SVMMSREXITREAD enmRead, SVMMSREXITWRITE enmWrite);
267 static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu);
268 static void hmR0SvmLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
305 DECLINLINE(int) hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient);
472 PVMCPU pVCpu = &pVM->aCpus[i];
473 AssertPtr(pVCpu);
475 if (pVCpu->hm.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
477 RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcbHost, false);
478 pVCpu->hm.s.svm.pvVmcbHost = 0;
479 pVCpu->hm.s.svm.HCPhysVmcbHost = 0;
480 pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
483 if (pVCpu->hm.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
485 RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcb, false);
486 pVCpu->hm.s.svm.pvVmcb = 0;
487 pVCpu->hm.s.svm.HCPhysVmcb = 0;
488 pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
491 if (pVCpu->hm.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
493 RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjMsrBitmap, false);
494 pVCpu->hm.s.svm.pvMsrBitmap = 0;
495 pVCpu->hm.s.svm.HCPhysMsrBitmap = 0;
496 pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
529 PVMCPU pVCpu = &pVM->aCpus[i];
530 pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
531 pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
532 pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
537 PVMCPU pVCpu = &pVM->aCpus[i];
543 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcbHost, 1 << PAGE_SHIFT, false /* fExecutable */);
547 pVCpu->hm.s.svm.pvVmcbHost = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcbHost);
548 pVCpu->hm.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcbHost, 0 /* iPage */);
549 Assert(pVCpu->hm.s.svm.HCPhysVmcbHost < _4G);
550 ASMMemZeroPage(pVCpu->hm.s.svm.pvVmcbHost);
555 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcb, 1 << PAGE_SHIFT, false /* fExecutable */);
559 pVCpu->hm.s.svm.pvVmcb = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcb);
560 pVCpu->hm.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcb, 0 /* iPage */);
561 Assert(pVCpu->hm.s.svm.HCPhysVmcb < _4G);
562 ASMMemZeroPage(pVCpu->hm.s.svm.pvVmcb);
568 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjMsrBitmap, 2 << PAGE_SHIFT, false /* fExecutable */);
572 pVCpu->hm.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjMsrBitmap);
573 pVCpu->hm.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
575 ASMMemFill32(pVCpu->hm.s.svm.pvMsrBitmap, 2 << PAGE_SHIFT, UINT32_C(0xffffffff));
602 * @param pVCpu Pointer to the VMCPU.
607 static void hmR0SvmSetMsrPermission(PVMCPU pVCpu, unsigned uMsr, SVMMSREXITREAD enmRead, SVMMSREXITWRITE enmWrite)
610 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
656 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
676 PVMCPU pVCpu = &pVM->aCpus[i];
682 Assert(!pVCpu->hm.s.idxExitHistoryFree);
683 HMCPU_EXIT_HISTORY_RESET(pVCpu);
750 pVmcb->ctrl.u64MSRPMPhysAddr = pVCpu->hm.s.svm.HCPhysMsrBitmap;
791 if (pVCpu->hm.s.fGIMTrapXcptUD)
798 hmR0SvmSetMsrPermission(pVCpu, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
799 hmR0SvmSetMsrPermission(pVCpu, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
800 hmR0SvmSetMsrPermission(pVCpu, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
801 hmR0SvmSetMsrPermission(pVCpu, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
802 hmR0SvmSetMsrPermission(pVCpu, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
803 hmR0SvmSetMsrPermission(pVCpu, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
804 hmR0SvmSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
805 hmR0SvmSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
806 hmR0SvmSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
807 hmR0SvmSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
819 * @param pVCpu Pointer to the VMCPU.
822 VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
827 bool fFlushPending = pVM->hm.s.svm.fAlwaysFlushTLB || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
834 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
839 if (CPUMIsGuestInLongMode(pVCpu))
840 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
845 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
856 * @param pVCpu Pointer to the VMCPU.
858 static void hmR0SvmFlushTaggedTlb(PVMCPU pVCpu)
860 PVM pVM = pVCpu->CTX_SUFF(pVM);
861 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
872 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
873 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
875 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
876 pVCpu->hm.s.fForceTLBFlush = true;
881 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
884 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
886 pVCpu->hm.s.fForceTLBFlush = true;
887 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
898 pVCpu->hm.s.uCurrentAsid = 1;
899 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
905 else if (pVCpu->hm.s.fForceTLBFlush)
944 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
945 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
946 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
956 pVCpu->hm.s.fForceTLBFlush = false;
964 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
967 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
968 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
969 SVMR0InvlpgA(pVCpu->hm.s.TlbShootdown.aPages[i], pVmcb->ctrl.TLBCtrl.n.u32ASID);
971 pVCpu->hm.s.TlbShootdown.cPages = 0;
972 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
979 if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hm.s.uCurrentAsid)
981 pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hm.s.uCurrentAsid;
985 AssertMsg(pVCpu->hm.s.idLastCpu == pCpu->idCpu,
986 ("vcpu idLastCpu=%x pcpu idCpu=%x\n", pVCpu->hm.s.idLastCpu, pCpu->idCpu));
987 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
988 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
991 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
992 ("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
996 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1000 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1005 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1027 * @param pVCpu Pointer to the VMCPU.
1029 DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
1038 aParam[6] = VM_RC_ADDR(pVM, pVCpu);
1041 return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_SVMRCVMRun64, RT_ELEMENTS(aParam), &aParam[0]);
1050 * @param pVCpu Pointer to the VMCPU.
1056 VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
1067 CPUMR0SetLApic(pVCpu, idHostCpu);
1070 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
1071 CPUMSetHyperEIP(pVCpu, enmOp);
1073 CPUMPushHyper(pVCpu, paParam[i]);
1075 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
1077 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
1078 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
1137 static void hmR0SvmLoadSharedCR0(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1142 PVM pVM = pVCpu->CTX_SUFF(pVM);
1143 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
1165 if (CPUMIsGuestFPUStateActive(pVCpu))
1196 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
1205 * @param pVCpu Pointer to the VMCPU.
1211 static int hmR0SvmLoadGuestControlRegs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1213 PVM pVM = pVCpu->CTX_SUFF(pVM);
1218 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR2))
1222 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
1228 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
1240 pVmcb->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
1246 pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1249 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
1256 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
1261 switch (pVCpu->hm.s.enmShadowMode)
1297 pVCpu->hm.s.fLoadSaveGuestXcr0 = (u64GuestCR4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1299 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
1310 * @param pVCpu Pointer to the VMCPU.
1316 static void hmR0SvmLoadGuestSegmentRegs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1319 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
1330 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
1334 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
1337 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
1341 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
1344 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
1348 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
1353 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
1357 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
1362 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
1370 * @param pVCpu Pointer to the VMCPU.
1376 static void hmR0SvmLoadGuestMsrs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1388 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
1392 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
1426 * @param pVCpu Pointer to the VMCPU.
1433 static void hmR0SvmLoadSharedDebugState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1435 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
1448 bool const fStepping = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
1451 pVCpu->hm.s.fClearTrapFlag = true;
1458 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1469 && !CPUMIsHyperDebugStateActivePending(pVCpu))
1471 CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1472 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
1473 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
1477 if (!CPUMIsHyperDebugStateActive(pVCpu))
1479 CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1480 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1481 Assert(CPUMIsHyperDebugStateActive(pVCpu));
1486 || pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1488 pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1491 pVCpu->hm.s.fUsingHyperDR7 = true;
1511 pVCpu->hm.s.fUsingHyperDR7 = false;
1522 && !CPUMIsGuestDebugStateActivePending(pVCpu))
1524 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1525 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1526 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
1527 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
1531 if (!CPUMIsGuestDebugStateActive(pVCpu))
1533 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1534 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1535 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1536 Assert(CPUMIsGuestDebugStateActive(pVCpu));
1545 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
1546 && !CPUMIsGuestDebugStateActive(pVCpu))
1548 else if (!CPUMIsGuestDebugStateActive(pVCpu))
1584 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
1592 * @param pVCpu Pointer to the VMCPU.
1596 static int hmR0SvmLoadGuestApicState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1598 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE))
1603 int rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
1609 pVCpu->hm.s.svm.fSyncVTpr = false;
1612 if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
1618 hmR0SvmSetMsrPermission(pVCpu, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
1621 hmR0SvmSetMsrPermission(pVCpu, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1622 pVCpu->hm.s.svm.fSyncVTpr = true;
1636 pVCpu->hm.s.svm.fSyncVTpr = true;
1642 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
1651 * @param pVCpu Pointer to the VMCPU.
1655 static int hmR0SvmLoadGuestXcptIntercepts(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1658 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
1661 if (pVCpu->hm.s.fGIMTrapXcptUD)
1665 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
1675 * @param pVCpu Pointer to the VMCPU.
1680 static int hmR0SvmSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pCtx)
1687 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
1690 pVCpu->hm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
1693 pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun64;
1699 pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun;
1710 * @param pVCpu Pointer to the VMCPU.
1713 VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1716 AssertPtr(pVCpu);
1721 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1722 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
1724 pVCpu->hm.s.fLeaveDone = false;
1733 * @param pVCpu Pointer to the VMCPU.
1735 * @thread EMT(pVCpu)
1737 VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
1746 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
1747 VMCPU_ASSERT_EMT(pVCpu);
1749 PVM pVM = pVCpu->CTX_SUFF(pVM);
1750 PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1753 VMMRZCallRing3Disable(pVCpu);
1755 if (!pVCpu->hm.s.fLeaveDone)
1757 hmR0SvmLeave(pVM, pVCpu, pCtx);
1758 pVCpu->hm.s.fLeaveDone = true;
1762 int rc = HMR0LeaveCpu(pVCpu);
1766 VMMRZCallRing3Enable(pVCpu);
1767 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
1774 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
1775 VMCPU_ASSERT_EMT(pVCpu);
1778 VMMRZCallRing3Disable(pVCpu);
1784 int rc = HMR0EnterCpu(pVCpu);
1786 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
1788 pVCpu->hm.s.fLeaveDone = false;
1791 VMMRZCallRing3Enable(pVCpu);
1806 * @param pVCpu Pointer to the VMCPU.
1810 VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1813 NOREF(pVCpu);
1815 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
1829 * @param pVCpu Pointer to the VMCPU.
1834 static int hmR0SvmLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1836 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
1839 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
1841 int rc = hmR0SvmLoadGuestControlRegs(pVCpu, pVmcb, pCtx);
1842 AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestControlRegs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
1844 hmR0SvmLoadGuestSegmentRegs(pVCpu, pVmcb, pCtx);
1845 hmR0SvmLoadGuestMsrs(pVCpu, pVmcb, pCtx);
1852 rc = hmR0SvmLoadGuestApicState(pVCpu, pVmcb, pCtx);
1853 AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
1855 rc = hmR0SvmLoadGuestXcptIntercepts(pVCpu, pVmcb, pCtx);
1856 AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
1858 rc = hmR0SvmSetupVMRunHandler(pVCpu, pCtx);
1859 AssertLogRelMsgRCReturn(rc, ("hmR0SvmSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
1862 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP /* Unused (loaded unconditionally). */
1875 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
1876 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
1877 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
1880 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
1889 * @param pVCpu Pointer to the VMCPU.
1895 static void hmR0SvmLoadSharedState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1898 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1900 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
1901 hmR0SvmLoadSharedCR0(pVCpu, pVmcb, pCtx);
1903 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
1904 hmR0SvmLoadSharedDebugState(pVCpu, pVmcb, pCtx);
1907 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
1909 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
1910 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
1920 * @param pVCpu Pointer to the VMCPU.
1925 static void hmR0SvmSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1927 Assert(VMMRZCallRing3IsEnabled(pVCpu));
1929 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
1940 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
1941 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1942 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2035 if (!pVCpu->hm.s.fUsingHyperDR7)
2042 Assert(pVmcb->guest.u64DR7 == CPUMGetHyperDR7(pVCpu));
2043 CPUMSetHyperDR6(pVCpu, pVmcb->guest.u64DR6);
2050 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
2053 CPUMSetGuestCR3(pVCpu, pVmcb->guest.u64CR3);
2054 PGMUpdateCR3(pVCpu, pVmcb->guest.u64CR3);
2064 * @param pVCpu Pointer to the VMCPU.
2069 static void hmR0SvmLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2072 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2073 Assert(VMMR0IsLogFlushDisabled(pVCpu));
2081 if (CPUMIsGuestFPUStateActive(pVCpu))
2083 CPUMR0SaveGuestFPU(pVM, pVCpu, pCtx);
2084 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2085 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
2092 if (CPUMIsHyperDebugStateActive(pVCpu))
2094 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
2099 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */))
2100 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
2102 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2103 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2105 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
2106 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
2107 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2108 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2109 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
2111 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
2120 * @param pVCpu Pointer to the VMCPU.
2123 static int hmR0SvmLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2126 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2131 if (!pVCpu->hm.s.fLeaveDone)
2133 hmR0SvmLeave(pVM, pVCpu, pCtx);
2134 pVCpu->hm.s.fLeaveDone = true;
2143 VMMR0ThreadCtxHooksDeregister(pVCpu);
2146 int rc = HMR0LeaveCpu(pVCpu);
2158 * @param pVCpu Pointer to the VMCPU.
2163 static int hmR0SvmLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2165 return hmR0SvmLeaveSession(pVM, pVCpu, pCtx);
2174 * @param pVCpu Pointer to the VMCPU.
2179 DECLCALLBACK(int) hmR0SvmCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
2188 VMMRZCallRing3RemoveNotification(pVCpu);
2189 VMMRZCallRing3Disable(pVCpu);
2193 if (CPUMIsGuestFPUStateActive(pVCpu))
2194 CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
2197 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2200 VMMR0ThreadCtxHooksDeregister(pVCpu);
2203 HMR0LeaveCpu(pVCpu);
2209 Assert(pVCpu);
2211 Assert(VMMRZCallRing3IsEnabled(pVCpu));
2214 VMMRZCallRing3Disable(pVCpu);
2215 Assert(VMMR0IsLogFlushDisabled(pVCpu));
2218 int rc = hmR0SvmLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
2221 VMMRZCallRing3Enable(pVCpu);
2234 * @param pVCpu Pointer to the VMCPU.
2239 static void hmR0SvmExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rcExit)
2242 Assert(pVCpu);
2247 VMMRZCallRing3Disable(pVCpu);
2251 if (pVCpu->hm.s.Event.fPending)
2253 hmR0SvmPendingEventToTrpmTrap(pVCpu);
2254 Assert(!pVCpu->hm.s.Event.fPending);
2259 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
2260 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
2263 hmR0SvmLeaveSession(pVM, pVCpu, pCtx);
2264 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
2266 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2267 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
2276 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
2281 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
2283 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
2286 VMMRZCallRing3RemoveNotification(pVCpu);
2287 VMMRZCallRing3Enable(pVCpu);
2296 * @param pVCpu Pointer to the VMCPU.
2300 static void hmR0SvmUpdateTscOffsetting(PVM pVM, PVMCPU pVCpu)
2303 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
2304 bool fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVmcb->ctrl.u64TSCOffset, &fParavirtTsc);
2309 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
2315 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
2325 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
2333 * @param pVCpu Pointer to the VMCPU.
2341 DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPU pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
2343 Assert(!pVCpu->hm.s.Event.fPending);
2346 pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
2347 pVCpu->hm.s.Event.fPending = true;
2348 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
2353 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
2361 * @param pVCpu Pointer to the VMCPU.
2369 DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx, PSVMEVENT pEvent)
2371 NOREF(pVCpu); NOREF(pCtx);
2374 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
2386 * @param pVCpu Pointer to the VMCPU.
2388 static void hmR0SvmTrpmTrapToPendingEvent(PVMCPU pVCpu)
2390 Assert(TRPMHasTrap(pVCpu));
2391 Assert(!pVCpu->hm.s.Event.fPending);
2399 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
2440 rc = TRPMResetTrap(pVCpu);
2446 hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
2447 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
2457 static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu)
2459 Assert(pVCpu->hm.s.Event.fPending);
2460 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
2463 Event.u = pVCpu->hm.s.Event.u64IntInfo;
2489 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
2493 TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
2498 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
2499 Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
2506 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
2508 pVCpu->hm.s.Event.fPending = false;
2516 * @param pVCpu Pointer to the VMCPU.
2522 DECLINLINE(uint32_t) hmR0SvmGetGuestIntrShadow(PVMCPU pVCpu, PCPUMCTX pCtx)
2529 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2531 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2537 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2607 * @param pVCpu Pointer to the VMCPU.
2610 static void hmR0SvmEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
2612 Assert(!pVCpu->hm.s.Event.fPending);
2615 bool const fIntShadow = RT_BOOL(hmR0SvmGetGuestIntrShadow(pVCpu, pCtx));
2617 bool const fBlockNmi = RT_BOOL(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
2618 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
2623 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
2637 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2639 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
2642 else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)))
2653 int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
2662 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2667 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
2668 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
2681 * @param pVCpu Pointer to the VMCPU.
2684 static void hmR0SvmInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
2686 Assert(!TRPMHasTrap(pVCpu));
2687 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2689 bool const fIntShadow = RT_BOOL(hmR0SvmGetGuestIntrShadow(pVCpu, pCtx));
2691 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
2693 if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
2696 Event.u = pVCpu->hm.s.Event.u64IntInfo;
2709 hmR0SvmInjectEventVmcb(pVCpu, pVmcb, pCtx, &Event);
2710 pVCpu->hm.s.Event.fPending = false;
2714 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
2716 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
2730 * @param pVCpu Pointer to the VMCPU.
2736 static void hmR0SvmReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx)
2740 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
2744 HMDumpRegs(pVM, pVCpu, pCtx); NOREF(pVM);
2886 * @param pVCpu Pointer to the VMCPU.
2889 static int hmR0SvmCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2891 Assert(VMMRZCallRing3IsEnabled(pVCpu));
2894 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
2895 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
2897 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
2899 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
2903 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2905 int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2916 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
2918 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
2926 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
2968 * @param pVCpu Pointer to the VMCPU.
2972 static int hmR0SvmPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
2977 int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
2981 if (TRPMHasTrap(pVCpu))
2982 hmR0SvmTrpmTrapToPendingEvent(pVCpu);
2983 else if (!pVCpu->hm.s.Event.fPending)
2984 hmR0SvmEvaluatePendingEvent(pVCpu, pCtx);
2987 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
2991 rc = hmR0SvmLoadGuestState(pVM, pVCpu, pCtx);
2993 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
2999 if (pVCpu->hm.s.svm.fSyncVTpr)
3005 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
3015 VMMRZCallRing3Disable(pVCpu);
3029 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3032 VMMRZCallRing3Enable(pVCpu);
3033 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3039 VMMRZCallRing3Enable(pVCpu);
3040 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
3052 if (pVCpu->hm.s.Event.fPending)
3055 Event.u = pVCpu->hm.s.Event.u64IntInfo;
3059 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
3061 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3075 * @param pVCpu Pointer to the VMCPU.
3082 static void hmR0SvmPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
3084 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3085 Assert(VMMR0IsLogFlushDisabled(pVCpu));
3088 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
3089 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
3091 hmR0SvmInjectPendingEvent(pVCpu, pCtx);
3093 if ( pVCpu->hm.s.fPreloadGuestFpu
3094 && !CPUMIsGuestFPUStateActive(pVCpu))
3096 CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3097 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
3101 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
3102 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
3103 hmR0SvmLoadSharedState(pVCpu, pVmcb, pCtx);
3104 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT); /* Preemption might set this, nothing to do on AMD-V. */
3105 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
3110 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
3112 hmR0SvmUpdateTscOffsetting(pVM, pVCpu);
3117 if (idCurrentCpu != pVCpu->hm.s.idLastCpu)
3124 pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
3125 pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
3130 pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
3131 pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
3133 pSvmTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
3136 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
3137 hmR0SvmFlushTaggedTlb(pVCpu);
3138 Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
3140 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
3142 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
3154 hmR0SvmSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
3155 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
3156 uint64_t u64GuestTscAux = CPUMR0GetGuestTscAux(pVCpu);
3157 if (u64GuestTscAux != pVCpu->hm.s.u64HostTscAux)
3163 hmR0SvmSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
3178 * @param pVCpu Pointer to the VMCPU.
3183 DECLINLINE(int) hmR0SvmRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3191 return HMR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
3192 pVCpu->hm.s.svm.pfnVMRun);
3194 return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
3204 * @param pVCpu Pointer to the VMCPU.
3215 static void hmR0SvmPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient, int rcVMRun)
3217 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3219 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
3220 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
3222 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
3228 CPUMR0SetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
3229 if (u64GuestTscAuxMsr != pVCpu->hm.s.u64HostTscAux)
3230 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
3234 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVmcb->ctrl.u64TSCOffset);
3236 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
3237 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
3238 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
3242 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
3252 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmcb->ctrl.u64ExitCode); /* Update the #VMEXIT history array. */
3256 hmR0SvmSaveGuestState(pVCpu, pMixedCtx); /* Save the guest state from the VMCB to the guest-CPU context. */
3260 if (pVCpu->hm.s.svm.fSyncVTpr)
3266 int rc = PDMApicSetTPR(pVCpu, pMixedCtx->msrLSTAR & 0xff);
3268 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
3272 int rc = PDMApicSetTPR(pVCpu, pVmcb->ctrl.IntCtrl.n.u8VTPR << 4);
3274 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
3286 * @param pVCpu Pointer to the VMCPU.
3288 static int hmR0SvmRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3302 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
3303 rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
3312 hmR0SvmPreRunGuestCommitted(pVM, pVCpu, pCtx, &SvmTransient);
3313 rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
3317 hmR0SvmPostRunGuest(pVM, pVCpu, pCtx, &SvmTransient, rc);
3324 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
3325 hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3331 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
3332 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb);
3333 rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
3334 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
3339 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
3345 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
3355 * @param pVCpu Pointer to the VMCPU.
3358 static int hmR0SvmRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3370 AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
3371 ("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
3376 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
3377 rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
3386 VMMRZCallRing3Disable(pVCpu);
3387 VMMRZCallRing3RemoveNotification(pVCpu);
3388 hmR0SvmPreRunGuestCommitted(pVM, pVCpu, pCtx, &SvmTransient);
3390 rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
3396 hmR0SvmPostRunGuest(pVM, pVCpu, pCtx, &SvmTransient, rc);
3402 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
3403 hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3409 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
3410 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb);
3411 rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
3412 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
3417 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
3432 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
3438 if (pVCpu->hm.s.fClearTrapFlag)
3440 pVCpu->hm.s.fClearTrapFlag = false;
3444 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
3454 * @param pVCpu Pointer to the VMCPU.
3457 VMMR0DECL(int) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3459 Assert(VMMRZCallRing3IsEnabled(pVCpu));
3461 VMMRZCallRing3SetNotification(pVCpu, hmR0SvmCallRing3Callback, pCtx);
3464 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
3465 rc = hmR0SvmRunGuestCodeNormal(pVM, pVCpu, pCtx);
3467 rc = hmR0SvmRunGuestCodeStep(pVM, pVCpu, pCtx);
3475 hmR0SvmExitToRing3(pVM, pVCpu, pCtx, rc);
3476 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
3485 * @param pVCpu Pointer to the VMCPU.
3489 DECLINLINE(int) hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
3502 return hmR0SvmExitNestedPF(pVCpu, pCtx, pSvmTransient);
3505 return hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient);
3508 return hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient);
3511 return hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient);
3514 return hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient);
3517 return hmR0SvmExitXcptPF(pVCpu, pCtx, pSvmTransient);
3520 return hmR0SvmExitXcptNM(pVCpu, pCtx, pSvmTransient);
3523 return hmR0SvmExitXcptUD(pVCpu, pCtx, pSvmTransient);
3526 return hmR0SvmExitXcptMF(pVCpu, pCtx, pSvmTransient);
3529 return hmR0SvmExitXcptDB(pVCpu, pCtx, pSvmTransient);
3532 return hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient);
3535 return hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient);
3538 return hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient);
3543 return hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient);
3549 return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
3552 return hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient);
3555 return hmR0SvmExitVIntr(pVCpu, pCtx, pSvmTransient);
3560 return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
3563 return hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient);
3566 return hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient);
3569 return hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient);
3572 return hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient);
3575 return hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient);
3585 return hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
3591 return hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient);
3594 return hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient);
3597 return hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient);
3600 return hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient);
3603 return hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient);
3613 pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
3625 return hmR0SvmExitSetPendingXcptUD(pVCpu, pCtx, pSvmTransient);
3653 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
3663 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
3670 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
3676 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
3682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
3688 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
3693 pVCpu->hm.s.u32HMError = Event.n.u8Vector;
3698 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3706 pVCpu->hm.s.u32HMError = u32ExitCode;
3730 AssertPtr(pVCpu); \
3736 Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (uint32_t)pVCpu->idCpu)); \
3738 if (VMMR0IsLogFlushDisabled(pVCpu)) \
3742 # define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() do { NOREF(pVCpu); NOREF(pCtx); NOREF(pSvmTransient); } while (0)
3750 * @param pVCpu Pointer to the VMCPU.
3754 static int hmR0SvmInterpretInvlPgEx(PVMCPU pVCpu, PDISCPUSTATE pCpu, PCPUMCTX pCtx)
3770 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), GCPtrPage);
3796 static int hmR0SvmInterpretInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3799 if (CPUMGetGuestCodeBits(pVCpu) != 16)
3801 PDISSTATE pDis = &pVCpu->hm.s.DisState;
3802 int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
3806 rc = hmR0SvmInterpretInvlPgEx(pVCpu, pDis, pCtx);
3821 * @param pVCpu Pointer to the VMCPU.
3823 DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPU pVCpu)
3830 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3837 * @param pVCpu Pointer to the VMCPU.
3839 DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPU pVCpu)
3846 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3853 * @param pVCpu Pointer to the VMCPU.
3860 DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3874 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR2);
3877 hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3885 * @param pVCpu Pointer to the VMCPU.
3887 DECLINLINE(void) hmR0SvmSetPendingXcptNM(PVMCPU pVCpu)
3894 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3901 * @param pVCpu Pointer to the VMCPU.
3903 DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPU pVCpu)
3910 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3917 * @param pVCpu Pointer to the VMCPU.
3919 DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPU pVCpu)
3928 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3946 * @param pVCpu Pointer to the VMCPU.
3949 static int hmR0SvmEmulateMovTpr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3972 int rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPending, NULL /* pu8PendingIrq */);
3994 int rc2 = PDMApicSetTPR(pVCpu, u8Tpr);
3996 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4004 pVCpu->hm.s.u32HMError = pPatch->enmType;
4052 * @param pVCpu Pointer to the VMCPU.
4058 static int hmR0SvmCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4061 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
4089 Log4(("IDT: Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pCtx->cr2));
4104 Log4(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
4111 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
4149 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
4152 hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, 0 /* GCPtrFaultAddress */);
4162 hmR0SvmSetPendingXcptDF(pVCpu);
4188 * @param pVCpu Pointer to the VMCPU.
4195 DECLINLINE(void) hmR0SvmUpdateRip(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
4197 if (pVCpu->CTX_SUFF(pVM)->hm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
4199 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
4220 HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4225 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
4227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
4244 HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4248 hmR0SvmUpdateRip(pVCpu, pCtx, 2);
4249 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
4251 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4259 HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4263 hmR0SvmUpdateRip(pVCpu, pCtx, 2);
4264 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
4266 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4274 HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4277 PVM pVM = pVCpu->CTX_SUFF(pVM);
4278 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4281 hmR0SvmUpdateRip(pVCpu, pCtx, 2);
4282 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4289 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
4297 HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4300 PVM pVM = pVCpu->CTX_SUFF(pVM);
4301 int rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4304 hmR0SvmUpdateRip(pVCpu, pCtx, 2);
4308 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4315 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
4323 HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4326 int rc = EMInterpretRdtscp(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx);
4329 hmR0SvmUpdateRip(pVCpu, pCtx, 3);
4331 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4338 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
4346 HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4349 int rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
4352 hmR0SvmUpdateRip(pVCpu, pCtx, 2);
4353 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4360 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
4368 HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4371 PVM pVM = pVCpu->CTX_SUFF(pVM);
4375 int rc = hmR0SvmInterpretInvlpg(pVM, pVCpu, pCtx); /* Updates RIP if successful. */
4376 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
4378 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4386 HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4390 hmR0SvmUpdateRip(pVCpu, pCtx, 1);
4391 int rc = EMShouldContinueAfterHalt(pVCpu, pCtx) ? VINF_SUCCESS : VINF_EM_HALT;
4392 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4393 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
4395 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
4403 HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4406 int rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
4409 hmR0SvmUpdateRip(pVCpu, pCtx, 3);
4410 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4417 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
4425 HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4428 VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
4433 hmR0SvmUpdateRip(pVCpu, pCtx, 3);
4436 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
4440 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4449 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
4458 HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4468 HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4475 VBOXSTRICTRC rc2 = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
4480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0]);
4481 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4489 HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4493 VBOXSTRICTRC rc2 = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
4502 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
4506 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
4507 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
4511 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
4515 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4523 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4535 HMSVM_EXIT_DECL hmR0SvmExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4538 hmR0SvmSetPendingXcptUD(pVCpu);
4546 HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4549 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
4550 PVM pVM = pVCpu->CTX_SUFF(pVM);
4555 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
4564 int rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4566 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4568 hmR0SvmUpdateRip(pVCpu, pCtx, 2);
4570 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4576 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4580 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4587 rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */));
4589 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc); /* RIP updated by EMInterpretInstruction(). */
4605 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4608 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4616 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
4621 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4625 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4632 rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0));
4636 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4648 HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4651 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4657 pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
4666 Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
4670 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
4676 VMMRZCallRing3Disable(pVCpu);
4680 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
4681 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
4684 VMMRZCallRing3Enable(pVCpu);
4686 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
4694 VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
4700 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
4701 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
4712 HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4716 int rc = hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
4717 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4718 STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
4726 HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4731 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
4733 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
4735 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4737 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
4745 HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4755 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
4756 PVM pVM = pVCpu->CTX_SUFF(pVM);
4775 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
4780 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4785 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
4787 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
4791 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
4793 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
4806 rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
4808 HMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pVmcb->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port, uAndVal, cbValue);
4810 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
4816 rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
4823 HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pVmcb->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port, uAndVal, cbValue);
4825 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
4848 VMMRZCallRing3Disable(pVCpu);
4851 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
4852 CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /*fDr6*/);
4854 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, IoExitInfo.n.u16Port, cbValue);
4861 hmR0SvmSetPendingXcptDB(pVCpu);
4869 VMMRZCallRing3Enable(pVCpu);
4872 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
4901 HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4904 PVM pVM = pVCpu->CTX_SUFF(pVM);
4910 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
4923 && !CPUMGetGuestCPL(pVCpu)
4957 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
4973 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4979 TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
4980 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
4981 TRPMResetTrap(pVCpu);
4993 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
5004 HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5008 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
5017 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
5025 HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5030 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
5034 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
5036 && pVCpu->hm.s.Event.fPending) /** @todo fPending cannot be 'true', see hmR0SvmInjectPendingEvent(). See @bugref{7362}.*/
5043 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
5049 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
5057 HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5060 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
5063 int rc = hmR0SvmEmulateMovTpr(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx);
5066 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
5071 if (pVCpu->hm.s.fHypercallsEnabled)
5073 rc = GIMHypercall(pVCpu, pCtx);
5078 hmR0SvmUpdateRip(pVCpu, pCtx, 3);
5084 hmR0SvmSetPendingXcptUD(pVCpu);
5092 HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5097 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5100 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
5112 HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5119 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
5122 PVM pVM = pVCpu->CTX_SUFF(pVM);
5127 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
5131 hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
5138 hmR0SvmSetPendingXcptDF(pVCpu);
5141 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
5154 && !CPUMGetGuestCPL(pVCpu)
5163 int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL /* pfFlags */, &GCPhysPage);
5182 Assert(pVCpu->hm.s.Event.fPending);
5186 TRPMAssertXcptPF(pVCpu, uFaultAddress, u32ErrCode);
5187 int rc = PGMTrap0eHandler(pVCpu, u32ErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
5194 TRPMResetTrap(pVCpu);
5195 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
5196 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
5201 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
5206 u32ErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
5207 TRPMResetTrap(pVCpu);
5208 hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
5213 TRPMResetTrap(pVCpu);
5214 hmR0SvmSetPendingXcptDF(pVCpu);
5218 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
5222 TRPMResetTrap(pVCpu);
5223 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
5232 HMSVM_EXIT_DECL hmR0SvmExitXcptNM(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5239 VMMRZCallRing3Disable(pVCpu);
5247 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
5254 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx);
5255 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
5259 VMMRZCallRing3Enable(pVCpu);
5264 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
5265 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
5266 pVCpu->hm.s.fPreloadGuestFpu = true;
5272 hmR0SvmSetPendingXcptNM(pVCpu);
5273 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
5283 HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5289 if (pVCpu->hm.s.fGIMTrapXcptUD)
5290 GIMXcptUD(pVCpu, pCtx, NULL /* pDis */);
5292 hmR0SvmSetPendingXcptUD(pVCpu);
5294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
5303 HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5309 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
5313 PVM pVM = pVCpu->CTX_SUFF(pVM);
5314 PDISSTATE pDis = &pVCpu->hm.s.DisState;
5316 int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
5320 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
5329 hmR0SvmSetPendingXcptMF(pVCpu);
5338 HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5344 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
5348 PSVMVMCB pVmcb = (PSVMVMCB)pVCpu->hm.s.svm.pvVmcb;
5349 PVM pVM = pVCpu->CTX_SUFF(pVM);
5350 int rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
5354 if (CPUMIsHyperDebugStateActive(pVCpu))
5355 CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) | pVmcb->guest.u64DR6);
5358 hmR0SvmSetPendingXcptDB(pVCpu);
5365 if (CPUMIsHyperDebugStateActive(pVCpu))
5374 Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));