PGMAllBth.h revision ffea7b3614d7843b53411a7636b66195b52d524d
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * This file is a big challenge!
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * Copyright (C) 2006-2007 Sun Microsystems, Inc.
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * This file is part of VirtualBox Open Source Edition (OSE), as
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * available from http://www.virtualbox.org. This file is free software;
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * you can redistribute it and/or modify it under the terms of the GNU
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * General Public License (GPL) as published by the Free Software
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * Foundation, in version 2 as it comes in the "COPYING" file of the
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * Clara, CA 95054 USA or visit http://www.sun.com if you need
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * additional information or have any questions.
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync/*******************************************************************************
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync* Internal Functions *
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync*******************************************************************************/
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncDECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync# error "Invalid combination; PAE guest implies PAE shadow"
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT)
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT)
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * #PF Handler for raw-mode guest execution.
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * @returns VBox status code (appropriate for trap handling and GC return).
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * @param pVCpu VMCPU Handle.
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * @param uErr The trap error code.
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * @param pRegFrame Trap register frame.
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * @param pvFault The fault address.
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsyncPGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
ad77e3ec3cde24263bc7537575f5cae442bee3b1vboxsync# if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync * Hide the instruction fetch trap indicator for now.
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync /** @todo NXE will change this and we must fix NXE in the switcher too! */
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync# elif PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, pvFault, &iPDSrc, &PdpeSrc);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, pvFault, &pPml4eSrc, &PdpeSrc, &iPDSrc);
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync /* Quick check for a valid guest trap. (PAE & AMD64) */
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync# if PGM_GST_TYPE == PGM_TYPE_AMD64 && GC_ARCH_BITS == 64
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync LogFlow(("Trap0eHandler: guest PML4 %d not present CR3=%RGp\n", (int)((pvFault >> X86_PML4_SHIFT) & X86_PML4_MASK), CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync LogFlow(("Trap0eHandler: guest iPDSrc=%u not present CR3=%RGp\n", iPDSrc, CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
eb90548e8e40e597d65cdcc16ec958a3e09c1d73vboxsync# else /* !PGM_WITH_PAGING */
const unsigned iPDSrc = 0;
const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
return rc;
return rc;
return rc;
= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVCpu->pgm.s.StatRZTrap0eTime2DirtyAndAccessed : &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2SyncPT; });
return rc;
return VINF_PGM_SYNC_CR3;
while (iPT-- > 0)
return VINF_PGM_SYNC_CR3;
PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
if ( pCur
# ifdef IN_RC
rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
AssertFailed();
return rc;
return VINF_EM_RAW_GUEST_TRAP;
} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
bool fBigPagesSupported = true;
if (RT_SUCCESS(rc)) /** just handle the failure immediate (it returns) and make things easier to read. */
PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysFault);
if (pCur)
# ifdef PGM_SYNC_N_PAGES
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
return rc;
("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pCur->CTX_SUFF(pvUser));
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndPhys; });
return rc;
# ifdef PGM_SYNC_N_PAGES
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndVirt; });
return rc;
/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
if (pCur)
("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
# ifdef IN_RC
rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
return rc;
unsigned iPage;
if ( pCur
# ifdef IN_RC
RTGCPTR off = (iPage << PAGE_SHIFT) + (pvFault & PAGE_OFFSET_MASK) - (pCur->Core.Key & PAGE_OFFSET_MASK);
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
return rc;
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
return rc;
/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndUnhandled; });
return rc;
PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
if (pCur)
("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
# ifdef IN_RC
rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
return rc;
return VINF_EM_RAW_EMULATE_INSTR;
pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)));
/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
# ifdef CSAM_DETECT_NEW_CODE_PAGES
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2CSAM; });
return rc;
# ifdef CSAM_DETECT_NEW_CODE_PAGES
rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSync; });
return VINF_SUCCESS;
return rc;
return VINF_EM_NO_MEMORY;
# ifdef VBOX_STRICT
LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
AssertMsg((RT_SUCCESS(rc) && ((fPageShw & X86_PTE_RW) || pVM->cCPUs > 1 /* new monitor can be installed during trap e execution */)), ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndObs; });
return VINF_SUCCESS;
return rc;
# ifdef VBOX_STRICT
AssertMsg((fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)),
("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
return VINF_EM_RAW_GUEST_TRAP;
return VINF_EM_RAW_EMULATE_INSTR;
return VERR_INTERNAL_ERROR;
int rc;
return VINF_SUCCESS;
AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
return VINF_SUCCESS;
return VINF_SUCCESS;
return VINF_SUCCESS;
# if defined(IN_RC)
unsigned iPDSrc = 0;
if (pPDSrc)
PdeSrc.u = 0;
# ifdef IN_RING3
&& fIsBigPage
return VINF_SUCCESS;
LogFlow(("InvalidatePage: Out-of-sync PML4E (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
return VINF_SUCCESS;
LogFlow(("InvalidatePage: Out-of-sync PDPE (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
return VINF_SUCCESS;
# ifndef PGM_WITHOUT_MAPPING
else if (!fIsBigPage)
# ifdef PGMPOOL_WITH_USER_TRACKING
/* This is very unlikely with caching/monitoring enabled. */
LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
# if defined(IN_RC)
return VINF_SUCCESS;
# if defined(IN_RC)
return rc;
return VINF_SUCCESS;
#ifdef PGMPOOL_WITH_USER_TRACKING
DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
pRam;
while (iPage-- > 0)
DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
if (!u16)
Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
LogFlow(("SyncPageWorker: monitored page (%RHp) -> mark not present\n", PGM_PAGE_GET_HCPHYS(pPage)));
PteDst.u = 0;
PteDst.u = 0;
* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
#ifdef PGMPOOL_WITH_USER_TRACKING
PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
#ifdef PGMPOOL_WITH_USER_TRACKING
PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
# if defined(IN_RC)
if (!fBigPage)
if (!fBigPage)
# ifdef PGM_SYNC_N_PAGES
const unsigned offPTSrc = 0;
iPTDst = 0;
RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
#ifndef IN_RING0
# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
PteDst.u = 0;
# ifdef PGMPOOL_WITH_USER_TRACKING
PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
# if defined(IN_RC)
return VINF_SUCCESS;
# if defined(IN_RC)
return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
&& !defined(IN_RC)
# ifdef PGM_SYNC_N_PAGES
return rc;
iPTDst = 0;
RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT);
Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT);
return VINF_SUCCESS;
return VERR_INTERNAL_ERROR;
PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage)
bool fBigPagesSupported = true;
unsigned uPageFaultLevel;
int rc;
uPageFaultLevel = 0;
goto l_UpperLevelPageFault;
goto l_UpperLevelPageFault;
goto l_UpperLevelPageFault;
if (fWriteFault)
/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
# ifdef IN_RING0
if (pShwPage)
return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
return VINF_PGM_NO_DIRTY_BIT_TRACKING;
return VINF_EM_RAW_GUEST_TRAP;
if (fWriteFault)
# ifdef VBOX_WITH_STATISTICS
#ifndef IN_RING0
return VINF_SUCCESS;
if (pShwPage)
# ifdef VBOX_STRICT
if (pPage)
("Unexpected dirty bit tracking on monitored page %RGv (phys %RGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
# ifdef IN_RING0
return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
# ifdef VBOX_STRICT
return VINF_PGM_NO_DIRTY_BIT_TRACKING;
return rc;
/* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
return VINF_EM_RAW_GUEST_TRAP;
AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
# ifndef PGM_WITHOUT_MAPPINGS
# ifndef IN_RING3
return VERR_ADDRESS_CONFLICT;
int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
return rc;
# if defined(IN_RC)
if (fPageTable)
rc = pgmPoolAllocEx(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, pShwPde->idx, iPDDst, &pShwPage);
if (fPageTable)
| (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
| (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
# if defined(IN_RC)
return VINF_SUCCESS;
# if defined(IN_RC)
return VINF_PGM_SYNC_CR3;
if (fPageTable)
| (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
# if defined(IN_RC)
# ifdef PGM_SYNC_N_PAGES
iPTDst = 0;
unsigned iPTDst = 0;
const unsigned offPTSrc = 0;
# ifndef IN_RING0
Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
| (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
# if defined(IN_RC)
PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT);
unsigned iPTDst = 0;
# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
PteDst.u = 0;
# ifndef IN_RING0
PteDst.u = 0;
Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))));
# ifdef PGMPOOL_WITH_USER_TRACKING
PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
(RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
iHCPage++;
iPTDst++;
else if (pRam)
iPTDst++;
return rc;
&& !defined(IN_RC)
return rc;
rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
return rc;
return VERR_INTERNAL_ERROR;
#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
unsigned iPDSrc;
if (!pPDSrc)
unsigned iPDSrc;
if (!pPDSrc)
const unsigned iPDSrc = 0;
PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
return rc;
return rc;
return rc;
PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_TYPE_AMD64) \
# ifndef IN_RING0
unsigned iPDSrc = 0;
if (pPDSrc)
return VINF_EM_RAW_GUEST_TRAP;
unsigned iPDSrc;
if (!pPDSrc)
return VINF_EM_RAW_GUEST_TRAP;
const unsigned iPDSrc = 0;
PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
return rc;
return rc;
# if defined(IN_RC)
# if defined(IN_RC)
return rc;
# if defined(IN_RC)
return rc;
return VERR_INTERNAL_ERROR;
#define MY_STAM_COUNTER_INC(a) do { } while (0)
return VINF_SUCCESS;
return VINF_SUCCESS;
#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
# ifdef PGM_WITHOUT_MAPPINGS
return VINF_SUCCESS;
return VINF_SUCCESS;
return VINF_PGM_SYNC_CR3;
return VINF_SUCCESS;
#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
#ifdef VBOX_STRICT
#ifdef IN_RC
#ifdef IN_RING3
VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
unsigned cErrors = 0;
AssertFailed();
bool fBigPagesSupported = true;
# ifndef IN_RING0
int rc;
AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
cErrors++;
AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
cErrors++;
AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
cErrors++;
unsigned iPDSrc;
AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
cErrors++;
AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
cErrors++;
AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
cErrors++;
AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
cErrors++;
if (!pPoolPage)
cErrors++;
AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
cErrors++;
cErrors++;
cErrors++;
|| !fBigPagesSupported)
cErrors++;
!= (!PdeSrc.b.u1Size || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
cErrors++;
if (!pPhysPage)
cErrors++;
cErrors++;
|| !fBigPagesSupported)
AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
cErrors++;
/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
cErrors++;
cErrors++;
const unsigned offPTSrc = 0;
if (!(PteDst.u & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
# ifdef IN_RING3
AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
cErrors++;
uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
# ifdef IN_RING3
cErrors++;
AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
if (!pPhysPage)
cErrors++;
AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
cErrors++;
cErrors++;
cErrors++;
AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
# ifdef DEBUG_sandervl
AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
cErrors++;
cErrors++;
AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX;
# ifdef IN_RING3
cErrors++;
AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
if (!pPhysPage)
cErrors++;
AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
&& (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
cErrors++;
# ifdef DEBUG
if (cErrors)
return cErrors;
int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhysCR3 & GST_CR3_PAGE_MASK, (void **)&HCPtrGuestCR3);
# ifdef IN_RC
# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RCPTRTYPE(uint8_t *))pVM->pgm.s.GCPtrCR3Mapping + off);
# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
# ifdef IN_RC
# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
# ifdef IN_RC
# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3);
# ifdef IN_RC
# ifdef IN_RING0
# ifndef PGM_WITHOUT_MAPPINGS
Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) || VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
# ifdef IN_RC
if (pOldShwPageCR3)
# ifndef PGM_WITHOUT_MAPPINGS
return rc;
# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
# ifndef PGM_WITHOUT_MAPPINGS
pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
return rc;