PGMRZDynMap.cpp revision b40179b44fea65b72b2f226f62af1ed7bd3c48fc
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * PGM - Page Manager and Monitor, dynamic mapping cache.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Copyright (C) 2008-2010 Oracle Corporation
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * This file is part of VirtualBox Open Source Edition (OSE), as
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * available from http://www.virtualbox.org. This file is free software;
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * you can redistribute it and/or modify it under the terms of the GNU
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * General Public License (GPL) as published by the Free Software
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Foundation, in version 2 as it comes in the "COPYING" file of the
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/*******************************************************************************
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster* Internal Functions *
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster*******************************************************************************/
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/*******************************************************************************
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster* Defined Constants And Macros *
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster*******************************************************************************/
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** The max size of the mapping cache (in pages). */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster# define PGMR0DYNMAP_MAX_PAGES ((16*_1M) >> PAGE_SHIFT)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** The small segment size that is adopted on out-of-memory conditions with a
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * single big segment. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** The number of pages we reserve per CPU. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** The minimum number of pages we reserve per CPU.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * This must be equal or larger than the autoset size. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Calcs the overload threshold (safety margin). Current set at 50%. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster# define PGMR0DYNMAP_CALC_OVERLOAD(cPages) ((cPages) / 2)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** The number of guard pages.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * @remarks Never do tuning of the hashing or whatnot with a strict build! */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster#endif /* IN_RING0 */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** The dummy physical address of guard pages. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS UINT32_C(0x7777feed)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** The dummy reference count of guard pages. (Must be non-zero.) */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT INT32_C(0x7777feed)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Define this to just clear the present bit on guard pages.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * The alternative is to replace the entire PTE with an bad not-present
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * PTE. Either way, XNU will screw us. :-/ */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** The dummy PTE value for a page. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE X86_PTE_PG_MASK
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** The dummy PTE value for a page. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE UINT64_MAX /*X86_PTE_PAE_PG_MASK*/
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster#ifdef IN_RING0 /* Note! Assertions causes panics if preemption is disabled,
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * disable this to work around that. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Acquire the spinlock.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * This will declare a temporary variable and expands to two statements!
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster RTSPINLOCKTMP MySpinlockTmp = RTSPINLOCKTMP_INITIALIZER; \
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster RTSpinlockAcquire((pThis)->hSpinlock, &MySpinlockTmp)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Releases the spinlock.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster RTSpinlockRelease((pThis)->hSpinlock, &MySpinlockTmp)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Re-acquires the spinlock.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) \
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster RTSpinlockAcquire((pThis)->hSpinlock, &MySpinlockTmp)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) do { } while (0)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) do { } while (0)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) do { } while (0)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Converts a PGMCPUM::AutoSet pointer into a PVMCPU. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster#define PGMRZDYNMAP_SET_2_VMCPU(pSet) (RT_FROM_MEMBER(pSet, VMCPU, pgm.s.AutoSet))
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster#define PGMRZDYNMAP_SET_2_VM(pSet) (PGMRZDYNMAP_SET_2_VMCPU(pSet)->CTX_SUFF(pVM))
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (PGMRZDYNMAP_SET_2_VM(pSet)->pgm.s.pRCDynMap)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (g_pPGMR0DynMap)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Gets the set index of the current CPU.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * This always returns 0 when in raw-mode context because there is only ever
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * one EMT in that context (at least presently).
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster# define PGMRZDYNMAP_CUR_CPU() RTMpCpuIdToSetIndex(RTMpCpuId())
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** PGMRZDYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Zaps an set entry. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** @def PGMRZDYNMAP_STRICT_RELEASE
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Define this to force pages to be released and make non-present ASAP after
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * use. This should not normally be enabled as it is a bit expensive. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster#if 0 || defined(DOXYGEN_RUNNING)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/*******************************************************************************
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster* Structures and Typedefs *
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster*******************************************************************************/
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Ring-0 dynamic mapping cache segment.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * The dynamic mapping cache can be extended with additional segments if the
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * load is found to be too high. This done the next time a VM is created, under
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * the protection of the init mutex. The arrays is reallocated and the new
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * segment is added to the end of these. Nothing is rehashed of course, as the
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * indexes / addresses must remain unchanged.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * This structure is only modified while owning the init mutex or during module
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * init / term.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** Pointer to the next segment. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The memory object for the virtual address range that we're abusing. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The start page in the cache. (I.e. index into the arrays.) */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The number of pages this segment contributes. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The number of page tables. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The memory objects for the page tables. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Pointer to a ring-0 dynamic mapping cache segment. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Ring-0 dynamic mapping cache entry.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * @sa PGMRZDYNMAPENTRY, PGMRCDYNMAPENTRY.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The physical address of the currently mapped page.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * This is duplicate for three reasons: cache locality, cache policy of the PT
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * mappings and sanity checks. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** Pointer to the page. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The number of references. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** PTE pointer union. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** PTE pointer, 32-bit legacy version. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** PTE pointer, PAE version. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** PTE pointer, the void version. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** CPUs that haven't invalidated this entry after it's last update. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Pointer a mapping cache entry for the ring-0.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * @sa PPGMRZDYNMAPENTRY, PPGMRCDYNMAPENTRY, */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Dynamic mapping cache for ring-0.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * This is initialized during VMMR0 module init but no segments are allocated
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * at that time. Segments will be added when the first VM is started and
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * removed again when the last VM shuts down, thus avoid consuming memory while
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * dormant. At module termination, the remaining bits will be freed up.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * @sa PPGMRZDYNMAP, PGMRCDYNMAP.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** Spinlock serializing the normal operation of the cache. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** Array for tracking and managing the pages. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The cache size given as a number of pages. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The current load.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * This does not include guard pages. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The max load ever.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * This is maintained to trigger the adding of more mapping space. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** Initialization / termination lock. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The number of guard pages. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The number of users (protected by hInitLock). */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** Array containing a copy of the original page tables.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** List of segments. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster /** The paging mode. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Paging level data.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster uint32_t cLevels; /**< The number of levels. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster RTHCPHYS HCPhys; /**< The address of the page for the current level,
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * i.e. what hMemObj/hMapObj is currently mapping. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster RTHCPHYS fPhysMask; /**< Mask for extracting HCPhys from uEntry. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster RTR0MEMOBJ hMemObj; /**< Memory object for HCPhys, PAGE_SIZE. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster RTR0MEMOBJ hMapObj; /**< Mapping object for hMemObj. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster uint32_t fPtrShift; /**< The pointer shift count. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster uint64_t fPtrMask; /**< The mask to apply to the shifted pointer to get the table index. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster uint64_t fAndMask; /**< And mask to check entry flags. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster uint64_t fResMask; /**< The result from applying fAndMask. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster PX86PGUINT paLegacy; /**< Legacy table view. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster PX86PGPAEUINT paPae; /**< PAE/AMD64 table view. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Pointer to paging level data. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Mapping cache entry for the current context.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Fostertypedef CTX_MID(PGM,DYNMAPENTRY) PGMRZDYNMAPENTRY;
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Pointer a mapping cache entry for the current context.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Pointer the mapping cache instance for the current context.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * @sa PGMR0DYNMAP, PGMRCDYNMAP */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/*******************************************************************************
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster* Global Variables *
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster*******************************************************************************/
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** Pointer to the ring-0 dynamic mapping cache. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/** For overflow testing. */
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Fosterstatic bool g_fPGMR0DynMapTestRunning = false;
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster/*******************************************************************************
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster* Internal Functions *
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster*******************************************************************************/
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Fosterstatic void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs);
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Fosterstatic int pgmR0DynMapSetup(PPGMRZDYNMAP pThis);
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Fosterstatic int pgmR0DynMapExpand(PPGMRZDYNMAP pThis);
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Fosterstatic void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis);
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster#if 0 /*def DEBUG*/
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Initializes the auto mapping sets for a VM.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * @returns VINF_SUCCESS on success, VERR_INTERNAL_ERROR on failure.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * @param pVM The VM in question.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Fosterstatic int pgmRZDynMapInitAutoSetsForVM(PVM pVM)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster AssertReturn(idCpu > 0 && idCpu <= VMM_MAX_CPU_COUNT, VERR_INTERNAL_ERROR);
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster while (idCpu-- > 0)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster while (j-- > 0)
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Initializes the ring-0 dynamic mapping cache.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * @returns VBox status code.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster * Create and initialize the cache instance.
4fe4e4f798a84a46e567f64ceadd3648eb0582d4Allan Foster PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)RTMemAllocZ(sizeof(*pThis));
return VINF_SUCCESS;
return rc;
if (pThis)
AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER);
return rc;
return VINF_SUCCESS;
return rc;
#ifdef VBOX_STRICT
while (idCpu-- > 0)
while (iPage-- > 0)
while (iPage-- > 0)
return rc;
return cPages;
case SUPPAGINGMODE_32_BIT:
case SUPPAGINGMODE_PAE:
case SUPPAGINGMODE_PAE_GLOBAL:
case SUPPAGINGMODE_PAE_NX:
case SUPPAGINGMODE_AMD64:
case SUPPAGINGMODE_AMD64_NX:
AssertFailed();
ASMIntEnable();
int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE, RTMEM_CACHE_POLICY_DONT_CARE);
AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv));
return VINF_TRY_AGAIN;
return rc;
LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n"
i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask,
return VERR_INTERNAL_ERROR;
/*Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));*/
return VINF_SUCCESS;
#ifdef PGMR0DYNMAP_GUARD_NP
int rc2;
void *pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * (pThis->cPages + cPages));
if (!pvSavedPTEs)
return VERR_NO_MEMORY;
if (!pvPages)
pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages);
if (pvSavedPTEs)
return VERR_NO_MEMORY;
PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs]));
if (!pSeg)
return VERR_NO_MEMORY;
#ifndef IN_RC
#ifdef VBOX_STRICT
LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n",
ASMIntEnable();
#if PGMR0DYNMAP_GUARD_PAGES > 0
return VINF_SUCCESS;
return rc;
#if PGMR0DYNMAP_GUARD_PAGES > 0
return rc;
return VINF_SUCCESS;
#if PGMR0DYNMAP_GUARD_PAGES > 0
return rc;
while (iPage-- > 0)
while (iPage-- > 0)
int rc;
while (iPT-- > 0)
#ifdef IN_RC
return rc;
return rc;
return VINF_SUCCESS;
if (!cRefs)
#ifdef PGMRZDYNMAP_STRICT_RELEASE
static uint32_t pgmR0DynMapPageSlow(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVMCPU pVCpu, bool *pfNew)
#ifdef VBOX_WITH_STATISTICS
bool fLooped = false;
*pfNew = false;
return iFreePage;
return UINT32_MAX;
#ifdef VBOX_WITH_STATISTICS
fLooped = true;
if (!fLooped)
*pfNew = true;
/*Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));*/
#ifndef IN_RC
/*Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));*/
return iFreePage;
DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVMCPU pVCpu, void **ppvPage)
bool fNew = false;
return iPage;
AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages));
#ifndef IN_RC
#ifdef IN_RC
return iPage;
if (!pThis)
return VINF_SUCCESS;
return VERR_INVALID_PARAMETER;
RTAssertMsg2Weak a; \
return VERR_INTERNAL_ERROR; \
#ifdef IN_RING0
while (iPage-- > 0)
CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
#ifdef PGMR0DYNMAP_GUARD_NP
cGuard++;
CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
#ifdef IN_RING0
cLoad++;
#ifdef IN_RING0
while (iPage-- > 0)
CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
#ifdef PGMR0DYNMAP_GUARD_NP
cGuard++;
CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
#ifdef IN_RING0
cLoad++;
#ifdef IN_RING0
return VINF_SUCCESS;
#ifdef IN_RING0
#ifdef IN_RC
Log2(("pgmDynMapOptimizeAutoSet: #%u/%u/%u pvPage=%p iPage=%u cRefs=%u cInlinedRefs=%u cUnrefs=%u cTotalRefs=%u\n",
cTotalRefs));
Log2(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
//LogFlow(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
#ifdef IN_RING0
if (fStartIt)
return fStartIt;
#ifdef IN_RC
#ifdef IN_RC
if ( cEntries != 0
#ifdef IN_RC
STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
#ifdef IN_RC
STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
#ifndef IN_RC
if (i != PGMMAPSET_CLOSED)
if ( i > iSubset
while (i-- > iSubset)
AssertMsg(iPrevSubset <= pSet->iSubset || iPrevSubset == UINT32_MAX, ("iPrevSubset=%#x iSubset=%#x\n", iPrevSubset, pSet->iSubset));
return iPrevSubset;
LogFlow(("PGMRZDynMapPopAutoSubset: pVCpu=%p iPrevSubset=%u iSubset=%u cEntries=%u\n", pVCpu, iPrevSubset, pSet->iSubset, cEntries));
AssertMsgReturnVoid(pSet->iSubset >= iPrevSubset || iPrevSubset == UINT32_MAX, ("iPrevSubset=%u iSubset=%u cEntries=%u\n", iPrevSubset, pSet->iSubset, cEntries));
#ifdef IN_RC
STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
#ifdef LOG_ENABLED
bool fFound = false;
while (iEntry-- > 0)
fFound = true;
pSet->aEntries[0].iPage, pSet->aEntries[0].cRefs, pSet->aEntries[0].cInlinedRefs, pSet->aEntries[0].cUnrefs, pSet->aEntries[0].pvPage,
pSet->aEntries[1].iPage, pSet->aEntries[1].cRefs, pSet->aEntries[1].cInlinedRefs, pSet->aEntries[1].cUnrefs, pSet->aEntries[1].pvPage,
pSet->aEntries[2].iPage, pSet->aEntries[2].cRefs, pSet->aEntries[2].cInlinedRefs, pSet->aEntries[2].cUnrefs, pSet->aEntries[2].pvPage,
pSet->aEntries[3].iPage, pSet->aEntries[3].cRefs, pSet->aEntries[3].cInlinedRefs, pSet->aEntries[3].cUnrefs, pSet->aEntries[3].pvPage,
pSet->aEntries[4].iPage, pSet->aEntries[4].cRefs, pSet->aEntries[4].cInlinedRefs, pSet->aEntries[4].cUnrefs, pSet->aEntries[4].pvPage,
pSet->aEntries[5].iPage, pSet->aEntries[5].cRefs, pSet->aEntries[5].cInlinedRefs, pSet->aEntries[5].cUnrefs, pSet->aEntries[5].pvPage));
uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[iEntry].cRefs + pSet->aEntries[iEntry].cInlinedRefs;
LogFlow(("pgmRZDynMapUnusedHint: pvHint=%p #%u cRefs=%d cInlinedRefs=%d cUnrefs=%d (+1) cTotalRefs=%d %s(%d) %s\n",
pvHint, iEntry, pSet->aEntries[iEntry].cRefs, pSet->aEntries[iEntry].cInlinedRefs, cUnrefs, cTotalRefs, pszFile, iLine, pszFunction));
Log(("pgmRZDynMapUnusedHint: pvHint=%p ignored because of overflow! %s(%d) %s\n", pvHint, pszFile, iLine, pszFunction));
#ifdef PGMRZDYNMAP_STRICT_RELEASE
AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
void *pvPage;
return VERR_PGM_DYNMAP_FAILED;
LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/0/0 iPage=%#x [a] %s(%d) %s\n",
LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [0] %s(%d) %s\n", pSet, HCPhys, i - 0, pSet->cEntries, pvPage, pSet->aEntries[i - 0].cRefs, pSet->aEntries[i - 0].cInlinedRefs, pSet->aEntries[i - 0].cUnrefs, iPage, pszFile, iLine, pszFunction));
LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [1] %s(%d) %s\n", pSet, HCPhys, i - 1, pSet->cEntries, pvPage, pSet->aEntries[i - 1].cRefs, pSet->aEntries[i - 1].cInlinedRefs, pSet->aEntries[i - 1].cUnrefs, iPage, pszFile, iLine, pszFunction));
LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [2] %s(%d) %s\n", pSet, HCPhys, i - 2, pSet->cEntries, pvPage, pSet->aEntries[i - 2].cRefs, pSet->aEntries[i - 2].cInlinedRefs, pSet->aEntries[i - 2].cUnrefs, iPage, pszFile, iLine, pszFunction));
LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 3, pSet->cEntries, pvPage, pSet->aEntries[i - 3].cRefs, pSet->aEntries[i - 3].cInlinedRefs, pSet->aEntries[i - 3].cUnrefs, iPage, pszFile, iLine, pszFunction));
LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 4, pSet->cEntries, pvPage, pSet->aEntries[i - 4].cRefs, pSet->aEntries[i - 4].cInlinedRefs, pSet->aEntries[i - 4].cUnrefs, iPage, pszFile, iLine, pszFunction));
LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [b] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [c] %s(%d) %s\n", pSet, HCPhys, i, pSet->cEntries, pvPage, pSet->aEntries[i].cRefs, pSet->aEntries[i].cInlinedRefs, pSet->aEntries[i].cUnrefs, iPage, pszFile, iLine, pszFunction));
STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [d] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
return VERR_PGM_DYNMAP_FULL_SET;
return VINF_SUCCESS;
typedef struct PGMR0DYNMAPTEST
uint32_t i;
return rc;
g_fPGMR0DynMapTestRunning = true;
ASMIntEnable();
LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
ASMIntEnable();
LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2));
ASMIntEnable();
LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
ASMIntEnable();
ASMIntEnable();
ASMIntEnable();
ASMIntEnable();
ASMIntEnable();
g_fPGMR0DynMapTestRunning = false;
pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
return rc;