ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * CDDL HEADER START
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * The contents of this file are subject to the terms of the
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Common Development and Distribution License (the "License").
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * You may not use this file except in compliance with the License.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * or http://www.opensolaris.org/os/licensing.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * See the License for the specific language governing permissions
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * and limitations under the License.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * When distributing Covered Code, include this CDDL HEADER in each
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * If applicable, add the following below this CDDL HEADER, with the
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * fields enclosed by brackets "[]" replaced with your own identifying
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * information: Portions Copyright [yyyy] [name of copyright owner]
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * CDDL HEADER END
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
613b28719c10e84c1202c1045df44d77767de21dRichard Bean * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Use is subject to license terms.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/types.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/systm.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/archsystm.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/debug.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/bootconf.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/bootsvcs.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/bootinfo.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/mman.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/cmn_err.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/param.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/machparam.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/machsystm.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/promif.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <sys/kobj.h>
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev#include <sys/hypervisor.h>
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <vm/kboot_mmu.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <vm/hat_pte.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <vm/hat_i86.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#include <vm/seg_kmem.h>
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#if 0
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Joe's debug printing
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#define DBG(x) \
613b28719c10e84c1202c1045df44d77767de21dRichard Bean bop_printf(NULL, "kboot_mmu.c: %s is %" PRIx64 "\n", #x, (uint64_t)(x));
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#define DBG(x) /* naught */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Page table and memory stuff.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjstatic caddr_t window;
ae115bc77f6fcde83175c75b4206dc2e50747966mrjstatic caddr_t pte_to_window;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * this are needed by mmu_init()
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjint kbm_nx_support = 0; /* NX bit in PTEs is in use */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjint kbm_pae_support = 0; /* PAE is 64 bit Page table entries */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjint kbm_pge_support = 0; /* PGE is Page table global bit enabled */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjint kbm_largepage_support = 0;
ae115bc77f6fcde83175c75b4206dc2e50747966mrjuint_t kbm_nucleus_size = 0;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#define BOOT_SHIFT(l) (shift_amt[l])
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#define BOOT_SZ(l) ((size_t)1 << BOOT_SHIFT(l))
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#define BOOT_OFFSET(l) (BOOT_SZ(l) - 1)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#define BOOT_MASK(l) (~BOOT_OFFSET(l))
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Initialize memory management parameters for boot time page table management
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjvoid
ae115bc77f6fcde83175c75b4206dc2e50747966mrjkbm_init(struct xboot_info *bi)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj /*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * configure mmu information
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj kbm_nucleus_size = (uintptr_t)bi->bi_kseg_size;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj kbm_largepage_support = bi->bi_use_largepage;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj kbm_nx_support = bi->bi_use_nx;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj kbm_pae_support = bi->bi_use_pae;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj kbm_pge_support = bi->bi_use_pge;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj window = bi->bi_pt_window;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj DBG(window);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj pte_to_window = bi->bi_pte_to_pt_window;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj DBG(pte_to_window);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support) {
ae115bc77f6fcde83175c75b4206dc2e50747966mrj shift_amt = shift_amt_pae;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj ptes_per_table = 512;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj pte_size = 8;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj lpagesize = TWO_MEG;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#ifdef __amd64
ae115bc77f6fcde83175c75b4206dc2e50747966mrj top_level = 3;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj top_level = 2;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj } else {
ae115bc77f6fcde83175c75b4206dc2e50747966mrj shift_amt = shift_amt_nopae;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj ptes_per_table = 1024;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj pte_size = 4;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj lpagesize = FOUR_MEG;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj top_level = 1;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj }
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev xen_info = bi->bi_xen_start_info;
843e19887f64dde75055cf8842fc4db2171eff45johnlev mfn_list = (mfn_t *)xen_info->mfn_list;
843e19887f64dde75055cf8842fc4db2171eff45johnlev DBG(mfn_list);
843e19887f64dde75055cf8842fc4db2171eff45johnlev mfn_count = xen_info->nr_pages;
843e19887f64dde75055cf8842fc4db2171eff45johnlev DBG(mfn_count);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj top_page_table = bi->bi_top_page_table;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj DBG(top_page_table);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Change the addressible page table window to point at a given page
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*ARGSUSED*/
ae115bc77f6fcde83175c75b4206dc2e50747966mrjvoid *
ae115bc77f6fcde83175c75b4206dc2e50747966mrjkbm_remap_window(paddr_t physaddr, int writeable)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
843e19887f64dde75055cf8842fc4db2171eff45johnlev x86pte_t pt_bits = PT_NOCONSIST | PT_VALID | PT_WRITABLE;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj DBG(physaddr);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev if (!writeable)
843e19887f64dde75055cf8842fc4db2171eff45johnlev pt_bits &= ~PT_WRITABLE;
843e19887f64dde75055cf8842fc4db2171eff45johnlev if (HYPERVISOR_update_va_mapping((uintptr_t)window,
843e19887f64dde75055cf8842fc4db2171eff45johnlev pa_to_ma(physaddr) | pt_bits, UVMF_INVLPG | UVMF_LOCAL) < 0)
843e19887f64dde75055cf8842fc4db2171eff45johnlev bop_panic("HYPERVISOR_update_va_mapping() failed");
843e19887f64dde75055cf8842fc4db2171eff45johnlev#else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *((x86pte_t *)pte_to_window) = physaddr | pt_bits;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *((x86pte32_t *)pte_to_window) = physaddr | pt_bits;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj mmu_tlbflush_entry(window);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj DBG(window);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (window);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Add a mapping for the physical page at the given virtual address.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjvoid
ae115bc77f6fcde83175c75b4206dc2e50747966mrjkbm_map(uintptr_t va, paddr_t pa, uint_t level, uint_t is_kernel)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj x86pte_t *ptep;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj paddr_t pte_physaddr;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj x86pte_t pteval;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (khat_running)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj panic("kbm_map() called too late");
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj pteval = pa_to_ma(pa) | PT_NOCONSIST | PT_VALID | PT_WRITABLE;
02bc52be7430b2f7fafe1a2c981bff49ef11d6fakchow if (level >= 1)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj pteval |= PT_PAGESIZE;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pge_support && is_kernel)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj pteval |= PT_GLOBAL;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev /*
843e19887f64dde75055cf8842fc4db2171eff45johnlev * try update_va_mapping first - fails if page table is missing.
843e19887f64dde75055cf8842fc4db2171eff45johnlev */
843e19887f64dde75055cf8842fc4db2171eff45johnlev if (HYPERVISOR_update_va_mapping(va, pteval,
843e19887f64dde75055cf8842fc4db2171eff45johnlev UVMF_INVLPG | UVMF_LOCAL) == 0)
843e19887f64dde75055cf8842fc4db2171eff45johnlev return;
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
843e19887f64dde75055cf8842fc4db2171eff45johnlev
ae115bc77f6fcde83175c75b4206dc2e50747966mrj /*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Find the pte that will map this address. This creates any
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * missing intermediate level page tables.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj ptep = find_pte(va, &pte_physaddr, level, 0);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (ptep == NULL)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj bop_panic("kbm_map: find_pte returned NULL");
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev if (HYPERVISOR_update_va_mapping(va, pteval, UVMF_INVLPG | UVMF_LOCAL))
843e19887f64dde75055cf8842fc4db2171eff45johnlev bop_panic("HYPERVISOR_update_va_mapping() failed");
843e19887f64dde75055cf8842fc4db2171eff45johnlev#else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *ptep = pteval;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *((x86pte32_t *)ptep) = pteval;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj mmu_tlbflush_entry((caddr_t)va);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev
843e19887f64dde75055cf8842fc4db2171eff45johnlev/*
843e19887f64dde75055cf8842fc4db2171eff45johnlev * Add a mapping for the machine page at the given virtual address.
843e19887f64dde75055cf8842fc4db2171eff45johnlev */
843e19887f64dde75055cf8842fc4db2171eff45johnlevvoid
843e19887f64dde75055cf8842fc4db2171eff45johnlevkbm_map_ma(maddr_t ma, uintptr_t va, uint_t level)
843e19887f64dde75055cf8842fc4db2171eff45johnlev{
843e19887f64dde75055cf8842fc4db2171eff45johnlev paddr_t pte_physaddr;
843e19887f64dde75055cf8842fc4db2171eff45johnlev x86pte_t pteval;
843e19887f64dde75055cf8842fc4db2171eff45johnlev
843e19887f64dde75055cf8842fc4db2171eff45johnlev pteval = ma | PT_NOCONSIST | PT_VALID | PT_REF | PT_WRITABLE;
843e19887f64dde75055cf8842fc4db2171eff45johnlev if (level == 1)
843e19887f64dde75055cf8842fc4db2171eff45johnlev pteval |= PT_PAGESIZE;
843e19887f64dde75055cf8842fc4db2171eff45johnlev
843e19887f64dde75055cf8842fc4db2171eff45johnlev /*
843e19887f64dde75055cf8842fc4db2171eff45johnlev * try update_va_mapping first - fails if page table is missing.
843e19887f64dde75055cf8842fc4db2171eff45johnlev */
843e19887f64dde75055cf8842fc4db2171eff45johnlev if (HYPERVISOR_update_va_mapping(va,
843e19887f64dde75055cf8842fc4db2171eff45johnlev pteval, UVMF_INVLPG | UVMF_LOCAL) == 0)
843e19887f64dde75055cf8842fc4db2171eff45johnlev return;
843e19887f64dde75055cf8842fc4db2171eff45johnlev
843e19887f64dde75055cf8842fc4db2171eff45johnlev /*
843e19887f64dde75055cf8842fc4db2171eff45johnlev * Find the pte that will map this address. This creates any
843e19887f64dde75055cf8842fc4db2171eff45johnlev * missing intermediate level page tables
843e19887f64dde75055cf8842fc4db2171eff45johnlev */
843e19887f64dde75055cf8842fc4db2171eff45johnlev (void) find_pte(va, &pte_physaddr, level, 0);
843e19887f64dde75055cf8842fc4db2171eff45johnlev
843e19887f64dde75055cf8842fc4db2171eff45johnlev if (HYPERVISOR_update_va_mapping(va,
843e19887f64dde75055cf8842fc4db2171eff45johnlev pteval, UVMF_INVLPG | UVMF_LOCAL) != 0)
843e19887f64dde75055cf8842fc4db2171eff45johnlev bop_panic("HYPERVISOR_update_va_mapping failed");
843e19887f64dde75055cf8842fc4db2171eff45johnlev}
843e19887f64dde75055cf8842fc4db2171eff45johnlev
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif /* __xpv */
843e19887f64dde75055cf8842fc4db2171eff45johnlev
843e19887f64dde75055cf8842fc4db2171eff45johnlev
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Probe the boot time page tables to find the first mapping
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * including va (or higher) and return non-zero if one is found.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * va is updated to the starting address and len to the pagesize.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * pp will be set to point to the 1st page_t of the mapped page(s).
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Note that if va is in the middle of a large page, the returned va
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * will be less than what was asked for.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjint
ae115bc77f6fcde83175c75b4206dc2e50747966mrjkbm_probe(uintptr_t *va, size_t *len, pfn_t *pfn, uint_t *prot)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj uintptr_t probe_va;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj x86pte_t *ptep;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj paddr_t pte_physaddr;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj x86pte_t pte_val;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj level_t l;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (khat_running)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj panic("kbm_probe() called too late");
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *len = 0;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *pfn = PFN_INVALID;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *prot = 0;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj probe_va = *va;
ae115bc77f6fcde83175c75b4206dc2e50747966mrjrestart_new_va:
ae115bc77f6fcde83175c75b4206dc2e50747966mrj l = top_level;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj for (;;) {
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (IN_VA_HOLE(probe_va))
ae115bc77f6fcde83175c75b4206dc2e50747966mrj probe_va = mmu.hole_end;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (IN_HYPERVISOR_VA(probe_va))
843e19887f64dde75055cf8842fc4db2171eff45johnlev#if defined(__amd64) && defined(__xpv)
843e19887f64dde75055cf8842fc4db2171eff45johnlev probe_va = HYPERVISOR_VIRT_END;
843e19887f64dde75055cf8842fc4db2171eff45johnlev#else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (0);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj /*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * If we don't have a valid PTP/PTE at this level
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * then we can bump VA by this level's pagesize and try again.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * When the probe_va wraps around, we are done.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj ptep = find_pte(probe_va, &pte_physaddr, l, 1);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (ptep == NULL)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj bop_panic("kbm_probe: find_pte returned NULL");
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj pte_val = *ptep;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj pte_val = *((x86pte32_t *)ptep);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (!PTE_ISVALID(pte_val)) {
ae115bc77f6fcde83175c75b4206dc2e50747966mrj probe_va = (probe_va & BOOT_MASK(l)) + BOOT_SZ(l);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (probe_va <= *va)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (0);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj goto restart_new_va;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj }
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj /*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * If this entry is a pointer to a lower level page table
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * go down to it.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (!PTE_ISPAGE(pte_val, l)) {
ae115bc77f6fcde83175c75b4206dc2e50747966mrj ASSERT(l > 0);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj --l;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj continue;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj }
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj /*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * We found a boot level page table entry
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *len = BOOT_SZ(l);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *va = probe_va & ~(*len - 1);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *pfn = PTE2PFN(pte_val, l);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *prot = PROT_READ | PROT_EXEC;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (PTE_GET(pte_val, PT_WRITABLE))
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *prot |= PROT_WRITE;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj /*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * pt_nx is cleared if processor doesn't support NX bit
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (PTE_GET(pte_val, mmu.pt_nx))
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *prot &= ~PROT_EXEC;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (1);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj }
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Destroy a boot loader page table 4K mapping.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjvoid
ae115bc77f6fcde83175c75b4206dc2e50747966mrjkbm_unmap(uintptr_t va)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (khat_running)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj panic("kbm_unmap() called too late");
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else {
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev (void) HYPERVISOR_update_va_mapping(va, 0,
843e19887f64dde75055cf8842fc4db2171eff45johnlev UVMF_INVLPG | UVMF_LOCAL);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj x86pte_t *ptep;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj level_t level = 0;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj uint_t probe_only = 1;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj ptep = find_pte(va, NULL, level, probe_only);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (ptep == NULL)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *ptep = 0;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *((x86pte32_t *)ptep) = 0;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj mmu_tlbflush_entry((caddr_t)va);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj }
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Change a boot loader page table 4K mapping.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Returns the pfn of the old mapping.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjpfn_t
ae115bc77f6fcde83175c75b4206dc2e50747966mrjkbm_remap(uintptr_t va, pfn_t pfn)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj x86pte_t *ptep;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj level_t level = 0;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj uint_t probe_only = 1;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj x86pte_t pte_val = pa_to_ma(pfn_to_pa(pfn)) | PT_WRITABLE |
ae115bc77f6fcde83175c75b4206dc2e50747966mrj PT_NOCONSIST | PT_VALID;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj x86pte_t old_pte;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (khat_running)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj panic("kbm_remap() called too late");
ae115bc77f6fcde83175c75b4206dc2e50747966mrj ptep = find_pte(va, NULL, level, probe_only);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (ptep == NULL)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj bop_panic("kbm_remap: find_pte returned NULL");
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj old_pte = *ptep;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj old_pte = *((x86pte32_t *)ptep);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev if (HYPERVISOR_update_va_mapping(va, pte_val, UVMF_INVLPG | UVMF_LOCAL))
843e19887f64dde75055cf8842fc4db2171eff45johnlev bop_panic("HYPERVISOR_update_va_mapping() failed");
843e19887f64dde75055cf8842fc4db2171eff45johnlev#else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *((x86pte_t *)ptep) = pte_val;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *((x86pte32_t *)ptep) = pte_val;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj mmu_tlbflush_entry((caddr_t)va);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (!(old_pte & PT_VALID) || ma_to_pa(old_pte) == -1)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (PFN_INVALID);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (mmu_btop(ma_to_pa(old_pte)));
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * Change a boot loader page table 4K mapping to read only.
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjvoid
ae115bc77f6fcde83175c75b4206dc2e50747966mrjkbm_read_only(uintptr_t va, paddr_t pa)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj x86pte_t pte_val = pa_to_ma(pa) |
ae115bc77f6fcde83175c75b4206dc2e50747966mrj PT_NOCONSIST | PT_REF | PT_MOD | PT_VALID;
843e19887f64dde75055cf8842fc4db2171eff45johnlev
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev if (HYPERVISOR_update_va_mapping(va, pte_val, UVMF_INVLPG | UVMF_LOCAL))
843e19887f64dde75055cf8842fc4db2171eff45johnlev bop_panic("HYPERVISOR_update_va_mapping() failed");
843e19887f64dde75055cf8842fc4db2171eff45johnlev#else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj x86pte_t *ptep;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj level_t level = 0;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj ptep = find_pte(va, NULL, level, 0);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (ptep == NULL)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj bop_panic("kbm_read_only: find_pte returned NULL");
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *ptep = pte_val;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *((x86pte32_t *)ptep) = pte_val;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj mmu_tlbflush_entry((caddr_t)va);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj/*
ae115bc77f6fcde83175c75b4206dc2e50747966mrj * interfaces for kernel debugger to access physical memory
ae115bc77f6fcde83175c75b4206dc2e50747966mrj */
ae115bc77f6fcde83175c75b4206dc2e50747966mrjstatic x86pte_t save_pte;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrjvoid *
ae115bc77f6fcde83175c75b4206dc2e50747966mrjkbm_push(paddr_t pa)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj static int first_time = 1;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (first_time) {
ae115bc77f6fcde83175c75b4206dc2e50747966mrj first_time = 0;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (window);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj }
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj save_pte = *((x86pte_t *)pte_to_window);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj save_pte = *((x86pte32_t *)pte_to_window);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (kbm_remap_window(pa, 0));
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrjvoid
ae115bc77f6fcde83175c75b4206dc2e50747966mrjkbm_pop(void)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev if (HYPERVISOR_update_va_mapping((uintptr_t)window, save_pte,
843e19887f64dde75055cf8842fc4db2171eff45johnlev UVMF_INVLPG | UVMF_LOCAL) < 0)
843e19887f64dde75055cf8842fc4db2171eff45johnlev bop_panic("HYPERVISOR_update_va_mapping() failed");
843e19887f64dde75055cf8842fc4db2171eff45johnlev#else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *((x86pte_t *)pte_to_window) = save_pte;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *((x86pte32_t *)pte_to_window) = save_pte;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj mmu_tlbflush_entry(window);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrjx86pte_t
ae115bc77f6fcde83175c75b4206dc2e50747966mrjget_pteval(paddr_t table, uint_t index)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj void *table_ptr = kbm_remap_window(table, 0);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (((x86pte_t *)table_ptr)[index]);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (((x86pte32_t *)table_ptr)[index]);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifndef __xpv
ae115bc77f6fcde83175c75b4206dc2e50747966mrjvoid
ae115bc77f6fcde83175c75b4206dc2e50747966mrjset_pteval(paddr_t table, uint_t index, uint_t level, x86pte_t pteval)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj void *table_ptr = kbm_remap_window(table, 0);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (kbm_pae_support)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj ((x86pte_t *)table_ptr)[index] = pteval;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj ((x86pte32_t *)table_ptr)[index] = pteval;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (level == top_level && level == 2)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj reload_cr3();
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrjpaddr_t
ae115bc77f6fcde83175c75b4206dc2e50747966mrjmake_ptable(x86pte_t *pteval, uint_t level)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj paddr_t new_table;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj void *table_ptr;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj new_table = do_bop_phys_alloc(MMU_PAGESIZE, MMU_PAGESIZE);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj table_ptr = kbm_remap_window(new_table, 1);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj bzero(table_ptr, MMU_PAGESIZE);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#ifdef __xpv
843e19887f64dde75055cf8842fc4db2171eff45johnlev /* Remove write permission to the new page table. */
843e19887f64dde75055cf8842fc4db2171eff45johnlev (void) kbm_remap_window(new_table, 0);
843e19887f64dde75055cf8842fc4db2171eff45johnlev#endif
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj if (level == top_level && level == 2)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *pteval = pa_to_ma(new_table) | PT_VALID;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj else
ae115bc77f6fcde83175c75b4206dc2e50747966mrj *pteval = pa_to_ma(new_table) |
ae115bc77f6fcde83175c75b4206dc2e50747966mrj PT_VALID | PT_REF | PT_USER | PT_WRITABLE;
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return (new_table);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}
ae115bc77f6fcde83175c75b4206dc2e50747966mrj
ae115bc77f6fcde83175c75b4206dc2e50747966mrjx86pte_t *
ae115bc77f6fcde83175c75b4206dc2e50747966mrjmap_pte(paddr_t table, uint_t index)
ae115bc77f6fcde83175c75b4206dc2e50747966mrj{
ae115bc77f6fcde83175c75b4206dc2e50747966mrj void *table_ptr = kbm_remap_window(table, 0);
ae115bc77f6fcde83175c75b4206dc2e50747966mrj return ((x86pte_t *)((caddr_t)table_ptr + index * pte_size));
ae115bc77f6fcde83175c75b4206dc2e50747966mrj}