#pragma ident "%Z%%M% %I% %E% SMI"

#include <sys/machsystm.h>

#include <sys/archsystm.h>

#include <sys/vm.h>

#include <sys/cpu.h>

#include <sys/atomic.h>

#include <sys/reboot.h>

#include <sys/kdi.h>

#include <sys/bootconf.h>

#include <sys/memlist_plat.h>

#include <sys/memlist_impl.h>

#include <sys/prom_plat.h>

#include <sys/prom_isa.h>

#include <sys/autoconf.h>

#include <sys/intreg.h>

#include <sys/ivintr.h>

#include <sys/fpu/fpusystm.h>

#include <sys/iommutsb.h>

#include <vm/vm_dep.h>

#include <vm/seg_dev.h>

#include <vm/seg_kmem.h>

#include <vm/seg_kpm.h>

#include <vm/seg_map.h>

#include <vm/seg_kp.h>

#include <sys/sysconf.h>

#include <vm/hat_sfmmu.h>

#include <sys/kobj.h>

#include <sys/sun4asi.h>

#include <sys/clconf.h>

#include <sys/platform_module.h>

#include <sys/panic.h>

#include <sys/cpu_sgnblk_defs.h>

#include <sys/clock.h>

#include <sys/cmn_err.h>

#include <sys/promif.h>

#include <sys/prom_debug.h>

#include <sys/traptrace.h>

#include <sys/memnode.h>

#include <sys/mem_cage.h>

#include <sys/mmu.h>

extern void setup_trap_table(void);

extern void cpu_intrq_setup(struct cpu *);

extern void cpu_intrq_register(struct cpu *);

extern void contig_mem_init(void);

extern void mach_dump_buffer_init(void);

extern void mach_descrip_init(void);

extern void mach_descrip_startup_fini(void);

extern void mach_memscrub(void);

extern void mach_fpras(void);

extern void mach_cpu_halt_idle(void);

extern void mach_hw_copy_limit(void);

extern void load_mach_drivers(void);

extern void load_tod_module(void);

#pragma weak load_tod_module

extern int ndata_alloc_mmfsa(struct memlist *ndata);

#pragma weak ndata_alloc_mmfsa

extern void cif_init(void);

#pragma weak cif_init

extern void parse_idprom(void);

extern void add_vx_handler(char *, int, void (*)(cell_t *));

extern void mem_config_init(void);

extern void memseg_remap_init(void);

extern int vac_size; /* cache size in bytes */

extern uint_t vac_mask; /* VAC alignment consistency mask */

extern uint_t vac_colors;

void *romp; /* veritas driver won't load without romp 4154976 */

pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */

pgcnt_t segkpsize =

btop(SEGKPDEFSIZE); /* size of segkp segment in pages */

uint_t segmap_percent = 12; /* Size of segmap segment */

int use_cache = 1; /* cache not reliable (605 bugs) with MP */

int vac_copyback = 1;

char *cache_mode = NULL;

int use_mix = 1;

int prom_debug = 0;

struct bootops *bootops = 0; /* passed in from boot in %o2 */

caddr_t boot_tba; /* %tba at boot - used by kmdb */

uint_t tba_taken_over = 0;

caddr_t s_text; /* start of kernel text segment */

caddr_t e_text; /* end of kernel text segment */

caddr_t s_data; /* start of kernel data segment */

caddr_t e_data; /* end of kernel data segment */

caddr_t modtext; /* beginning of module text */

size_t modtext_sz; /* size of module text */

caddr_t moddata; /* beginning of module data reserve */

caddr_t e_moddata; /* end of module data reserve */

caddr_t econtig32; /* end of first blk of contiguous kernel */

caddr_t ncbase; /* beginning of non-cached segment */

caddr_t ncend; /* end of non-cached segment */

caddr_t sdata; /* beginning of data segment */

caddr_t extra_etva; /* beginning of unused nucleus text */

pgcnt_t extra_etpg; /* number of pages of unused nucleus text */

size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */

caddr_t nalloc_base; /* beginning of nucleus allocation */

caddr_t nalloc_end; /* end of nucleus allocatable memory */

caddr_t valloc_base; /* beginning of kvalloc segment */

caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */

caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */

uintptr_t shm_alignment = 0; /* VAC address consistency modulus */

struct memlist *phys_install; /* Total installed physical memory */

struct memlist *phys_avail; /* Available (unreserved) physical memory */

struct memlist *virt_avail; /* Available (unmapped?) virtual memory */

struct memlist ndata; /* memlist of nucleus allocatable memory */

int memexp_flag; /* memory expansion card flag */

uint64_t ecache_flushaddr; /* physical address used for flushing E$ */

pgcnt_t obp_pages; /* Physical pages used by OBP */

long page_hashsz; /* Size of page hash table (power of two) */

struct page *pp_base; /* Base of system page struct array */

size_t pp_sz; /* Size in bytes of page struct array */

struct page **page_hash; /* Page hash table */

struct seg ktextseg; /* Segment used for kernel executable image */

struct seg kvalloc; /* Segment used for "valloc" mapping */

struct seg kpseg; /* Segment used for pageable kernel virt mem */

struct seg ktexthole; /* Segment used for nucleus text hole */

struct seg kmapseg; /* Segment used for generic kernel mappings */

struct seg kpmseg; /* Segment used for physical mapping */

struct seg kdebugseg; /* Segment used for the kernel debugger */

uintptr_t kpm_pp_base; /* Base of system kpm_page array */

size_t kpm_pp_sz; /* Size of system kpm_page array */

pgcnt_t kpm_npages; /* How many kpm pages are managed */

struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */

struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */

struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */

struct vnode kdebugvp;

struct seg kmem64;

struct memseg *memseg_base;

size_t memseg_sz; /* Used to translate a va to page */

struct vnode unused_pages_vp;

size_t pagehash_sz;

uint64_t memlist_sz;

char tbr_wr_addr_inited = 0;

static void memlist_add(uint64_t, uint64_t, struct memlist **,

struct memlist **);

static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t,

pgcnt_t);

static void kvm_init(void);

static void startup_init(void);

static void startup_memlist(void);

static void startup_modules(void);

static void startup_bop_gone(void);

static void startup_vm(void);

static void startup_end(void);

static void setup_cage_params(void);

static void startup_create_io_node(void);

static pgcnt_t npages;

static struct memlist *memlist;

void *memlist_end;

static pgcnt_t bop_alloc_pages;

static caddr_t hblk_base;

uint_t hblk_alloc_dynamic = 0;

uint_t hblk1_min = H1MIN;

uint_t hblk8_min;

int iam_positron(void);

#pragma weak iam_positron

static void do_prom_version_check(void);

static void kpm_init(void);

static void kpm_npages_setup(int);

static void kpm_memseg_init(void);

int thermal_powerdown_delay = 1200;

int page_relocate_ready = 0;

#ifdef DEBUGGING_MEM

static int debugging_mem;

static void

printmemlist(char *title, struct memlist *list)

{

if (!debugging_mem)

return;

printf("%s\n", title);

while (list) {

prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",

(uint32_t)(list->address >> 32), (uint32_t)list->address,

(uint32_t)(list->size >> 32), (uint32_t)(list->size));

list = list->next;

}

}

printmemseg(struct memseg *memseg)

{

if (!debugging_mem)

return;

printf("memseg\n");

while (memseg) {

prom_printf("\tpage = 0x%p, epage = 0x%p, "

"pfn = 0x%x, epfn = 0x%x\n",

memseg->pages, memseg->epages,

memseg->pages_base, memseg->pages_end);

memseg = memseg->next;

}

}

#define debug_pause(str) halt((str))

#define MPRINTF(str) if (debugging_mem) prom_printf((str))

#define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a))

#define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b))

#define MPRINTF3(str, a, b, c) \

if (debugging_mem) prom_printf((str), (a), (b), (c))

#else /* DEBUGGING_MEM */

#define MPRINTF(str)

#define MPRINTF1(str, a)

#define MPRINTF2(str, a, b)

#define MPRINTF3(str, a, b, c)

#endif /* DEBUGGING_MEM */

#ifdef DEBUG

static int bootops_gone_on = 0;

#define BOOTOPS_GONE() \

if (bootops_gone_on) \

prom_printf("The bootops vec is zeroed now!\n");

#else

#define BOOTOPS_GONE()

#endif /* DEBUG */

extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);

extern uint64_t ecache_flush_address(void);

#pragma weak load_platform_modules

#pragma weak plat_startup_memlist

#pragma weak ecache_init_scrub_flush_area

#pragma weak ecache_flush_address

int kernel_cage_enable = 1;

static void

setup_cage_params(void)

{

void (*func)(void);

func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);

if (func != NULL) {

(*func)();

return;

}

if (kernel_cage_enable == 0) {

return;

}

kcage_range_lock();

if (kcage_range_init(phys_avail, 1) == 0) {

kcage_init(total_pages / 256);

}

kcage_range_unlock();

if (kcage_on) {

cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");

} else {

cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");

}

}

startup(void)

{

startup_init();

if (&startup_platform)

startup_platform();

startup_memlist();

startup_modules();

setup_cage_params();

startup_bop_gone();

startup_vm();

startup_end();

}

struct regs sync_reg_buf;

uint64_t sync_tt;

sync_handler(void)

{

struct trap_info ti;

int i;

/*

for (i = 0; i < NCPU; i++) {

if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {

cpu[i]->cpu_flags &= ~CPU_READY;

cpu[i]->cpu_flags |= CPU_QUIESCED;

CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);

}

}

/*

if (!panicstr && (curthread->t_panic_trap == NULL)) {

ti.trap_type = sync_tt;

ti.trap_regs = &sync_reg_buf;

ti.trap_addr = NULL;

ti.trap_mmu_fsr = 0x0;

curthread->t_panic_trap = &ti;

}

/*

if (panicstr)

CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);

nopanicdebug = 1; /* do not perform debug_enter() prior to dump */

panic("sync initiated");

}

static void

startup_init(void)

{

/*

char sync_str[] =

"warning @ warning off : sync "

"%%tl-c %%tstate h# %p x! "

"%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "

"%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "

"%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "

"%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "

"%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "

"%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "

"%%y h# %p l! %%tl-c %%tt h# %p x! "

"sync ; warning !";

/*

char bp[sizeof (sync_str) + 16 * 20];

(void) check_boot_version(BOP_GETVERSION(bootops));

/*

ptl1_init_cpu(&cpu0);

/*

ppmapinit();

/*

init_vx_handler();

/*

(void) sprintf((char *)bp, sync_str,

(void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,

(void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,

(void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,

(void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,

(void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,

(void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,

(void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,

(void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,

(void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,

(void *)&sync_reg_buf.r_y, (void *)&sync_tt);

prom_interpret(bp, 0, 0, 0, 0, 0);

add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);

}

static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail;

static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;

#define IVSIZE ((MAXIVNUM + 1) * sizeof (struct intr_vector))

#define MINMOVE_RAM_MB ((size_t)1900)

#define MB_TO_BYTES(mb) ((mb) * 1048576ul)

pgcnt_t tune_npages = (pgcnt_t)

(MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);

static void

startup_memlist(void)

{

size_t alloc_sz;

size_t ctrs_sz;

caddr_t alloc_base;

caddr_t ctrs_base, ctrs_end;

caddr_t memspace;

caddr_t va;

int memblocks = 0;

struct memlist *cur;

size_t syslimit = (size_t)SYSLIMIT;

size_t sysbase = (size_t)SYSBASE;

int alloc_alignsize = MMU_PAGESIZE;

extern void page_coloring_init(void);

/*

/*

nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);

moddata = nalloc_base;

e_moddata = nalloc_base + MODDATA;

nalloc_base = e_moddata;

nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);

valloc_base = nalloc_base;

/*

sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M);

PRM_DEBUG(moddata);

PRM_DEBUG(nalloc_base);

PRM_DEBUG(nalloc_end);

PRM_DEBUG(sdata);

/*

PRM_DEBUG(e_text);

modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);

if (((uintptr_t)modtext & MMU_PAGEMASK4M) != (uintptr_t)s_text)

panic("nucleus text overflow");

modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -

modtext;

PRM_DEBUG(modtext);

PRM_DEBUG(modtext_sz);

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

/*

installed_top_size_memlist_array(boot_physinstalled,

boot_physinstalled_len, &physmax, &physinstalled);

PRM_DEBUG(physinstalled);

PRM_DEBUG(physmax);

/* Fill out memory nodes config structure */

startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);

/*

size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks);

/*

obp_pages = physinstalled - npages;

/*

if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) &&

modtext_sz > MODTEXT_SM_CAP) {

extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP);

modtext_sz = MODTEXT_SM_CAP;

} else

extra_etpg = 0;

PRM_DEBUG(extra_etpg);

PRM_DEBUG(modtext_sz);

extra_etva = modtext + modtext_sz;

PRM_DEBUG(extra_etva);

/*

npages += extra_etpg;

npages += mmu_btopr(nalloc_end - nalloc_base);

PRM_DEBUG(npages);

/*

/*

ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);

/*

if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))

cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");

/*

if (ndata_alloc_tsbs(&ndata, npages) != 0)

cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");

/*

if (ndata_alloc_cpus(&ndata) != 0)

cmn_err(CE_PANIC, "no more nucleus memory after cpu alloc");

/*

if (ndata_alloc_dmv(&ndata) != 0)

cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");

page_coloring_init();

/*

if (ndata_alloc_page_freelists(&ndata, 0) != 0)

cmn_err(CE_PANIC,

"no more nucleus memory after page free lists alloc");

if (kpm_enable) {

kpm_init();

/*

kpm_npages_setup(memblocks + 4);

}

/*

if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0)

cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");

/*

/*

alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);

alloc_base = sfmmu_ktsb_alloc(alloc_base);

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);

PRM_DEBUG(alloc_base);

/*

alloc_base = iommu_tsb_init(alloc_base);

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,

ecache_alignsize);

PRM_DEBUG(alloc_base);

/*

if (&plat_startup_memlist) {

alloc_base = plat_startup_memlist(alloc_base);

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,

ecache_alignsize);

PRM_DEBUG(alloc_base);

}

/*

alloc_alignsize = ((npages >= tune_npages) ? MMU_PAGESIZE4M :

MMU_PAGESIZE);

PRM_DEBUG(tune_npages);

PRM_DEBUG(alloc_alignsize);

/*

econtig32 = alloc_base;

PRM_DEBUG(econtig32);

if (econtig32 > (caddr_t)KERNEL_LIMIT32)

cmn_err(CE_PANIC, "econtig32 too big");

/*

kmem64_base = (caddr_t)syslimit;

PRM_DEBUG(kmem64_base);

alloc_base = (caddr_t)roundup((uintptr_t)kmem64_base, alloc_alignsize);

/*

if (khme_hash == NULL || uhme_hash == NULL) {

/*

alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize);

PRM_DEBUG(alloc_base);

PRM_DEBUG(khme_hash);

PRM_DEBUG(uhme_hash);

}

/*

if (max_mem_nodes > 1) {

int mnode;

caddr_t alloc_start = alloc_base;

for (mnode = 1; mnode < max_mem_nodes; mnode++) {

alloc_base = alloc_page_freelists(mnode, alloc_base,

ecache_alignsize);

}

if (alloc_base > alloc_start) {

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,

alloc_alignsize);

if ((caddr_t)BOP_ALLOC(bootops, alloc_start,

alloc_base - alloc_start,

alloc_alignsize) != alloc_start)

cmn_err(CE_PANIC,

"Unable to alloc page freelists\n");

}

PRM_DEBUG(alloc_base);

}

if (!mml_table) {

size_t mmltable_sz;

/*

mmltable_sz = sizeof (kmutex_t) * mml_table_sz;

alloc_sz = roundup(mmltable_sz, alloc_alignsize);

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,

alloc_alignsize);

if ((mml_table = (kmutex_t *)BOP_ALLOC(bootops, alloc_base,

alloc_sz, alloc_alignsize)) != (kmutex_t *)alloc_base)

panic("mml_table alloc failure");

alloc_base += alloc_sz;

PRM_DEBUG(mml_table);

PRM_DEBUG(alloc_base);

}

if (kpm_enable && !(kpmp_table || kpmp_stable)) {

size_t kpmptable_sz;

caddr_t table;

/*

kpmptable_sz = (kpm_smallpages == 0) ?

sizeof (kpm_hlk_t) * kpmp_table_sz :

sizeof (kpm_shlk_t) * kpmp_stable_sz;

alloc_sz = roundup(kpmptable_sz, alloc_alignsize);

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,

alloc_alignsize);

table = BOP_ALLOC(bootops, alloc_base, alloc_sz,

alloc_alignsize);

if (table != alloc_base)

panic("kpmp_table or kpmp_stable alloc failure");

if (kpm_smallpages == 0) {

kpmp_table = (kpm_hlk_t *)table;

PRM_DEBUG(kpmp_table);

} else {

kpmp_stable = (kpm_shlk_t *)table;

PRM_DEBUG(kpmp_stable);

}

alloc_base += alloc_sz;

PRM_DEBUG(alloc_base);

}

if (&ecache_init_scrub_flush_area) {

/*

alloc_base = ecache_init_scrub_flush_area(alloc_base);

PRM_DEBUG(alloc_base);

}

/*

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

/*

size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks);

PRM_DEBUG(npages);

/*

npages += extra_etpg;

/*

ndata_remain_sz = ndata_maxsize(&ndata);

PRM_DEBUG(ndata_remain_sz);

pp_sz = sizeof (struct page) * npages;

/*

if (ndata_remain_sz > pp_sz) {

size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize);

npages += mmu_btop(spare);

pp_sz = npages * sizeof (struct page);

pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize);

}

/*

if (physmem == 0 || physmem > npages)

physmem = npages;

/*

if (pp_base == NULL) {

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,

alloc_alignsize);

alloc_sz = roundup(pp_sz, alloc_alignsize);

if ((pp_base = (struct page *)BOP_ALLOC(bootops,

alloc_base, alloc_sz, alloc_alignsize)) !=

(struct page *)alloc_base)

panic("page alloc failure");

alloc_base += alloc_sz;

}

/*

page_hashsz = npages / PAGE_HASHAVELEN;

page_hashsz = 1 << highbit((ulong_t)page_hashsz);

pagehash_sz = sizeof (struct page *) * page_hashsz;

/*

page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize);

alloc_sz = (page_hash == NULL ? pagehash_sz : 0);

/*

ctrs_sz = page_ctrs_sz();

ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize);

if (ctrs_base == NULL)

alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz;

/*

memseg_sz = sizeof (struct memseg) * (memblocks + 4);

memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize);

if (memseg_base == NULL)

alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz;

if (kpm_enable) {

/*

kpm_npages_setup(memblocks + 4);

kpm_pp_sz = (kpm_smallpages == 0) ?

kpm_npages * sizeof (kpm_page_t):

kpm_npages * sizeof (kpm_spage_t);

kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz,

ecache_alignsize);

if (kpm_pp_base == NULL)

alloc_sz = roundup(alloc_sz, ecache_alignsize) +

kpm_pp_sz;

}

if (alloc_sz > 0) {

uintptr_t bop_base;

/*

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,

alloc_alignsize);

alloc_sz = roundup(alloc_sz, alloc_alignsize);

if ((bop_base = (uintptr_t)BOP_ALLOC(bootops, alloc_base,

alloc_sz, alloc_alignsize)) != (uintptr_t)alloc_base)

panic("system page struct alloc failure");

alloc_base += alloc_sz;

if (page_hash == NULL) {

page_hash = (struct page **)bop_base;

bop_base = roundup(bop_base + pagehash_sz,

ecache_alignsize);

}

if (ctrs_base == NULL) {

ctrs_base = (caddr_t)bop_base;

bop_base = roundup(bop_base + ctrs_sz,

ecache_alignsize);

}

if (memseg_base == NULL) {

memseg_base = (struct memseg *)bop_base;

bop_base = roundup(bop_base + memseg_sz,

ecache_alignsize);

}

if (kpm_enable && kpm_pp_base == NULL) {

kpm_pp_base = (uintptr_t)bop_base;

bop_base = roundup(bop_base + kpm_pp_sz,

ecache_alignsize);

}

ASSERT(bop_base <= (uintptr_t)alloc_base);

}

/*

ctrs_end = page_ctrs_alloc(ctrs_base);

ASSERT(ctrs_base + ctrs_sz >= ctrs_end);

PRM_DEBUG(page_hash);

PRM_DEBUG(memseg_base);

PRM_DEBUG(kpm_pp_base);

PRM_DEBUG(kpm_pp_sz);

PRM_DEBUG(pp_base);

PRM_DEBUG(pp_sz);

PRM_DEBUG(alloc_base);

#ifdef TRAPTRACE

/*

alloc_base = trap_trace_alloc(alloc_base);

PRM_DEBUG(alloc_base);

#endif /* TRAPTRACE */

if (kmem64_base) {

/*

kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base,

alloc_alignsize);

PRM_DEBUG(kmem64_base);

PRM_DEBUG(kmem64_end);

}

/*

memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vector,

IVSIZE, MMU_PAGESIZE);

if (memspace != (caddr_t)intr_vector)

panic("interrupt table allocation failure");

/*

if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1)

panic("could not retrieve property \"extent\"");

/*

va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE +

roundup(IVSIZE, MMU_PAGESIZE));

memlist_sz *= 4;

memlist_sz = roundup(memlist_sz, MMU_PAGESIZE);

memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN);

if (memspace == NULL)

halt("Boot allocation failed.");

memlist = (struct memlist *)memspace;

memlist_end = (char *)memspace + memlist_sz;

PRM_DEBUG(memlist);

PRM_DEBUG(memlist_end);

PRM_DEBUG(sysbase);

PRM_DEBUG(syslimit);

kernelheap_init((void *)sysbase, (void *)syslimit,

(caddr_t)sysbase + PAGESIZE, NULL, NULL);

/*

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

/*

virt_avail = memlist;

copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);

for (cur = virt_avail; cur->next; cur = cur->next) {

uint64_t range_base, range_size;

if ((range_base = cur->address + cur->size) < (uint64_t)sysbase)

continue;

if (range_base >= (uint64_t)syslimit)

break;

/*

range_size = MIN(cur->next->address,

(uint64_t)syslimit) - range_base;

(void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,

0, 0, (void *)range_base, (void *)(range_base + range_size),

VM_NOSLEEP | VM_BESTFIT | VM_PANIC);

}

phys_avail = memlist;

(void) copy_physavail(boot_physavail, boot_physavail_len,

&memlist, 0, 0);

/*

if (extra_etpg)

memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg),

&memlist, &phys_avail);

/*

if ((nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE)) == NULL)

nalloc_base = nalloc_end;

ndata_remain_sz = nalloc_end - nalloc_base;

if (ndata_remain_sz >= MMU_PAGESIZE)

memlist_add(va_to_pa(nalloc_base),

(uint64_t)ndata_remain_sz, &memlist, &phys_avail);

PRM_DEBUG(memlist);

PRM_DEBUG(memlist_sz);

PRM_DEBUG(memspace);

if ((caddr_t)memlist > (memspace + memlist_sz))

panic("memlist overflow");

PRM_DEBUG(pp_base);

PRM_DEBUG(memseg_base);

PRM_DEBUG(npages);

/*

kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages);

availrmem_initial = availrmem = freemem;

PRM_DEBUG(availrmem);

/*

page_lock_init();

/*

kmem_init();

/*

bp_init(shm_alignment, HAT_STRICTORDER);

/*

(void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,

panicbuf, panicbuf + PANICBUFSIZE,

VM_NOSLEEP | VM_BESTFIT | VM_PANIC);

(void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,

intr_vector, (caddr_t)intr_vector + IVSIZE,

VM_NOSLEEP | VM_BESTFIT | VM_PANIC);

mem_config_init();

}

static void

startup_modules(void)

{

int proplen, nhblk1, nhblk8;

size_t nhblksz;

pgcnt_t hblk_pages, pages_per_hblk;

size_t hme8blk_sz, hme1blk_sz;

/*

proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message");

if (proplen > 0) {

char *msg;

size_t len = (size_t)proplen;

msg = kmem_zalloc(len, KM_SLEEP);

(void) BOP_GETPROP(bootops, "boot-message", msg);

cmn_err(CE_CONT, "?%s\n", msg);

kmem_free(msg, len);

}

/*

if (&set_platform_defaults)

set_platform_defaults();

/*

param_calc(0);

mod_setup();

/*

if (&iam_positron && iam_positron()) {

cmn_err(CE_WARN, "This hardware platform is not supported"

" by this release of Solaris.\n");

#ifdef DEBUG

prom_enter_mon(); /* Type 'go' to resume */

cmn_err(CE_WARN, "Booting an unsupported platform.\n");

cmn_err(CE_WARN, "Booting with down-rev firmware.\n");

#else /* DEBUG */

halt(0);

#endif /* DEBUG */

}

/*

do_prom_version_check();

/*

param_init();

/*

maxmem = physmem;

/* Set segkp limits. */

ncbase = (caddr_t)SEGDEBUGBASE;

ncend = (caddr_t)SEGDEBUGBASE;

/*

hat_init();

/*

seg_init();

/*

create_va_to_tte();

/*

forthdebug_init();

/*

startup_create_io_node();

if (modloadonly("fs", "specfs") == -1)

halt("Can't load specfs");

if (modloadonly("fs", "devfs") == -1)

halt("Can't load devfs");

if (modloadonly("misc", "swapgeneric") == -1)

halt("Can't load swapgeneric");

(void) modloadonly("sys", "lbl_edition");

dispinit();

/*

parse_idprom();

/* Read cluster configuration data. */

clconf_init();

setup_ddi();

/*

if (loadrootmodules() != 0)

debug_enter("Can't load the root filesystem");

/*

if (&load_tod_module)

load_tod_module();

/*

if (&load_platform_modules)

load_platform_modules();

setcpudelay();

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);

/*

/* sfmmu_init_nucleus_hblks expects properly aligned data structures. */

hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));

hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));

pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));

bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);

nhblk8 = bop_alloc_pages / pages_per_hblk;

nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1;

hblk_pages = btopr(nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz);

bop_alloc_pages += hblk_pages;

bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);

nhblk8 = bop_alloc_pages / pages_per_hblk;

nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1;

if (nhblk1 < hblk1_min)

nhblk1 = hblk1_min;

if (nhblk8 < hblk8_min)

nhblk8 = hblk8_min;

/*

nhblk8 += boot_virtavail_len;

nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;

/* Allocate in pagesize chunks */

nhblksz = roundup(nhblksz, MMU_PAGESIZE);

hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);

sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);

}

static void

startup_bop_gone(void)

{

extern int bop_io_quiesced;

/*

mach_descrip_startup_fini();

/*

BOP_QUIESCE_IO(bootops);

bop_io_quiesced = 1;

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

/*

phys_install = memlist;

copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);

/*

if (&ecache_flush_address) {

ecache_flushaddr = ecache_flush_address();

if (ecache_flushaddr == (uint64_t)-1) {

cmn_err(CE_PANIC,

"startup: no memory to set ecache_flushaddr");

}

}

/*

ASSERT(virt_avail != NULL);

memlist_free_list(virt_avail);

virt_avail = memlist;

copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);

/*

}

startup_fixup_physavail(void)

{

struct memlist *cur;

/*

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

/*

cur = memlist;

(void) copy_physavail(boot_physavail, boot_physavail_len,

&memlist, 0, 0);

/*

if (extra_etpg)

memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg),

&memlist, &cur);

if (ndata_remain_sz >= MMU_PAGESIZE)

memlist_add(va_to_pa(nalloc_base),

(uint64_t)ndata_remain_sz, &memlist, &cur);

/*

if ((caddr_t)memlist > (caddr_t)memlist_end)

cmn_err(CE_PANIC, "startup: memlist size exceeded");

/*

fix_prom_pages(phys_avail, cur);

ASSERT(phys_avail != NULL);

memlist_free_list(phys_avail);

phys_avail = cur;

/*

bootops = (struct bootops *)NULL;

BOOTOPS_GONE();

}

static void

startup_vm(void)

{

size_t i;

struct segmap_crargs a;

struct segkpm_crargs b;

uint64_t avmem;

caddr_t va;

pgcnt_t max_phys_segkp;

int mnode;

extern int exec_lpg_disable, use_brk_lpg, use_stk_lpg, use_zmap_lpg;

/*

hat_kern_setup();

/*

setup_trap_table();

/*

install_va_to_tte();

/*

tba_taken_over = 1;

/* initialize MMU primary context register */

mmu_init_kcontext();

/*

CPUSET_ADD(cpu_ready_set, CPU->cpu_id);

CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);

/*

tbr_wr_addr_inited = 1;

/*

kvm_init();

/*

/*

if (physmem < npages) {

pgcnt_t diff, off;

struct page *pp;

struct seg kseg;

cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);

off = 0;

diff = npages - physmem;

diff -= mmu_btopr(diff * sizeof (struct page));

kseg.s_as = &kas;

while (diff--) {

pp = page_create_va(&unused_pages_vp, (offset_t)off,

MMU_PAGESIZE, PG_WAIT | PG_EXCL,

&kseg, (caddr_t)off);

if (pp == NULL)

cmn_err(CE_PANIC, "limited physmem too much!");

page_io_unlock(pp);

page_downgrade(pp);

availrmem--;

off += MMU_PAGESIZE;

}

}

/*

cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",

(ulong_t)(physinstalled) << (PAGESHIFT - 10),

(ulong_t)(physinstalled) << (PAGESHIFT - 12));

avmem = (uint64_t)freemem << PAGESHIFT;

cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);

/* For small memory systems disable automatic large pages. */

if (physmem < auto_lpg_min_physmem) {

exec_lpg_disable = 1;

use_brk_lpg = 0;

use_stk_lpg = 0;

use_zmap_lpg = 0;

}

/*

if (&plat_freelist_process) {

for (mnode = 0; mnode < max_mem_nodes; mnode++)

if (mem_node_config[mnode].exists)

plat_freelist_process(mnode);

}

/*

/* XXX - cache alignment? */

va = (caddr_t)SEGKPBASE;

ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);

max_phys_segkp = (physmem * 2);

if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {

segkpsize = btop(SEGKPDEFSIZE);

cmn_err(CE_WARN, "Illegal value for segkpsize. "

"segkpsize has been reset to %ld pages", segkpsize);

}

i = ptob(MIN(segkpsize, max_phys_segkp));

rw_enter(&kas.a_lock, RW_WRITER);

if (seg_attach(&kas, va, i, segkp) < 0)

cmn_err(CE_PANIC, "startup: cannot attach segkp");

if (segkp_create(segkp) != 0)

cmn_err(CE_PANIC, "startup: segkp_create failed");

rw_exit(&kas.a_lock);

/*

segmap_kpm = kpm_enable &&

segmap_kpm && PAGESIZE == MAXBSIZE;

if (kpm_enable) {

rw_enter(&kas.a_lock, RW_WRITER);

/*

if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,

segkpm) < 0)

cmn_err(CE_PANIC, "cannot attach segkpm");

b.prot = PROT_READ | PROT_WRITE;

b.nvcolors = shm_alignment >> MMU_PAGESHIFT;

if (segkpm_create(segkpm, (caddr_t)&b) != 0)

panic("segkpm_create segkpm");

rw_exit(&kas.a_lock);

}

/*

va = (caddr_t)SEGMAPBASE;

/*

ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);

/*

i = mmu_ptob((freemem * segmap_percent) / 100);

if (i < MINMAPSIZE)

i = MINMAPSIZE;

if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))

i = MIN(SEGMAPSIZE, mmu_ptob(freemem));

i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */

rw_enter(&kas.a_lock, RW_WRITER);

if (seg_attach(&kas, va, i, segkmap) < 0)

cmn_err(CE_PANIC, "cannot attach segkmap");

a.prot = PROT_READ | PROT_WRITE;

a.shmsize = shm_alignment;

a.nfreelist = 0; /* use segmap driver defaults */

if (segmap_create(segkmap, (caddr_t)&a) != 0)

panic("segmap_create segkmap");

rw_exit(&kas.a_lock);

segdev_init();

}

static void

startup_end(void)

{

if ((caddr_t)memlist > (caddr_t)memlist_end)

panic("memlist overflow 2");

memlist_free_block((caddr_t)memlist,

((caddr_t)memlist_end - (caddr_t)memlist));

memlist = NULL;

/* enable page_relocation since OBP is now done */

page_relocate_ready = 1;

/*

kern_setup1();

/*

contig_mem_init();

mach_descrip_init();

cpu_intrq_setup(CPU);

cpu_intrq_register(CPU);

mach_htraptrace_setup(CPU->cpu_id);

mach_htraptrace_configure(CPU->cpu_id);

mach_dump_buffer_init();

/*

cpu_intr_alloc(CPU, NINTR_THREADS);

(void) splzs(); /* allow hi clock ints but not zs */

/*

error_init();

/*

mach_hw_copy_limit();

/*

(void) prom_set_preprom(kern_preprom);

(void) prom_set_postprom(kern_postprom);

CPU->cpu_m.mutex_ready = 1;

/*

segnf_init();

/*

configure(); /* set up devices */

mach_cpu_halt_idle();

}

post_startup(void)

{

#ifdef PTL1_PANIC_DEBUG

extern void init_ptl1_thread(void);

#endif /* PTL1_PANIC_DEBUG */

extern void abort_sequence_init(void);

/*

bind_hwcap();

/*

mach_memscrub();

/*

abort_sequence_init();

/*

(void) mod_sysctl(SYS_FORCELOAD, NULL);

/*

(void) modload("fs", "procfs");

/* load machine class specific drivers */

load_mach_drivers();

/* load platform specific drivers */

if (&load_platform_drivers)

load_platform_drivers();

/* load vis simulation module, if we are running w/fpu off */

if (!fpu_exists) {

if (modload("misc", "vis") == -1)

halt("Can't load vis");

}

mach_fpras();

maxmem = freemem;

#ifdef PTL1_PANIC_DEBUG

init_ptl1_thread();

#endif /* PTL1_PANIC_DEBUG */

if (&cif_init)

cif_init();

}

#ifdef PTL1_PANIC_DEBUG

int ptl1_panic_test = 0;

int ptl1_panic_xc_one_test = 0;

int ptl1_panic_xc_all_test = 0;

int ptl1_panic_xt_one_test = 0;

int ptl1_panic_xt_all_test = 0;

kthread_id_t ptl1_thread_p = NULL;

kcondvar_t ptl1_cv;

kmutex_t ptl1_mutex;

int ptl1_recurse_count_threshold = 0x40;

int ptl1_recurse_trap_threshold = 0x3d;

extern void ptl1_recurse(int, int);

extern void ptl1_panic_xt(int, int);

static void

ptl1_wakeup(void *arg)

{

mutex_enter(&ptl1_mutex);

cv_signal(&ptl1_cv);

mutex_exit(&ptl1_mutex);

}

static void

ptl1_panic_xc(void)

{

ptl1_recurse(ptl1_recurse_count_threshold,

ptl1_recurse_trap_threshold);

}

static void

ptl1_thread(void)

{

mutex_enter(&ptl1_mutex);

while (ptl1_thread_p) {

cpuset_t other_cpus;

int cpu_id;

int my_cpu_id;

int target_cpu_id;

int target_found;

if (ptl1_panic_test) {

ptl1_recurse(ptl1_recurse_count_threshold,

ptl1_recurse_trap_threshold);

}

/*

kpreempt_disable();

my_cpu_id = CPU->cpu_id;

other_cpus = cpu_ready_set;

CPUSET_DEL(other_cpus, CPU->cpu_id);

target_found = 0;

if (!CPUSET_ISNULL(other_cpus)) {

/*

for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {

if (cpu_id == my_cpu_id)

continue;

if (CPU_XCALL_READY(cpu_id)) {

target_cpu_id = cpu_id;

target_found = 1;

break;

}

}

ASSERT(target_found);

if (ptl1_panic_xc_one_test) {

xc_one(target_cpu_id,

(xcfunc_t *)ptl1_panic_xc, 0, 0);

}

if (ptl1_panic_xc_all_test) {

xc_some(other_cpus,

(xcfunc_t *)ptl1_panic_xc, 0, 0);

}

if (ptl1_panic_xt_one_test) {

xt_one(target_cpu_id,

(xcfunc_t *)ptl1_panic_xt, 0, 0);

}

if (ptl1_panic_xt_all_test) {

xt_some(other_cpus,

(xcfunc_t *)ptl1_panic_xt, 0, 0);

}

}

kpreempt_enable();

(void) timeout(ptl1_wakeup, NULL, hz);

(void) cv_wait(&ptl1_cv, &ptl1_mutex);

}

mutex_exit(&ptl1_mutex);

}

init_ptl1_thread(void)

{

ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,

&p0, TS_RUN, 0);

}

#endif /* PTL1_PANIC_DEBUG */

static void

memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,

struct memlist **curmemlistp)

{

struct memlist *new;

new = *memlistp;

new->address = start;

new->size = len;

*memlistp = new + 1;

memlist_insert(new, curmemlistp);

}

static void

memseg_list_add(struct memseg *memsegp)

{

struct memseg **prev_memsegp;

pgcnt_t num;

/* insert in memseg list, decreasing number of pages order */

num = MSEG_NPAGES(memsegp);

for (prev_memsegp = &memsegs; *prev_memsegp;

prev_memsegp = &((*prev_memsegp)->next)) {

if (num > MSEG_NPAGES(*prev_memsegp))

break;

}

memsegp->next = *prev_memsegp;

*prev_memsegp = memsegp;

if (kpm_enable) {

memsegp->nextpa = (memsegp->next) ?

va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;

if (prev_memsegp != &memsegs) {

struct memseg *msp;

msp = (struct memseg *)((caddr_t)prev_memsegp -

offsetof(struct memseg, next));

msp->nextpa = va_to_pa(memsegp);

} else {

memsegspa = va_to_pa(memsegs);

}

}

}

#define PREFETCH_BYTES 64

add_physmem_cb(page_t *pp, pfn_t pnum)

{

extern void prefetch_page_w(void *);

pp->p_pagenum = pnum;

/*

/*LINTED*/

ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);

prefetch_page_w((char *)pp);

}

static void

kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages,

uintptr_t kpm_pp, pgcnt_t kpm_npages)

{

struct memlist *pmem;

struct memseg *msp;

pfn_t base;

pgcnt_t num;

pfn_t lastseg_pages_end = 0;

pgcnt_t nelem_used = 0;

ASSERT(page_hash != NULL && page_hashsz != 0);

msp = memsegp;

for (pmem = phys_avail; pmem && npages; pmem = pmem->next) {

/*

num = btop(pmem->size);

if (num > npages)

num = npages;

npages -= num;

base = btop(pmem->address);

msp->pages = pp;

msp->epages = pp + num;

msp->pages_base = base;

msp->pages_end = base + num;

if (kpm_enable) {

pfn_t pbase_a;

pfn_t pend_a;

pfn_t prev_pend_a;

pgcnt_t nelem;

msp->pagespa = va_to_pa(pp);

msp->epagespa = va_to_pa(pp + num);

pbase_a = kpmptop(ptokpmp(base));

pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs;

nelem = ptokpmp(pend_a - pbase_a);

msp->kpm_nkpmpgs = nelem;

msp->kpm_pbase = pbase_a;

if (lastseg_pages_end) {

/*

ASSERT(base + num > lastseg_pages_end);

prev_pend_a = kpmptop(

ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs;

if (prev_pend_a > pbase_a) {

/*

if (kpm_smallpages == 0) {

msp->kpm_pages =

(kpm_page_t *)kpm_pp - 1;

kpm_pp = (uintptr_t)

((kpm_page_t *)kpm_pp

+ nelem - 1);

} else {

msp->kpm_spages =

(kpm_spage_t *)kpm_pp - 1;

kpm_pp = (uintptr_t)

((kpm_spage_t *)kpm_pp

+ nelem - 1);

}

nelem_used += nelem - 1;

} else {

if (kpm_smallpages == 0) {

msp->kpm_pages =

(kpm_page_t *)kpm_pp;

kpm_pp = (uintptr_t)

((kpm_page_t *)kpm_pp

+ nelem);

} else {

msp->kpm_spages =

(kpm_spage_t *)kpm_pp;

kpm_pp = (uintptr_t)

((kpm_spage_t *)

kpm_pp + nelem);

}

nelem_used += nelem;

}

} else {

if (kpm_smallpages == 0) {

msp->kpm_pages = (kpm_page_t *)kpm_pp;

kpm_pp = (uintptr_t)

((kpm_page_t *)kpm_pp + nelem);

} else {

msp->kpm_spages = (kpm_spage_t *)kpm_pp;

kpm_pp = (uintptr_t)

((kpm_spage_t *)kpm_pp + nelem);

}

nelem_used = nelem;

}

if (nelem_used > kpm_npages)

panic("kphysm_init: kpm_pp overflow\n");

msp->kpm_pagespa = va_to_pa(msp->kpm_pages);

lastseg_pages_end = msp->pages_end;

}

memseg_list_add(msp);

/*

add_physmem(pp, num, base);

pp += num;

msp++;

}

build_pfn_hash();

}

static void

kvm_init(void)

{

/*

rw_enter(&kas.a_lock, RW_WRITER);

as_avlinit(&kas);

(void) seg_attach(&kas, (caddr_t)KERNELBASE,

(size_t)(e_moddata - KERNELBASE), &ktextseg);

(void) segkmem_create(&ktextseg);

(void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),

(size_t)(MMU_PAGESIZE4M), &ktexthole);

(void) segkmem_create(&ktexthole);

(void) seg_attach(&kas, (caddr_t)valloc_base,

(size_t)(econtig32 - valloc_base), &kvalloc);

(void) segkmem_create(&kvalloc);

if (kmem64_base) {

(void) seg_attach(&kas, (caddr_t)kmem64_base,

(size_t)(kmem64_end - kmem64_base), &kmem64);

(void) segkmem_create(&kmem64);

}

/*

(void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);

(void) segkmem_create(&kvseg);

hblk_alloc_dynamic = 1;

/*

memseg_remap_init();

/* at this point we are ready to use large page heap */

segkmem_heap_lp_init();

(void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,

&kvseg32);

(void) segkmem_create(&kvseg32);

/*

(void) seg_attach(&kas, (caddr_t)SEGDEBUGBASE, (size_t)SEGDEBUGSIZE,

&kdebugseg);

(void) segkmem_create(&kdebugseg);

rw_exit(&kas.a_lock);

}

char obp_tte_str[] =

"h# %x constant MMU_PAGESHIFT "

"h# %x constant TTE8K "

"h# %x constant SFHME_SIZE "

"h# %x constant SFHME_TTE "

"h# %x constant HMEBLK_TAG "

"h# %x constant HMEBLK_NEXT "

"h# %x constant HMEBLK_MISC "

"h# %x constant HMEBLK_HME1 "

"h# %x constant NHMENTS "

"h# %x constant HBLK_SZMASK "

"h# %x constant HBLK_RANGE_SHIFT "

"h# %x constant HMEBP_HBLK "

"h# %x constant HMEBUCKET_SIZE "

"h# %x constant HTAG_SFMMUPSZ "

"h# %x constant HTAG_REHASHSZ "

"h# %x constant mmu_hashcnt "

"h# %p constant uhme_hash "

"h# %p constant khme_hash "

"h# %x constant UHMEHASH_SZ "

"h# %x constant KHMEHASH_SZ "

"h# %p constant KCONTEXT "

"h# %p constant KHATID "

"h# %x constant ASI_MEM "

": PHYS-X@ ( phys -- data ) "

" ASI_MEM spacex@ "

"; "

": PHYS-W@ ( phys -- data ) "

" ASI_MEM spacew@ "

"; "

": PHYS-L@ ( phys -- data ) "

" ASI_MEM spaceL@ "

"; "

": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "

" 3 * MMU_PAGESHIFT + "

"; "

": TTE_IS_VALID ( ttep -- flag ) "

" PHYS-X@ 0< "

"; "

": HME_HASH_SHIFT ( ttesz -- hmeshift ) "

" dup TTE8K = if "

" drop HBLK_RANGE_SHIFT "

" else "

" TTE_PAGE_SHIFT "

" then "

"; "

": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "

" tuck >> swap MMU_PAGESHIFT - << "

"; "

": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "

" >> over xor swap ( hash sfmmup ) "

" KHATID <> if ( hash ) "

" UHMEHASH_SZ and ( bucket ) "

" HMEBUCKET_SIZE * uhme_hash + ( hmebp ) "

" else ( hash ) "

" KHMEHASH_SZ and ( bucket ) "

" HMEBUCKET_SIZE * khme_hash + ( hmebp ) "

" then ( hmebp ) "

"; "

": HME_HASH_TABLE_SEARCH "

" ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) "

" >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "

" dup if ( sfmmup hmeblkp ) ( r: hblktag ) "

" dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) "

" dup hmeblk_tag + 8 + phys-x@ 2 pick = if "

" true ( sfmmup hmeblkp true ) ( r: hblktag ) "

" else "

" hmeblk_next + phys-x@ false "

" ( sfmmup hmeblkp false ) ( r: hblktag ) "

" then "

" else "

" hmeblk_next + phys-x@ false "

" ( sfmmup hmeblkp false ) ( r: hblktag ) "

" then "

" else "

" true "

" then "

" until r> drop "

"; "

": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "

" over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) "

" HTAG_REHASHSZ << or nip ( hblktag ) "

"; "

": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "

" over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) "

" TTE8K = if ( hmeblkp addr ) "

" MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) "

" else ( hmeblkp addr ) "

" drop 0 ( hmeblkp 0 ) "

" then ( hmeblkp hme-index ) "

" SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) "

" SFHME_TTE + ( ttep ) "

"; "

": unix-tte ( addr cnum -- false | tte-data true ) "

" KCONTEXT = if ( addr ) "

" KHATID ( addr khatid ) "

" else ( addr ) "

" drop false exit ( false ) "

" then "

" ( addr khatid ) "

" mmu_hashcnt 1+ 1 do ( addr sfmmup ) "

" 2dup swap i HME_HASH_SHIFT "

"( addr sfmmup sfmmup addr hmeshift ) "

" HME_HASH_FUNCTION ( addr sfmmup hmebp ) "

" over i 4 pick "

"( addr sfmmup hmebp sfmmup rehash addr ) "

" HME_HASH_TAG ( addr sfmmup hmebp hblktag ) "

" HME_HASH_TABLE_SEARCH "

"( addr sfmmup { null | hmeblkp } ) "

" ?dup if ( addr sfmmup hmeblkp ) "

" nip swap HBLK_TO_TTEP ( ttep ) "

" dup TTE_IS_VALID if ( valid-ttep ) "

" PHYS-X@ true ( tte-data true ) "

" else ( invalid-tte ) "

" drop false ( false ) "

" then ( false | tte-data true ) "

" unloop exit ( false | tte-data true ) "

" then ( addr sfmmup ) "

" loop ( addr sfmmup ) "

" 2drop false ( false ) "

"; "

;

create_va_to_tte(void)

{

char *bp;

extern int khmehash_num, uhmehash_num;

extern struct hmehash_bucket *khme_hash, *uhme_hash;

#define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field))

bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);

/*

(void) sprintf(bp, obp_tte_str,

MMU_PAGESHIFT,

TTE8K,

sizeof (struct sf_hment),

OFFSET(struct sf_hment, hme_tte),

OFFSET(struct hme_blk, hblk_tag),

OFFSET(struct hme_blk, hblk_nextpa),

OFFSET(struct hme_blk, hblk_misc),

OFFSET(struct hme_blk, hblk_hme),

NHMENTS,

HBLK_SZMASK,

HBLK_RANGE_SHIFT,

OFFSET(struct hmehash_bucket, hmeh_nextpa),

sizeof (struct hmehash_bucket),

HTAG_SFMMUPSZ,

HTAG_REHASHSZ,

mmu_hashcnt,

(caddr_t)va_to_pa((caddr_t)uhme_hash),

(caddr_t)va_to_pa((caddr_t)khme_hash),

UHMEHASH_SZ,

KHMEHASH_SZ,

KCONTEXT,

KHATID,

ASI_MEM);

prom_interpret(bp, 0, 0, 0, 0, 0);

kobj_free(bp, MMU_PAGESIZE);

}

install_va_to_tte(void)

{

/*

prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);

}

static char *create_node =

"\" /\" find-device "

"new-device "

"\" os-io\" device-name "

"\" display\" device-type "

": cb-r/w ( adr,len method$ -- #read/#written ) "

" 2>r swap 2 2r> ['] $callback catch if "

" 2drop 3drop 0 "

" then "

"; "

": read ( adr,len -- #read ) "

" \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then "

" ( retN ... ret1 N ) "

" ?dup if "

" swap >r 1- 0 ?do drop loop r> "

" else "

" -2 "

" then "

"; "

": write ( adr,len -- #written ) "

" \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then "

" ( retN ... ret1 N ) "

" ?dup if "

" swap >r 1- 0 ?do drop loop r> "

" else "

" 0 "

" then "

"; "

": poll-tty ( -- ) ; "

": install-abort ( -- ) ['] poll-tty d# 10 alarm ; "

": remove-abort ( -- ) ['] poll-tty 0 alarm ; "

": cb-give/take ( $method -- ) "

" 0 -rot ['] $callback catch ?dup if "

" >r 2drop 2drop r> throw "

" else "

" 0 ?do drop loop "

" then "

"; "

": give ( -- ) \" exit-input\" cb-give/take ; "

": take ( -- ) \" enter-input\" cb-give/take ; "

": open ( -- ok? ) true ; "

": close ( -- ) ; "

"finish-device "

"device-end ";

static void

startup_create_io_node(void)

{

prom_interpret(create_node, 0, 0, 0, 0, 0);

}

static void

do_prom_version_check(void)

{

int i;

pnode_t node;

char buf[64];

static char drev[] = "Down-rev firmware detected%s\n"

"\tPlease upgrade to the following minimum version:\n"

"\t\t%s\n";

i = prom_version_check(buf, sizeof (buf), &node);

if (i == PROM_VER64_OK)

return;

if (i == PROM_VER64_UPGRADE) {

cmn_err(CE_WARN, drev, "", buf);

#ifdef DEBUG

prom_enter_mon(); /* Type 'go' to continue */

cmn_err(CE_WARN, "Booting with down-rev firmware\n");

return;

#else

halt(0);

#endif

}

/*

cmn_err(CE_WARN, drev, " on one or more CPU boards", buf);

}

static void

kpm_init()

{

kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;

kpm_pgsz = 1ull << kpm_pgshft;

kpm_pgoff = kpm_pgsz - 1;

kpmp2pshft = kpm_pgshft - PAGESHIFT;

kpmpnpgs = 1 << kpmp2pshft;

ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0);

}

kpm_npages_setup(int memblocks)

{

/*

kpm_npages = ptokpmpr(npages) + memblocks;

}

extern caddr_t modtext;

extern size_t modtext_sz;

extern caddr_t moddata;

#define HEAPTEXT_ARENA(addr) \

((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \

(((uintptr_t)(addr) - HEAPTEXT_BASE) / \

(HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))

#define HEAPTEXT_OVERSIZED(addr) \

((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)

vmem_t *texthole_source[HEAPTEXT_NARENAS];

vmem_t *texthole_arena[HEAPTEXT_NARENAS];

kmutex_t texthole_lock;

char kern_bootargs[OBP_MAXPATHLEN];

kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)

{

uintptr_t addr, limit;

addr = HEAPTEXT_BASE;

limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;

/*

for (; addr + HEAPTEXT_UNMAPPED <= limit;

addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {

(void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,

0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),

VM_NOSLEEP | VM_BESTFIT | VM_PANIC);

}

/*

(void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,

(void *)limit, (void *)(limit + PAGESIZE),

VM_NOSLEEP | VM_BESTFIT | VM_PANIC);

*text_arena = vmem_create("module_text", modtext, modtext_sz,

sizeof (uintptr_t), segkmem_alloc, segkmem_free,

heaptext_arena, 0, VM_SLEEP);

*data_arena = vmem_create("module_data", moddata, MODDATA, 1,

segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);

}

kobj_text_alloc(vmem_t *arena, size_t size)

{

caddr_t rval, better;

/*

rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);