#pragma ident "%Z%%M% %I% %E% SMI"

#include <sys/machsystm.h>

#include <sys/archsystm.h>

#include <sys/vm.h>

#include <sys/cpu.h>

#include <sys/atomic.h>

#include <sys/reboot.h>

#include <sys/kdi.h>

#include <sys/bootconf.h>

#include <sys/memlist_plat.h>

#include <sys/memlist_impl.h>

#include <sys/prom_plat.h>

#include <sys/prom_isa.h>

#include <sys/autoconf.h>

#include <sys/intreg.h>

#include <sys/ivintr.h>

#include <sys/fpu/fpusystm.h>

#include <sys/iommutsb.h>

#include <vm/vm_dep.h>

#include <vm/seg_dev.h>

#include <vm/seg_kmem.h>

#include <vm/seg_kpm.h>

#include <vm/seg_map.h>

#include <vm/seg_kp.h>

#include <sys/sysconf.h>

#include <vm/hat_sfmmu.h>

#include <sys/kobj.h>

#include <sys/sun4asi.h>

#include <sys/clconf.h>

#include <sys/platform_module.h>

#include <sys/panic.h>

#include <sys/cpu_sgnblk_defs.h>

#include <sys/clock.h>

#include <sys/cmn_err.h>

#include <sys/promif.h>

#include <sys/prom_debug.h>

#include <sys/traptrace.h>

#include <sys/memnode.h>

#include <sys/mem_cage.h>

#include <sys/mmu.h>

extern void setup_trap_table(void);

extern int cpu_intrq_setup(struct cpu *);

extern void cpu_intrq_register(struct cpu *);

extern void contig_mem_init(void);

extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);

extern void mach_dump_buffer_init(void);

extern void mach_descrip_init(void);

extern void mach_descrip_startup_fini(void);

extern void mach_memscrub(void);

extern void mach_fpras(void);

extern void mach_cpu_halt_idle(void);

extern void mach_hw_copy_limit(void);

extern void load_mach_drivers(void);

extern void load_tod_module(void);

#pragma weak load_tod_module

extern int ndata_alloc_mmfsa(struct memlist *ndata);

#pragma weak ndata_alloc_mmfsa

extern void cif_init(void);

#pragma weak cif_init

extern void parse_idprom(void);

extern void add_vx_handler(char *, int, void (*)(cell_t *));

extern void mem_config_init(void);

extern void memseg_remap_init(void);

extern void mach_kpm_init(void);

extern int size_pse_array(pgcnt_t, int);

extern int vac_size; /* cache size in bytes */

extern uint_t vac_mask; /* VAC alignment consistency mask */

extern uint_t vac_colors;

void *romp; /* veritas driver won't load without romp 4154976 */

pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */

pgcnt_t segkpsize =

btop(SEGKPDEFSIZE); /* size of segkp segment in pages */

uint_t segmap_percent = 12; /* Size of segmap segment */

int use_cache = 1; /* cache not reliable (605 bugs) with MP */

int vac_copyback = 1;

char *cache_mode = NULL;

int use_mix = 1;

int prom_debug = 0;

caddr_t boot_tba; /* %tba at boot - used by kmdb */

uint_t tba_taken_over = 0;

caddr_t s_text; /* start of kernel text segment */

caddr_t e_text; /* end of kernel text segment */

caddr_t s_data; /* start of kernel data segment */

caddr_t e_data; /* end of kernel data segment */

caddr_t modtext; /* beginning of module text */

size_t modtext_sz; /* size of module text */

caddr_t moddata; /* beginning of module data reserve */

caddr_t e_moddata; /* end of module data reserve */

caddr_t econtig32; /* end of first blk of contiguous kernel */

caddr_t ncbase; /* beginning of non-cached segment */

caddr_t ncend; /* end of non-cached segment */

size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */

caddr_t nalloc_base; /* beginning of nucleus allocation */

caddr_t nalloc_end; /* end of nucleus allocatable memory */

caddr_t valloc_base; /* beginning of kvalloc segment */

caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */

caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */

size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */

caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */

int kmem64_szc; /* page size code */

uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */

uintptr_t shm_alignment; /* VAC address consistency modulus */

struct memlist *phys_install; /* Total installed physical memory */

struct memlist *phys_avail; /* Available (unreserved) physical memory */

struct memlist *virt_avail; /* Available (unmapped?) virtual memory */

struct memlist *nopp_list; /* pages with no backing page structs */

struct memlist ndata; /* memlist of nucleus allocatable memory */

int memexp_flag; /* memory expansion card flag */

uint64_t ecache_flushaddr; /* physical address used for flushing E$ */

pgcnt_t obp_pages; /* Physical pages used by OBP */

long page_hashsz; /* Size of page hash table (power of two) */

struct page *pp_base; /* Base of system page struct array */

size_t pp_sz; /* Size in bytes of page struct array */

struct page **page_hash; /* Page hash table */

pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */

size_t pse_table_size; /* Number of mutexes in pse_mutex[] */

int pse_shift; /* log2(pse_table_size) */

struct seg ktextseg; /* Segment used for kernel executable image */

struct seg kvalloc; /* Segment used for "valloc" mapping */

struct seg kpseg; /* Segment used for pageable kernel virt mem */

struct seg ktexthole; /* Segment used for nucleus text hole */

struct seg kmapseg; /* Segment used for generic kernel mappings */

struct seg kpmseg; /* Segment used for physical mapping */

struct seg kdebugseg; /* Segment used for the kernel debugger */

void *kpm_pp_base; /* Base of system kpm_page array */

size_t kpm_pp_sz; /* Size of system kpm_page array */

pgcnt_t kpm_npages; /* How many kpm pages are managed */

struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */

struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */

struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */

int segzio_fromheap = 0; /* zio allocations occur from heap */

caddr_t segzio_base; /* Base address of segzio */

pgcnt_t segziosize = 0; /* size of zio segment in pages */

struct vnode kdebugvp;

const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;

const size_t kdi_segdebugsize = SEGDEBUGSIZE;

struct seg kmem64;

struct memseg *memseg_free;

struct vnode unused_pages_vp;

size_t pagehash_sz;

uint64_t memlist_sz;

char tbr_wr_addr_inited = 0;

caddr_t mpo_heap32_buf = NULL;

size_t mpo_heap32_bufsz = 0;

static int ndata_alloc_memseg(struct memlist *, size_t);

static void memlist_new(uint64_t, uint64_t, struct memlist **);

static void memlist_add(uint64_t, uint64_t,

struct memlist **, struct memlist **);

static void kphysm_init(void);

static void kvm_init(void);

static void install_kmem64_tte(void);

static void startup_init(void);

static void startup_memlist(void);

static void startup_modules(void);

static void startup_bop_gone(void);

static void startup_vm(void);

static void startup_end(void);

static void setup_cage_params(void);

static void startup_create_io_node(void);

static pgcnt_t npages;

static struct memlist *memlist;

void *memlist_end;

static pgcnt_t bop_alloc_pages;

static caddr_t hblk_base;

uint_t hblk_alloc_dynamic = 0;

uint_t hblk1_min = H1MIN;

int iam_positron(void);

#pragma weak iam_positron

static void do_prom_version_check(void);

int thermal_powerdown_delay = 1200;

int page_relocate_ready = 0;

#ifdef DEBUGGING_MEM

static int debugging_mem;

static void

printmemlist(char *title, struct memlist *list)

{

if (!debugging_mem)

return;

printf("%s\n", title);

while (list) {

prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",

(uint32_t)(list->address >> 32), (uint32_t)list->address,

(uint32_t)(list->size >> 32), (uint32_t)(list->size));

list = list->next;

}

}

printmemseg(struct memseg *memseg)

{

if (!debugging_mem)

return;

printf("memseg\n");

while (memseg) {

prom_printf("\tpage = 0x%p, epage = 0x%p, "

"pfn = 0x%x, epfn = 0x%x\n",

memseg->pages, memseg->epages,

memseg->pages_base, memseg->pages_end);

memseg = memseg->next;

}

}

#define debug_pause(str) halt((str))

#define MPRINTF(str) if (debugging_mem) prom_printf((str))

#define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a))

#define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b))

#define MPRINTF3(str, a, b, c) \

if (debugging_mem) prom_printf((str), (a), (b), (c))

#else /* DEBUGGING_MEM */

#define MPRINTF(str)

#define MPRINTF1(str, a)

#define MPRINTF2(str, a, b)

#define MPRINTF3(str, a, b, c)

#endif /* DEBUGGING_MEM */

extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);

extern uint64_t ecache_flush_address(void);

#pragma weak load_platform_modules

#pragma weak plat_startup_memlist

#pragma weak ecache_init_scrub_flush_area

#pragma weak ecache_flush_address

int kernel_cage_enable = 1;

static void

setup_cage_params(void)

{

void (*func)(void);

func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);

if (func != NULL) {

(*func)();

return;

}

if (kernel_cage_enable == 0) {

return;

}

kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);

if (kcage_on) {

cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");

} else {

cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");

}

}

startup(void)

{

startup_init();

if (&startup_platform)

startup_platform();

startup_memlist();

startup_modules();

setup_cage_params();

startup_bop_gone();

startup_vm();

startup_end();

}

struct regs sync_reg_buf;

uint64_t sync_tt;

sync_handler(void)

{

struct panic_trap_info ti;

int i;

/*

for (i = 0; i < NCPU; i++) {

if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {

cpu[i]->cpu_flags &= ~CPU_READY;

cpu[i]->cpu_flags |= CPU_QUIESCED;

CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);

}

}

/*

if (!panicstr && (curthread->t_panic_trap == NULL)) {

ti.trap_type = sync_tt;

ti.trap_regs = &sync_reg_buf;

ti.trap_addr = NULL;

ti.trap_mmu_fsr = 0x0;

curthread->t_panic_trap = &ti;

}

/*

if (panicstr)

CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);

nopanicdebug = 1; /* do not perform debug_enter() prior to dump */

panic("sync initiated");

}

static void

startup_init(void)

{

/*

char sync_str[] =

"warning @ warning off : sync "

"%%tl-c %%tstate h# %p x! "

"%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "

"%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "

"%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "

"%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "

"%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "

"%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "

"%%y h# %p l! %%tl-c %%tt h# %p x! "

"sync ; warning !";

/*

char bp[sizeof (sync_str) + 16 * 20];

/*

ptl1_init_cpu(&cpu0);

/*

ppmapinit();

/*

init_vx_handler();

/*

(void) sprintf((char *)bp, sync_str,

(void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,

(void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,

(void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,

(void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,

(void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,

(void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,

(void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,

(void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,

(void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,

(void *)&sync_reg_buf.r_y, (void *)&sync_tt);

prom_interpret(bp, 0, 0, 0, 0, 0);

add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);

}

calc_pp_sz(pgcnt_t npages)

{

return (npages * sizeof (struct page));

}

calc_kpmpp_sz(pgcnt_t npages)

{

kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;

kpm_pgsz = 1ull << kpm_pgshft;

kpm_pgoff = kpm_pgsz - 1;

kpmp2pshft = kpm_pgshft - PAGESHIFT;

kpmpnpgs = 1 << kpmp2pshft;

if (kpm_smallpages == 0) {

/*

kpm_npages = ptokpmpr(physinstalled);

return (kpm_npages * sizeof (kpm_page_t));

} else {

kpm_npages = npages;

return (kpm_npages * sizeof (kpm_spage_t));

}

}

calc_pagehash_sz(pgcnt_t npages)

{

/*

page_hashsz = npages / PAGE_HASHAVELEN;

page_hashsz = 1 << highbit(page_hashsz);

return (page_hashsz * sizeof (struct page *));

}

alloc_kmem64(caddr_t base, caddr_t end)

{

int i;

caddr_t aligned_end = NULL;

/*

ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));

ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));

for (i = max_bootlp_tteszc; i >= TTE8K; i--) {

size_t alloc_size, alignsize;

#if !defined(C_OBP)

unsigned long long pa;

#endif /* !C_OBP */

if ((mmu_exported_pagesize_mask & (1 << i)) == 0)

continue;

alignsize = TTEBYTES(i);

kmem64_szc = i;

/* limit page size for small memory */

if (mmu_btop(alignsize) > (npages >> 2))

continue;

aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);

alloc_size = aligned_end - base;

#if !defined(C_OBP)

if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {

if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {

kmem64_pabase = pa;

kmem64_aligned_end = aligned_end;

install_kmem64_tte();

break;

} else {

prom_free_phys(alloc_size, pa);

}

}

#else /* !C_OBP */

if (prom_alloc(base, alloc_size, alignsize) == base) {

kmem64_pabase = va_to_pa(kmem64_base);

kmem64_aligned_end = aligned_end;

break;

}

#endif /* !C_OBP */

if (i == TTE8K) {

prom_panic("kmem64 allocation failure");

}

}

ASSERT(aligned_end != NULL);

}

static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;

static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;

#define IVSIZE roundup(((MAXIVNUM * sizeof (intr_vec_t *)) + \

(MAX_RSVD_IV * sizeof (intr_vec_t)) + \

(MAX_RSVD_IVX * sizeof (intr_vecx_t))), PAGESIZE)

#if !defined(C_OBP)

char kmem64_obp_str[] =

"h# %lx constant kmem64-base "

"h# %lx constant kmem64-end "

"h# %lx constant kmem64-pagemask "

"h# %lx constant kmem64-template "

": kmem64-tte ( addr cnum -- false | tte-data true ) "

" if ( addr ) "

" drop false exit then ( false ) "

" dup kmem64-base kmem64-end within if ( addr ) "

" kmem64-pagemask and ( addr' ) "

" kmem64-base - ( addr' ) "

" kmem64-template + ( tte ) "

" true ( tte true ) "

" else ( addr ) "

" pgmap@ ( tte ) "

" dup 0< if true else drop false then ( tte true | false ) "

" then ( tte true | false ) "

"; "

"' kmem64-tte is va>tte-data "

;

static void

install_kmem64_tte()

{

char b[sizeof (kmem64_obp_str) + (4 * 16)];

tte_t tte;

PRM_DEBUG(kmem64_pabase);

PRM_DEBUG(kmem64_szc);

sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,

PROC_DATA | HAT_NOSYNC, kmem64_szc);

PRM_DEBUG(tte.ll);

(void) sprintf(b, kmem64_obp_str,

kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);

ASSERT(strlen(b) < sizeof (b));

prom_interpret(b, 0, 0, 0, 0, 0);

}

#endif /* !C_OBP */

#define MINMOVE_RAM_MB ((size_t)1900)

#define MB_TO_BYTES(mb) ((mb) * 1048576ul)

#define BYTES_TO_MB(b) ((b) / 1048576ul)

pgcnt_t tune_npages = (pgcnt_t)

(MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);

#pragma weak page_set_colorequiv_arr_cpu

extern void page_set_colorequiv_arr_cpu(void);

extern void page_set_colorequiv_arr(void);

static pgcnt_t ramdisk_npages;

static struct memlist *old_phys_avail;

kcage_dir_t kcage_startup_dir = KCAGE_DOWN;

static void

startup_memlist(void)

{

size_t hmehash_sz, pagelist_sz, tt_sz;

size_t psetable_sz;

caddr_t alloc_base;

caddr_t memspace;

struct memlist *cur;

size_t syslimit = (size_t)SYSLIMIT;

size_t sysbase = (size_t)SYSBASE;

/*

/*

moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);

e_moddata = moddata + MODDATA;

nalloc_base = e_moddata;

nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);

valloc_base = nalloc_base;

/*

if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)

prom_panic("nucleus data overflow");

PRM_DEBUG(moddata);

PRM_DEBUG(nalloc_base);

PRM_DEBUG(nalloc_end);

/*

PRM_DEBUG(e_text);

modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);

if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)

prom_panic("nucleus text overflow");

modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -

modtext;

PRM_DEBUG(modtext);

PRM_DEBUG(modtext_sz);

init_boot_memlists();

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

/*

installed_top_size_memlist_array(boot_physinstalled,

boot_physinstalled_len, &physmax, &physinstalled);

PRM_DEBUG(physinstalled);

PRM_DEBUG(physmax);

/* Fill out memory nodes config structure */

startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);

/*

npages = physinstalled - btop(MMU_PAGESIZE4M);

/*

ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);

if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))

cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");

if (ndata_alloc_tsbs(&ndata, npages) != 0)

cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");

if (ndata_alloc_dmv(&ndata) != 0)

cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");

if (ndata_alloc_page_mutexs(&ndata) != 0)

cmn_err(CE_PANIC,

"no more nucleus memory after page free lists alloc");

if (ndata_alloc_hat(&ndata, npages) != 0)

cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");

if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)

cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");

/*

alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);

PRM_DEBUG(alloc_base);

alloc_base = sfmmu_ktsb_alloc(alloc_base);

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);

PRM_DEBUG(alloc_base);

/*

alloc_base = iommu_tsb_init(alloc_base);

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);

PRM_DEBUG(alloc_base);

/*

alloc_base = contig_mem_prealloc(alloc_base, npages);

/*

if (&plat_startup_memlist) {

alloc_base = plat_startup_memlist(alloc_base);

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,

ecache_alignsize);

PRM_DEBUG(alloc_base);

}

/*

econtig32 = alloc_base;

PRM_DEBUG(econtig32);

if (econtig32 > (caddr_t)KERNEL_LIMIT32)

cmn_err(CE_PANIC, "econtig32 too big");

pp_sz = calc_pp_sz(npages);

PRM_DEBUG(pp_sz);

if (kpm_enable) {

kpm_pp_sz = calc_kpmpp_sz(npages);

PRM_DEBUG(kpm_pp_sz);

}

hmehash_sz = calc_hmehash_sz(npages);

PRM_DEBUG(hmehash_sz);

pagehash_sz = calc_pagehash_sz(npages);

PRM_DEBUG(pagehash_sz);

pagelist_sz = calc_free_pagelist_sz();

PRM_DEBUG(pagelist_sz);

#ifdef TRAPTRACE

tt_sz = calc_traptrace_sz();

PRM_DEBUG(tt_sz);

#else

tt_sz = 0;

#endif /* TRAPTRACE */

/*

pse_shift = size_pse_array(npages, max_ncpus);

PRM_DEBUG(pse_shift);

pse_table_size = 1 << pse_shift;

PRM_DEBUG(pse_table_size);

psetable_sz = roundup(

pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);

PRM_DEBUG(psetable_sz);

/*

kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +

pagelist_sz + tt_sz + psetable_sz;

kmem64_sz = roundup(kmem64_sz, PAGESIZE);

kmem64_base = (caddr_t)syslimit;

kmem64_end = kmem64_base + kmem64_sz;

alloc_kmem64(kmem64_base, kmem64_end);

if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))

cmn_err(CE_PANIC, "not enough kmem64 space");

PRM_DEBUG(kmem64_base);

PRM_DEBUG(kmem64_end);

PRM_DEBUG(kmem64_aligned_end);

/*

alloc_base = kmem64_base;

if (kpm_smallpages == 0) {

npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));

} else {

npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +

sizeof (kpm_spage_t));

}

pp_base = (page_t *)alloc_base;

pp_sz = npages * sizeof (struct page);

alloc_base += pp_sz;

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);

PRM_DEBUG(pp_base);

PRM_DEBUG(npages);

if (kpm_enable) {

kpm_pp_base = alloc_base;

if (kpm_smallpages == 0) {

/* kpm_npages based on physinstalled, don't reset */

kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);

} else {

kpm_npages = ptokpmpr(npages);

kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);

}

alloc_base += kpm_pp_sz;

alloc_base =

(caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);

PRM_DEBUG(kpm_pp_base);

}

alloc_base = alloc_hmehash(alloc_base);

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);

PRM_DEBUG(alloc_base);

page_hash = (page_t **)alloc_base;

alloc_base += pagehash_sz;

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);

PRM_DEBUG(page_hash);

alloc_base = alloc_page_freelists(alloc_base);

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);

PRM_DEBUG(alloc_base);

#ifdef TRAPTRACE

ttrace_buf = alloc_base;

alloc_base += tt_sz;

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);

PRM_DEBUG(alloc_base);

#endif /* TRAPTRACE */

pse_mutex = (pad_mutex_t *)alloc_base;

alloc_base += psetable_sz;

alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);

PRM_DEBUG(alloc_base);

/* adjust kmem64_end to what we really allocated */

kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);

kmem64_sz = kmem64_end - kmem64_base;

if (&ecache_init_scrub_flush_area) {

alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);

ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));

}

/*

if (physmem == 0 || physmem > npages)

physmem = npages;

/*

if (root_is_ramdisk == B_TRUE) {

ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;

physmem -= ramdisk_npages;

}

if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))

cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");

/*

memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);

if (memspace != (caddr_t)intr_vec_table)

prom_panic("interrupt vector table allocation failure");

/*

memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +

prom_phys_avail_len() + prom_virt_avail_len());

memlist_sz *= 2;

memlist_sz = roundup(memlist_sz, PAGESIZE);

memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);

if (memspace == NULL)

cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");

memlist = (struct memlist *)memspace;

memlist_end = (char *)memspace + memlist_sz;

PRM_DEBUG(memlist);

PRM_DEBUG(memlist_end);

PRM_DEBUG(sysbase);

PRM_DEBUG(syslimit);

kernelheap_init((void *)sysbase, (void *)syslimit,

(caddr_t)sysbase + PAGESIZE, NULL, NULL);

/*

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

/*

virt_avail = memlist;

copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);

for (cur = virt_avail; cur->next; cur = cur->next) {

uint64_t range_base, range_size;

if ((range_base = cur->address + cur->size) < (uint64_t)sysbase)

continue;

if (range_base >= (uint64_t)syslimit)

break;

/*

range_size = MIN(cur->next->address,

(uint64_t)syslimit) - range_base;

(void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,

0, 0, (void *)range_base, (void *)(range_base + range_size),

VM_NOSLEEP | VM_BESTFIT | VM_PANIC);

}

phys_avail = memlist;

copy_memlist(boot_physavail, boot_physavail_len, &memlist);

/*

nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);

if (nalloc_base == NULL)

nalloc_base = nalloc_end;

ndata_remain_sz = nalloc_end - nalloc_base;

/*

phys_install = memlist;

copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);

/*

nopp_list = memlist;

memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);

memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,

&memlist, &nopp_list);

memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);

if ((caddr_t)memlist > (memspace + memlist_sz))

prom_panic("memlist overflow");

/*

kphysm_init();

availrmem_initial = availrmem = freemem;

PRM_DEBUG(availrmem);

/*

page_lock_init();

/*

kmem_init();

/*

if (&page_set_colorequiv_arr_cpu != NULL)

page_set_colorequiv_arr_cpu();

else

page_set_colorequiv_arr();

/*

bp_init(shm_alignment, HAT_STRICTORDER);

/*

(void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,

panicbuf, panicbuf + PANICBUFSIZE,

VM_NOSLEEP | VM_BESTFIT | VM_PANIC);

(void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,

intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,

VM_NOSLEEP | VM_BESTFIT | VM_PANIC);

if (mpo_heap32_bufsz > (size_t)0) {

(void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,

PAGESIZE, 0, 0, mpo_heap32_buf,

mpo_heap32_buf + mpo_heap32_bufsz,

VM_NOSLEEP | VM_BESTFIT | VM_PANIC);

}

mem_config_init();

}

static void

startup_modules(void)

{

int nhblk1, nhblk8;

size_t nhblksz;

pgcnt_t pages_per_hblk;

size_t hme8blk_sz, hme1blk_sz;

/*

if (&set_platform_defaults)

set_platform_defaults();

/*

param_calc(0);

mod_setup();

/*

if (&iam_positron && iam_positron()) {

cmn_err(CE_WARN, "This hardware platform is not supported"

" by this release of Solaris.\n");

#ifdef DEBUG

prom_enter_mon(); /* Type 'go' to resume */

cmn_err(CE_WARN, "Booting an unsupported platform.\n");

cmn_err(CE_WARN, "Booting with down-rev firmware.\n");

#else /* DEBUG */

halt(0);

#endif /* DEBUG */

}

/*

do_prom_version_check();

/*

param_init();

/*

maxmem = physmem;

/* Set segkp limits. */

ncbase = kdi_segdebugbase;

ncend = kdi_segdebugbase;

/*

hat_init();

/*

seg_init();

/*

create_va_to_tte();

/*

forthdebug_init();

/*

startup_create_io_node();

if (modloadonly("fs", "specfs") == -1)

halt("Can't load specfs");

if (modloadonly("fs", "devfs") == -1)

halt("Can't load devfs");

if (modloadonly("misc", "swapgeneric") == -1)

halt("Can't load swapgeneric");

(void) modloadonly("sys", "lbl_edition");

dispinit();

/*

parse_idprom();

/* Read cluster configuration data. */

clconf_init();

setup_ddi();

/*

if (loadrootmodules() != 0)

debug_enter("Can't load the root filesystem");

/*

if (&load_tod_module)

load_tod_module();

/*

if (&load_platform_modules)

load_platform_modules();

setcpudelay();

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

/*

bop_alloc_pages = btopr(kmem64_end - kmem64_base);

pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));

bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);

nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;

bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);

/* sfmmu_init_nucleus_hblks expects properly aligned data structures */

hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));

hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));

bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);

pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));

nhblk8 = 0;

while (bop_alloc_pages > 1) {

bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);

nhblk8 += bop_alloc_pages /= pages_per_hblk;

bop_alloc_pages *= hme8blk_sz;

bop_alloc_pages = btopr(bop_alloc_pages);

}

nhblk8 += 2;

/*

nhblk8 += boot_virtavail_len;

nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;

/* Allocate in pagesize chunks */

nhblksz = roundup(nhblksz, MMU_PAGESIZE);

hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);

sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);

}

static void

startup_bop_gone(void)

{

/*

mach_descrip_startup_fini();

/*

bop_fini();

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

/*

if (&ecache_flush_address) {

ecache_flushaddr = ecache_flush_address();

if (ecache_flushaddr == (uint64_t)-1) {

cmn_err(CE_PANIC,

"startup: no memory to set ecache_flushaddr");

}

}

/*

ASSERT(virt_avail != NULL);

memlist_free_list(virt_avail);

virt_avail = memlist;

copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);

}

startup_fixup_physavail(void)

{

struct memlist *cur;

size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;

PRM_DEBUG(kmem64_overmap_size);

/*

copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,

&boot_physavail, &boot_physavail_len,

&boot_virtavail, &boot_virtavail_len);

/*

cur = memlist;

copy_memlist(boot_physavail, boot_physavail_len, &memlist);

/*

if (kmem64_overmap_size) {

memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),

kmem64_overmap_size, &memlist, &cur);

}

/*

if (ndata_remain_sz >= MMU_PAGESIZE)

memlist_add(va_to_pa(nalloc_base),

(uint64_t)ndata_remain_sz, &memlist, &cur);

/*

if ((caddr_t)memlist > (caddr_t)memlist_end)

cmn_err(CE_PANIC, "startup: memlist size exceeded");

/*

sync_memlists(phys_avail, cur);

ASSERT(phys_avail != NULL);

old_phys_avail = phys_avail;

phys_avail = cur;

}

update_kcage_ranges(uint64_t addr, uint64_t len)

{

pfn_t base = btop(addr);

pgcnt_t num = btop(len);

int rv;

rv = kcage_range_add(base, num, kcage_startup_dir);

if (rv == ENOMEM) {

cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",

(len == 0 ? 0 : BYTES_TO_MB(len)));

} else if (rv != 0) {

/* catch this in debug kernels */

ASSERT(0);

cmn_err(CE_WARN, "unexpected kcage_range_add"

" return value %d", rv);

}

}

static void

startup_vm(void)

{

size_t i;

struct segmap_crargs a;

struct segkpm_crargs b;

uint64_t avmem;

caddr_t va;

pgcnt_t max_phys_segkp;

int mnode;

extern int use_brk_lpg, use_stk_lpg;

/*

hat_kern_setup();

/*

setup_trap_table();

/*

install_va_to_tte();

/*

tba_taken_over = 1;

/* initialize MMU primary context register */

mmu_init_kcontext();

/*

CPUSET_ADD(cpu_ready_set, CPU->cpu_id);

CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);

/*

tbr_wr_addr_inited = 1;

/*

kvm_init();

ASSERT(old_phys_avail != NULL && phys_avail != NULL);

if (kernel_cage_enable) {

diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);

}

memlist_free_list(old_phys_avail);

/*

if (physmem + ramdisk_npages < npages) {

pgcnt_t diff, off;

struct page *pp;

struct seg kseg;

cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);

off = 0;

diff = npages - (physmem + ramdisk_npages);

diff -= mmu_btopr(diff * sizeof (struct page));

kseg.s_as = &kas;

while (diff--) {

pp = page_create_va(&unused_pages_vp, (offset_t)off,

MMU_PAGESIZE, PG_WAIT | PG_EXCL,

&kseg, (caddr_t)off);

if (pp == NULL)

cmn_err(CE_PANIC, "limited physmem too much!");

page_io_unlock(pp);

page_downgrade(pp);

availrmem--;

off += MMU_PAGESIZE;

}

}

/*

cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",

(ulong_t)(physinstalled) << (PAGESHIFT - 10),

(ulong_t)(physinstalled) << (PAGESHIFT - 12));

avmem = (uint64_t)freemem << PAGESHIFT;

cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);

/*

if (physmem < privm_lpg_min_physmem) {

use_brk_lpg = 0;

use_stk_lpg = 0;

}

/*

if (&plat_freelist_process) {

for (mnode = 0; mnode < max_mem_nodes; mnode++)

if (mem_node_config[mnode].exists)

plat_freelist_process(mnode);

}

/*

/* XXX - cache alignment? */

va = (caddr_t)SEGKPBASE;

ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);

max_phys_segkp = (physmem * 2);

if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {

segkpsize = btop(SEGKPDEFSIZE);

cmn_err(CE_WARN, "Illegal value for segkpsize. "

"segkpsize has been reset to %ld pages", segkpsize);

}

i = ptob(MIN(segkpsize, max_phys_segkp));

rw_enter(&kas.a_lock, RW_WRITER);

if (seg_attach(&kas, va, i, segkp) < 0)

cmn_err(CE_PANIC, "startup: cannot attach segkp");

if (segkp_create(segkp) != 0)

cmn_err(CE_PANIC, "startup: segkp_create failed");

rw_exit(&kas.a_lock);

/*

segmap_kpm = kpm_enable &&

segmap_kpm && PAGESIZE == MAXBSIZE;

if (kpm_enable) {

rw_enter(&kas.a_lock, RW_WRITER);

/*

if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,

segkpm) < 0)

cmn_err(CE_PANIC, "cannot attach segkpm");

b.prot = PROT_READ | PROT_WRITE;

b.nvcolors = shm_alignment >> MMU_PAGESHIFT;

if (segkpm_create(segkpm, (caddr_t)&b) != 0)

panic("segkpm_create segkpm");

rw_exit(&kas.a_lock);

mach_kpm_init();

}

if (!segzio_fromheap) {

size_t size;

size_t physmem_b = mmu_ptob(physmem);

/* size is in bytes, segziosize is in pages */

if (segziosize == 0) {

size = physmem_b;

} else {

size = mmu_ptob(segziosize);

}

if (size < SEGZIOMINSIZE) {

size = SEGZIOMINSIZE;

} else if (size > SEGZIOMAXSIZE) {

size = SEGZIOMAXSIZE;

/*

if (physmem_b > SEGZIOMAXSIZE) {

size += (physmem_b - SEGZIOMAXSIZE) / 2;

}

}

segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));

/* put the base of the ZIO segment after the kpm segment */

segzio_base = kpm_vbase + (kpm_size * vac_colors);

PRM_DEBUG(segziosize);

PRM_DEBUG(segzio_base);

/*

rw_enter(&kas.a_lock, RW_WRITER);

(void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),

&kzioseg);

(void) segkmem_zio_create(&kzioseg);

/* create zio area covering new segment */

segkmem_zio_init(segzio_base, mmu_ptob(segziosize));

rw_exit(&kas.a_lock);

}

/*

va = (caddr_t)SEGMAPBASE;

/*

ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);

/*

i = mmu_ptob((freemem * segmap_percent) / 100);

if (i < MINMAPSIZE)

i = MINMAPSIZE;

if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))

i = MIN(SEGMAPSIZE, mmu_ptob(freemem));

i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */

rw_enter(&kas.a_lock, RW_WRITER);

if (seg_attach(&kas, va, i, segkmap) < 0)

cmn_err(CE_PANIC, "cannot attach segkmap");

a.prot = PROT_READ | PROT_WRITE;

a.shmsize = shm_alignment;

a.nfreelist = 0; /* use segmap driver defaults */

if (segmap_create(segkmap, (caddr_t)&a) != 0)

panic("segmap_create segkmap");

rw_exit(&kas.a_lock);

segdev_init();

}

static void

startup_end(void)

{

if ((caddr_t)memlist > (caddr_t)memlist_end)

panic("memlist overflow 2");

memlist_free_block((caddr_t)memlist,

((caddr_t)memlist_end - (caddr_t)memlist));

memlist = NULL;

/* enable page_relocation since OBP is now done */

page_relocate_ready = 1;

/*

kern_setup1();

/*

contig_mem_init();

mach_descrip_init();

if (cpu_intrq_setup(CPU)) {

cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);

}

cpu_intrq_register(CPU);

mach_htraptrace_setup(CPU->cpu_id);

mach_htraptrace_configure(CPU->cpu_id);

mach_dump_buffer_init();

/*

cpu_intr_alloc(CPU, NINTR_THREADS);

(void) splzs(); /* allow hi clock ints but not zs */

/*

error_init();

/*

mach_hw_copy_limit();

/*

(void) prom_set_preprom(kern_preprom);

(void) prom_set_postprom(kern_postprom);

CPU->cpu_m.mutex_ready = 1;

/*

segnf_init();

/*

configure(); /* set up devices */

mach_cpu_halt_idle();

}

post_startup(void)

{

#ifdef PTL1_PANIC_DEBUG

extern void init_ptl1_thread(void);

#endif /* PTL1_PANIC_DEBUG */

extern void abort_sequence_init(void);

/*

bind_hwcap();

/*

mach_memscrub();

/*

abort_sequence_init();

/*

(void) mod_sysctl(SYS_FORCELOAD, NULL);

/*

(void) modload("fs", "procfs");

/* load machine class specific drivers */

load_mach_drivers();

/* load platform specific drivers */

if (&load_platform_drivers)

load_platform_drivers();

/* load vis simulation module, if we are running w/fpu off */

if (!fpu_exists) {

if (modload("misc", "vis") == -1)

halt("Can't load vis");

}

mach_fpras();

maxmem = freemem;

#ifdef PTL1_PANIC_DEBUG

init_ptl1_thread();

#endif /* PTL1_PANIC_DEBUG */

}

#ifdef PTL1_PANIC_DEBUG

int ptl1_panic_test = 0;

int ptl1_panic_xc_one_test = 0;

int ptl1_panic_xc_all_test = 0;

int ptl1_panic_xt_one_test = 0;

int ptl1_panic_xt_all_test = 0;

kthread_id_t ptl1_thread_p = NULL;

kcondvar_t ptl1_cv;

kmutex_t ptl1_mutex;

int ptl1_recurse_count_threshold = 0x40;

int ptl1_recurse_trap_threshold = 0x3d;

extern void ptl1_recurse(int, int);

extern void ptl1_panic_xt(int, int);

static void

ptl1_wakeup(void *arg)

{

mutex_enter(&ptl1_mutex);

cv_signal(&ptl1_cv);

mutex_exit(&ptl1_mutex);

}

static void

ptl1_panic_xc(void)

{

ptl1_recurse(ptl1_recurse_count_threshold,

ptl1_recurse_trap_threshold);

}

static void

ptl1_thread(void)

{

mutex_enter(&ptl1_mutex);

while (ptl1_thread_p) {

cpuset_t other_cpus;

int cpu_id;

int my_cpu_id;

int target_cpu_id;

int target_found;

if (ptl1_panic_test) {

ptl1_recurse(ptl1_recurse_count_threshold,

ptl1_recurse_trap_threshold);

}

/*

kpreempt_disable();

my_cpu_id = CPU->cpu_id;

other_cpus = cpu_ready_set;

CPUSET_DEL(other_cpus, CPU->cpu_id);

target_found = 0;

if (!CPUSET_ISNULL(other_cpus)) {

/*

for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {

if (cpu_id == my_cpu_id)

continue;

if (CPU_XCALL_READY(cpu_id)) {

target_cpu_id = cpu_id;

target_found = 1;

break;

}

}

ASSERT(target_found);

if (ptl1_panic_xc_one_test) {

xc_one(target_cpu_id,

(xcfunc_t *)ptl1_panic_xc, 0, 0);

}

if (ptl1_panic_xc_all_test) {

xc_some(other_cpus,

(xcfunc_t *)ptl1_panic_xc, 0, 0);

}

if (ptl1_panic_xt_one_test) {

xt_one(target_cpu_id,

(xcfunc_t *)ptl1_panic_xt, 0, 0);

}

if (ptl1_panic_xt_all_test) {

xt_some(other_cpus,

(xcfunc_t *)ptl1_panic_xt, 0, 0);

}

}

kpreempt_enable();

(void) timeout(ptl1_wakeup, NULL, hz);

(void) cv_wait(&ptl1_cv, &ptl1_mutex);

}

mutex_exit(&ptl1_mutex);

}

init_ptl1_thread(void)

{

ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,

&p0, TS_RUN, 0);

}

#endif /* PTL1_PANIC_DEBUG */

static void

memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)

{

struct memlist *new;

new = *memlistp;

new->address = start;

new->size = len;

*memlistp = new + 1;

}

static void

memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,

struct memlist **curmemlistp)

{

struct memlist *new = *memlistp;

memlist_new(start, len, memlistp);

memlist_insert(new, curmemlistp);

}

static int

ndata_alloc_memseg(struct memlist *ndata, size_t avail)

{

int nseg;

size_t memseg_sz;

struct memseg *msp;

/*

memseg_sz = (avail + 10) * sizeof (struct memseg);

memseg_sz = roundup(memseg_sz, PAGESIZE);

nseg = memseg_sz / sizeof (struct memseg);

msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);

if (msp == NULL)

return (1);

PRM_DEBUG(memseg_free);

while (nseg--) {

msp->next = memseg_free;

memseg_free = msp;

msp++;

}

return (0);

}

static void

memseg_list_add(struct memseg *memsegp)

{

struct memseg **prev_memsegp;

pgcnt_t num;

/* insert in memseg list, decreasing number of pages order */

num = MSEG_NPAGES(memsegp);

for (prev_memsegp = &memsegs; *prev_memsegp;

prev_memsegp = &((*prev_memsegp)->next)) {

if (num > MSEG_NPAGES(*prev_memsegp))

break;

}

memsegp->next = *prev_memsegp;

*prev_memsegp = memsegp;

if (kpm_enable) {

memsegp->nextpa = (memsegp->next) ?

va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;

if (prev_memsegp != &memsegs) {

struct memseg *msp;

msp = (struct memseg *)((caddr_t)prev_memsegp -

offsetof(struct memseg, next));

msp->nextpa = va_to_pa(memsegp);

} else {

memsegspa = va_to_pa(memsegs);

}

}

}

#define PREFETCH_BYTES 64

add_physmem_cb(page_t *pp, pfn_t pnum)

{

extern void prefetch_page_w(void *);

pp->p_pagenum = pnum;

/*

/*LINTED*/

ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);

prefetch_page_w((char *)pp);

}

static struct memseg *

memseg_find(pfn_t base, pfn_t *next)

{

struct memseg *seg;

if (next != NULL)

*next = LONG_MAX;

for (seg = memsegs; seg != NULL; seg = seg->next) {

if (base >= seg->pages_base && base < seg->pages_end)

return (seg);

if (next != NULL && seg->pages_base > base &&

seg->pages_base < *next)

*next = seg->pages_base;

}

return (NULL);

}

extern struct vnode prom_ppages;

static void

kphysm_erase(uint64_t addr, uint64_t len)

{

struct page *pp;

struct memseg *seg;

pfn_t base = btop(addr), next;

pgcnt_t num = btop(len);

while (num != 0) {

pgcnt_t off, left;

seg = memseg_find(base, &next);

if (seg == NULL) {

if (next == LONG_MAX)

break;

left = MIN(next - base, num);

base += left, num -= left;

continue;

}

off = base - seg->pages_base;

pp = seg->pages + off;

left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);

while (num != left) {

/*

add_physmem_cb(pp, base);

if (page_trylock(pp, SE_EXCL) == 0)

cmn_err(CE_PANIC, "prom page locked");

(void) page_hashin(pp, &prom_ppages,

(offset_t)base, NULL);

(void) page_pp_lock(pp, 0, 1);

pp++, base++, num--;

}

}

}

static page_t *ppnext;

static pgcnt_t ppleft;

static void *kpm_ppnext;

static pgcnt_t kpm_ppleft;

static void

kphysm_memseg(uint64_t addr, uint64_t len)

{

pfn_t base = btop(addr);

pgcnt_t num = btop(len);

struct memseg *seg;

seg = memseg_free;

memseg_free = seg->next;

ASSERT(seg != NULL);

seg->pages = ppnext;

seg->epages = ppnext + num;

seg->pages_base = base;

seg->pages_end = base + num;

ppnext += num;

ppleft -= num;

if (kpm_enable) {

pgcnt_t kpnum = ptokpmpr(num);

if (kpnum > kpm_ppleft)

panic("kphysm_memseg: kpm_pp overflow");

seg->pagespa = va_to_pa(seg->pages);

seg->epagespa = va_to_pa(seg->epages);

seg->kpm_pbase = kpmptop(ptokpmp(base));

seg->kpm_nkpmpgs = kpnum;

/*

if (kpm_smallpages) {

kpm_spage_t *kpm_pp = kpm_ppnext;

kpm_ppnext = kpm_pp + kpnum;

seg->kpm_spages = kpm_pp;

seg->kpm_pagespa = va_to_pa(seg->kpm_spages);

} else {

kpm_page_t *kpm_pp = kpm_ppnext;

kpm_ppnext = kpm_pp + kpnum;

seg->kpm_pages = kpm_pp;

seg->kpm_pagespa = va_to_pa(seg->kpm_pages);

/* ASSERT no kpm overlaps */

ASSERT(

memseg_find(base - pmodkpmp(base), NULL) == NULL);

ASSERT(memseg_find(

roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);

}

kpm_ppleft -= num;

}

memseg_list_add(seg);

}

kphysm_add(uint64_t addr, uint64_t len, int reclaim)

{

struct page *pp;

struct memseg *seg;

pfn_t base = btop(addr);

pgcnt_t num = btop(len);

seg = memseg_find(base, NULL);

ASSERT(seg != NULL);

pp = seg->pages + (base - seg->pages_base);

if (reclaim) {

struct page *rpp = pp;

struct page *lpp = pp + num;

/*

while (rpp < lpp) {

ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &prom_ppages);

page_pp_unlock(rpp, 0, 1);

page_hashout(rpp, NULL);

page_unlock(rpp);

rpp++;

}

}

/*

add_physmem(pp, num, base);

}

static void

kphysm_init(void)

{

struct memlist *pmem;

ASSERT(page_hash != NULL && page_hashsz != 0);

ppnext = pp_base;

ppleft = npages;

kpm_ppnext = kpm_pp_base;

kpm_ppleft = kpm_npages;

/*

diff_memlists(phys_install, nopp_list, kphysm_memseg);

/*

for (pmem = phys_avail; pmem != NULL; pmem = pmem->next)

kphysm_add(pmem->address, pmem->size, 0);

/*

diff_memlists(phys_install, phys_avail, kphysm_erase);

build_pfn_hash();

}

static void

kvm_init(void)

{

/*

rw_enter(&kas.a_lock, RW_WRITER);

as_avlinit(&kas);

(void) seg_attach(&kas, (caddr_t)KERNELBASE,

(size_t)(e_moddata - KERNELBASE), &ktextseg);

(void) segkmem_create(&ktextseg);

(void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),

(size_t)(MMU_PAGESIZE4M), &ktexthole);

(void) segkmem_create(&ktexthole);

(void) seg_attach(&kas, (caddr_t)valloc_base,

(size_t)(econtig32 - valloc_base), &kvalloc);

(void) segkmem_create(&kvalloc);

if (kmem64_base) {

(void) seg_attach(&kas, (caddr_t)kmem64_base,

(size_t)(kmem64_end - kmem64_base), &kmem64);

(void) segkmem_create(&kmem64);

}

/*

(void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);

(void) segkmem_create(&kvseg);

hblk_alloc_dynamic = 1;

/*

memseg_remap_init();

/* at this point we are ready to use large page heap */

segkmem_heap_lp_init();

(void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,

&kvseg32);

(void) segkmem_create(&kvseg32);

/*

(void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);

(void) segkmem_create(&kdebugseg);

rw_exit(&kas.a_lock);

}

char obp_tte_str[] =

"h# %x constant MMU_PAGESHIFT "

"h# %x constant TTE8K "

"h# %x constant SFHME_SIZE "

"h# %x constant SFHME_TTE "

"h# %x constant HMEBLK_TAG "

"h# %x constant HMEBLK_NEXT "

"h# %x constant HMEBLK_MISC "

"h# %x constant HMEBLK_HME1 "

"h# %x constant NHMENTS "

"h# %x constant HBLK_SZMASK "

"h# %x constant HBLK_RANGE_SHIFT "

"h# %x constant HMEBP_HBLK "

"h# %x constant HMEBUCKET_SIZE "

"h# %x constant HTAG_SFMMUPSZ "

"h# %x constant HTAG_BSPAGE_SHIFT "

"h# %x constant HTAG_REHASH_SHIFT "

"h# %x constant SFMMU_INVALID_SHMERID "

"h# %x constant mmu_hashcnt "

"h# %p constant uhme_hash "

"h# %p constant khme_hash "

"h# %x constant UHMEHASH_SZ "

"h# %x constant KHMEHASH_SZ "

"h# %p constant KCONTEXT "

"h# %p constant KHATID "

"h# %x constant ASI_MEM "

": PHYS-X@ ( phys -- data ) "

" ASI_MEM spacex@ "

"; "

": PHYS-W@ ( phys -- data ) "

" ASI_MEM spacew@ "

"; "

": PHYS-L@ ( phys -- data ) "

" ASI_MEM spaceL@ "

"; "

": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "

" 3 * MMU_PAGESHIFT + "

"; "

": TTE_IS_VALID ( ttep -- flag ) "

" PHYS-X@ 0< "

"; "

": HME_HASH_SHIFT ( ttesz -- hmeshift ) "

" dup TTE8K = if "

" drop HBLK_RANGE_SHIFT "

" else "

" TTE_PAGE_SHIFT "

" then "

"; "

": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "

" tuck >> swap MMU_PAGESHIFT - << "

"; "

": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "

" >> over xor swap ( hash sfmmup ) "

" KHATID <> if ( hash ) "

" UHMEHASH_SZ and ( bucket ) "

" HMEBUCKET_SIZE * uhme_hash + ( hmebp ) "

" else ( hash ) "

" KHMEHASH_SZ and ( bucket ) "

" HMEBUCKET_SIZE * khme_hash + ( hmebp ) "

" then ( hmebp ) "

"; "

": HME_HASH_TABLE_SEARCH "

" ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) "

" >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "

" dup if ( sfmmup hmeblkp ) ( r: hblktag ) "

" dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) "

" dup hmeblk_tag + 8 + phys-x@ 2 pick = if "

" true ( sfmmup hmeblkp true ) ( r: hblktag ) "

" else "

" hmeblk_next + phys-x@ false "

" ( sfmmup hmeblkp false ) ( r: hblktag ) "

" then "

" else "

" hmeblk_next + phys-x@ false "

" ( sfmmup hmeblkp false ) ( r: hblktag ) "

" then "

" else "

" true "

" then "

" until r> drop "

"; "

": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "

" over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) "

" HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )"

" swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )"

" SFMMU_INVALID_SHMERID or nip ( hblktag ) "

"; "

": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "

" over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) "

" TTE8K = if ( hmeblkp addr ) "

" MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) "

" else ( hmeblkp addr ) "

" drop 0 ( hmeblkp 0 ) "

" then ( hmeblkp hme-index ) "

" SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) "

" SFHME_TTE + ( ttep ) "

"; "

": unix-tte ( addr cnum -- false | tte-data true ) "

" KCONTEXT = if ( addr ) "

" KHATID ( addr khatid ) "

" else ( addr ) "

" drop false exit ( false ) "

" then "

" ( addr khatid ) "

" mmu_hashcnt 1+ 1 do ( addr sfmmup ) "

" 2dup swap i HME_HASH_SHIFT "

"( addr sfmmup sfmmup addr hmeshift ) "

" HME_HASH_FUNCTION ( addr sfmmup hmebp ) "

" over i 4 pick "

"( addr sfmmup hmebp sfmmup rehash addr ) "

" HME_HASH_TAG ( addr sfmmup hmebp hblktag ) "

" HME_HASH_TABLE_SEARCH "

"( addr sfmmup { null | hmeblkp } ) "

" ?dup if ( addr sfmmup hmeblkp ) "

" nip swap HBLK_TO_TTEP ( ttep ) "

" dup TTE_IS_VALID if ( valid-ttep ) "

" PHYS-X@ true ( tte-data true ) "

" else ( invalid-tte ) "

" drop false ( false ) "

" then ( false | tte-data true ) "

" unloop exit ( false | tte-data true ) "

" then ( addr sfmmup ) "

" loop ( addr sfmmup ) "

" 2drop false ( false ) "

"; "

;

create_va_to_tte(void)

{

char *bp;

extern int khmehash_num, uhmehash_num;

extern struct hmehash_bucket *khme_hash, *uhme_hash;

#define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field))

bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);

/*

(void) sprintf(bp, obp_tte_str,

MMU_PAGESHIFT,

TTE8K,

sizeof (struct sf_hment),

OFFSET(struct sf_hment, hme_tte),

OFFSET(struct hme_blk, hblk_tag),

OFFSET(struct hme_blk, hblk_nextpa),

OFFSET(struct hme_blk, hblk_misc),

OFFSET(struct hme_blk, hblk_hme),

NHMENTS,

HBLK_SZMASK,

HBLK_RANGE_SHIFT,

OFFSET(struct hmehash_bucket, hmeh_nextpa),

sizeof (struct hmehash_bucket),

HTAG_SFMMUPSZ,

HTAG_BSPAGE_SHIFT,

HTAG_REHASH_SHIFT,

SFMMU_INVALID_SHMERID,

mmu_hashcnt,

(caddr_t)va_to_pa((caddr_t)uhme_hash),

(caddr_t)va_to_pa((caddr_t)khme_hash),

UHMEHASH_SZ,

KHMEHASH_SZ,

KCONTEXT,

KHATID,

ASI_MEM);

prom_interpret(bp, 0, 0, 0, 0, 0);

kobj_free(bp, MMU_PAGESIZE);

}

install_va_to_tte(void)

{

/*

prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);

}

static char *create_node =

"\" /\" find-device "

"new-device "

"\" os-io\" device-name "

"\" "OBP_DISPLAY_CONSOLE"\" device-type "

": cb-r/w ( adr,len method$ -- #read/#written ) "

" 2>r swap 2 2r> ['] $callback catch if "

" 2drop 3drop 0 "

" then "

"; "

": read ( adr,len -- #read ) "

" \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then "

" ( retN ... ret1 N ) "

" ?dup if "

" swap >r 1- 0 ?do drop loop r> "

" else "

" -2 "

" then "

"; "

": write ( adr,len -- #written ) "

" \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then "

" ( retN ... ret1 N ) "

" ?dup if "

" swap >r 1- 0 ?do drop loop r> "

" else "

" 0 "

" then "

"; "

": poll-tty ( -- ) ; "

": install-abort ( -- ) ['] poll-tty d# 10 alarm ; "

": remove-abort ( -- ) ['] poll-tty 0 alarm ; "

": cb-give/take ( $method -- ) "

" 0 -rot ['] $callback catch ?dup if "

" >r 2drop 2drop r> throw "

" else "

" 0 ?do drop loop "

" then "

"; "

": give ( -- ) \" exit-input\" cb-give/take ; "

": take ( -- ) \" enter-input\" cb-give/take ; "

": open ( -- ok? ) true ; "

": close ( -- ) ; "

"finish-device "

"device-end ";

static void

startup_create_io_node(void)

{

prom_interpret(create_node, 0, 0, 0, 0, 0);

}

static void

do_prom_version_check(void)

{

int i;

pnode_t node;

char buf[64];

static char drev[] = "Down-rev firmware detected%s\n"

"\tPlease upgrade to the following minimum version:\n"

"\t\t%s\n";

i = prom_version_check(buf, sizeof (buf), &node);

if (i == PROM_VER64_OK)

return;

if (i == PROM_VER64_UPGRADE) {

cmn_err(CE_WARN, drev, "", buf);

#ifdef DEBUG

prom_enter_mon(); /* Type 'go' to continue */

cmn_err(CE_WARN, "Booting with down-rev firmware\n");

return;

#else

halt(0);

#endif

}

/*

cmn_err(CE_WARN, drev, " on one or more CPU boards", buf);

}

extern caddr_t modtext;

extern size_t modtext_sz;

extern caddr_t moddata;