mpcore.s revision dad255286ee5ada77255c1f9f132ceee0bc314aa
970N/A/*
1153N/A * CDDL HEADER START
970N/A *
970N/A * The contents of this file are subject to the terms of the
970N/A * Common Development and Distribution License (the "License").
970N/A * You may not use this file except in compliance with the License.
970N/A *
970N/A * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
970N/A * or http://www.opensolaris.org/os/licensing.
970N/A * See the License for the specific language governing permissions
970N/A * and limitations under the License.
970N/A *
970N/A * When distributing Covered Code, include this CDDL HEADER in each
970N/A * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
970N/A * If applicable, add the following below this CDDL HEADER, with the
970N/A * fields enclosed by brackets "[]" replaced with your own identifying
970N/A * information: Portions Copyright [yyyy] [name of copyright owner]
970N/A *
970N/A * CDDL HEADER END
970N/A */
970N/A/*
970N/A * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
970N/A */
1153N/A/*
1153N/A * Copyright (c) 2010, Intel Corporation.
1153N/A * All rights reserved.
1153N/A */
#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/x86_archext.h>
#if !defined(__lint)
#include <sys/segments.h>
#include "assym.h"
#endif
/*
* Our assumptions:
* - We are running in real mode.
* - Interrupts are disabled.
* - Selectors are equal (cs == ds == ss) for all real mode code
* - The GDT, IDT, ktss and page directory has been built for us
*
* Our actions:
* Start CPU:
* - We start using our GDT by loading correct values in the
* selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
* gs=KGS_SEL).
* - We change over to using our IDT.
* - We load the default LDT into the hardware LDT register.
* - We load the default TSS into the hardware task register.
* - call mp_startup(void) indirectly through the T_PC
* Stop CPU:
* - Put CPU into halted state with interrupts disabled
*
*/
#if defined(__lint)
void
real_mode_start_cpu(void)
{}
void
real_mode_stop_cpu_stage1(void)
{}
void
real_mode_stop_cpu_stage2(void)
{}
#else /* __lint */
#if defined(__amd64)
ENTRY_NP(real_mode_start_cpu)
/*
* NOTE: The GNU assembler automatically does the right thing to
* generate data size operand prefixes based on the code size
* generation mode (e.g. .code16, .code32, .code64) and as such
* prefixes need not be used on instructions EXCEPT in the case
* of address prefixes for code for which the reference is not
* automatically of the default operand size.
*/
.code16
cli
movw %cs, %ax
movw %ax, %ds /* load cs into ds */
movw %ax, %ss /* and into ss */
/*
* Helps in debugging by giving us the fault address.
*
* Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
*/
movl $0xffc, %esp
movl %cr0, %eax
/*
* Enable protected-mode, write protect, and alignment mask
*/
orl $(CR0_PE|CR0_WP|CR0_AM), %eax
movl %eax, %cr0
/*
* Do a jmp immediately after writing to cr0 when enabling protected
* mode to clear the real mode prefetch queue (per Intel's docs)
*/
jmp pestart
pestart:
/*
* 16-bit protected mode is now active, so prepare to turn on long
* mode.
*
* Note that we currently assume that if we're attempting to run a
* kernel compiled with (__amd64) #defined, the target CPU has long
* mode support.
*/
#if 0
/*
* If there's a chance this might not be true, the following test should
* be done, with the no_long_mode branch then doing something
* appropriate:
*/
movl $0x80000000, %eax /* get largest extended CPUID */
cpuid
cmpl $0x80000000, %eax /* check if > 0x80000000 */
jbe no_long_mode /* nope, no long mode */
movl $0x80000001, %eax
cpuid /* get extended feature flags */
btl $29, %edx /* check for long mode */
jnc no_long_mode /* long mode not supported */
#endif
/*
* Add any initial cr4 bits
*/
movl %cr4, %eax
addr32 orl CR4OFF, %eax
/*
* Enable PAE mode (CR4.PAE)
*/
orl $CR4_PAE, %eax
movl %eax, %cr4
/*
* Point cr3 to the 64-bit long mode page tables.
*
* Note that these MUST exist in 32-bit space, as we don't have
* a way to load %cr3 with a 64-bit base address for the page tables
* until the CPU is actually executing in 64-bit long mode.
*/
addr32 movl CR3OFF, %eax
movl %eax, %cr3
/*
* Set long mode enable in EFER (EFER.LME = 1)
*/
movl $MSR_AMD_EFER, %ecx
rdmsr
orl $AMD_EFER_LME, %eax
wrmsr
/*
* Finally, turn on paging (CR0.PG = 1) to activate long mode.
*/
movl %cr0, %eax
orl $CR0_PG, %eax
movl %eax, %cr0
/*
* The instruction after enabling paging in CR0 MUST be a branch.
*/
jmp long_mode_active
long_mode_active:
/*
* Long mode is now active but since we're still running with the
* original 16-bit CS we're actually in 16-bit compatability mode.
*
* We have to load an intermediate GDT and IDT here that we know are
* in 32-bit space before we can use the kernel's GDT and IDT, which
* may be in the 64-bit address space, and since we're in compatability
* mode, we only have access to 16 and 32-bit instructions at the
* moment.
*/
addr32 lgdtl TEMPGDTOFF /* load temporary GDT */
addr32 lidtl TEMPIDTOFF /* load temporary IDT */
/*
* Do a far transfer to 64-bit mode. Set the CS selector to a 64-bit
* long mode selector (CS.L=1) in the temporary 32-bit GDT and jump
* to the real mode platter address of long_mode 64 as until the 64-bit
* CS is in place we don't have access to 64-bit instructions and thus
* can't reference a 64-bit %rip.
*/
pushl $TEMP_CS64_SEL
addr32 pushl LM64OFF
lretl
.globl long_mode_64
long_mode_64:
.code64
/*
* We are now running in long mode with a 64-bit CS (EFER.LMA=1,
* CS.L=1) so we now have access to 64-bit instructions.
*
* First, set the 64-bit GDT base.
*/
.globl rm_platter_pa
movl rm_platter_pa, %eax
lgdtq GDTROFF(%rax) /* load 64-bit GDT */
/*
* Save the CPU number in %r11; get the value here since it's saved in
* the real mode platter.
*/
movl CPUNOFF(%rax), %r11d
/*
* Add rm_platter_pa to %rsp to point it to the same location as seen
* from 64-bit mode.
*/
addq %rax, %rsp
/*
* Now do an lretq to load CS with the appropriate selector for the
* kernel's 64-bit GDT and to start executing 64-bit setup code at the
* virtual address where boot originally loaded this code rather than
* the copy in the real mode platter's rm_code array as we've been
* doing so far.
*/
pushq $KCS_SEL
pushq $kernel_cs_code
lretq
.globl real_mode_start_cpu_end
real_mode_start_cpu_end:
nop
kernel_cs_code:
/*
* Complete the balance of the setup we need to before executing
* 64-bit kernel code (namely init rsp, TSS, LGDT, FS and GS).
*/
.globl rm_platter_va
movq rm_platter_va, %rax
lidtq IDTROFF(%rax)
movw $KDS_SEL, %ax
movw %ax, %ds
movw %ax, %es
movw %ax, %ss
movw $KTSS_SEL, %ax /* setup kernel TSS */
ltr %ax
xorw %ax, %ax /* clear LDTR */
lldt %ax
/*
* Set GS to the address of the per-cpu structure as contained in
* cpu[cpu_number].
*
* Unfortunately there's no way to set the 64-bit gsbase with a mov,
* so we have to stuff the low 32 bits in %eax and the high 32 bits in
* %edx, then call wrmsr.
*/
leaq cpu(%rip), %rdi
movl (%rdi, %r11, 8), %eax
movl 4(%rdi, %r11, 8), %edx
movl $MSR_AMD_GSBASE, %ecx
wrmsr
/*
* Init FS and KernelGSBase.
*
* Based on code in mlsetup(), set them both to 8G (which shouldn't be
* valid until some 64-bit processes run); this will then cause an
* exception in any code that tries to index off them before they are
* properly setup.
*/
xorl %eax, %eax /* low 32 bits = 0 */
movl $2, %edx /* high 32 bits = 2 */
movl $MSR_AMD_FSBASE, %ecx
wrmsr
movl $MSR_AMD_KGSBASE, %ecx
wrmsr
/*
* Init %rsp to the exception stack set in tss_ist1 and create a legal
* AMD64 ABI stack frame
*/
movq %gs:CPU_TSS, %rax
movq TSS_IST1(%rax), %rsp
pushq $0 /* null return address */
pushq $0 /* null frame pointer terminates stack trace */
movq %rsp, %rbp /* stack aligned on 16-byte boundary */
movq %cr0, %rax
andq $~(CR0_TS|CR0_EM), %rax /* clear emulate math chip bit */
orq $(CR0_MP|CR0_NE), %rax
movq %rax, %cr0 /* set machine status word */
/*
* Before going any further, enable usage of page table NX bit if
* that's how our page tables are set up.
*/
bt $X86FSET_NX, x86_featureset(%rip)
jnc 1f
movl $MSR_AMD_EFER, %ecx
rdmsr
orl $AMD_EFER_NXE, %eax
wrmsr
1:
/*
* Complete the rest of the setup and call mp_startup().
*/
movq %gs:CPU_THREAD, %rax /* get thread ptr */
call *T_PC(%rax) /* call mp_startup */
/* not reached */
int $20 /* whoops, returned somehow! */
SET_SIZE(real_mode_start_cpu)
#elif defined(__i386)
ENTRY_NP(real_mode_start_cpu)
#if !defined(__GNUC_AS__)
cli
D16 movw %cs, %eax
movw %eax, %ds /* load cs into ds */
movw %eax, %ss /* and into ss */
/*
* Helps in debugging by giving us the fault address.
*
* Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
*/
D16 movl $0xffc, %esp
D16 A16 lgdt %cs:GDTROFF
D16 A16 lidt %cs:IDTROFF
D16 A16 movl %cs:CR4OFF, %eax /* set up CR4, if desired */
D16 andl %eax, %eax
D16 A16 je no_cr4
D16 movl %eax, %ecx
D16 movl %cr4, %eax
D16 orl %ecx, %eax
D16 movl %eax, %cr4
no_cr4:
D16 A16 movl %cs:CR3OFF, %eax
A16 movl %eax, %cr3
movl %cr0, %eax
/*
* Enable protected-mode, paging, write protect, and alignment mask
*/
D16 orl $[CR0_PG|CR0_PE|CR0_WP|CR0_AM], %eax
movl %eax, %cr0
jmp pestart
pestart:
D16 pushl $KCS_SEL
D16 pushl $kernel_cs_code
D16 lret
.globl real_mode_start_cpu_end
real_mode_start_cpu_end:
nop
.globl kernel_cs_code
kernel_cs_code:
/*
* At this point we are with kernel's cs and proper eip.
*
* We will be executing not from the copy in real mode platter,
* but from the original code where boot loaded us.
*
* By this time GDT and IDT are loaded as is cr3.
*/
movw $KFS_SEL,%eax
movw %eax,%fs
movw $KGS_SEL,%eax
movw %eax,%gs
movw $KDS_SEL,%eax
movw %eax,%ds
movw %eax,%es
movl %gs:CPU_TSS,%esi
movw %eax,%ss
movl TSS_ESP0(%esi),%esp
movw $KTSS_SEL,%ax
ltr %ax
xorw %ax, %ax /* clear LDTR */
lldt %ax
movl %cr0,%edx
andl $-1![CR0_TS|CR0_EM],%edx /* clear emulate math chip bit */
orl $[CR0_MP|CR0_NE],%edx
movl %edx,%cr0 /* set machine status word */
/*
* Before going any further, enable usage of page table NX bit if
* that's how our page tables are set up.
*/
bt $X86FSET_NX, x86_featureset
jnc 1f
movl %cr4, %ecx
andl $CR4_PAE, %ecx
jz 1f
movl $MSR_AMD_EFER, %ecx
rdmsr
orl $AMD_EFER_NXE, %eax
wrmsr
1:
movl %gs:CPU_THREAD, %eax /* get thread ptr */
call *T_PC(%eax) /* call mp_startup */
/* not reached */
int $20 /* whoops, returned somehow! */
#else
cli
mov %cs, %ax
mov %eax, %ds /* load cs into ds */
mov %eax, %ss /* and into ss */
/*
* Helps in debugging by giving us the fault address.
*
* Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
*/
D16 mov $0xffc, %esp
D16 A16 lgdtl %cs:GDTROFF
D16 A16 lidtl %cs:IDTROFF
D16 A16 mov %cs:CR4OFF, %eax /* set up CR4, if desired */
D16 and %eax, %eax
D16 A16 je no_cr4
D16 mov %eax, %ecx
D16 mov %cr4, %eax
D16 or %ecx, %eax
D16 mov %eax, %cr4
no_cr4:
D16 A16 mov %cs:CR3OFF, %eax
A16 mov %eax, %cr3
mov %cr0, %eax
/*
* Enable protected-mode, paging, write protect, and alignment mask
*/
D16 or $(CR0_PG|CR0_PE|CR0_WP|CR0_AM), %eax
mov %eax, %cr0
jmp pestart
pestart:
D16 pushl $KCS_SEL
D16 pushl $kernel_cs_code
D16 lret
.globl real_mode_start_cpu_end
real_mode_start_cpu_end:
nop
.globl kernel_cs_code
kernel_cs_code:
/*
* At this point we are with kernel's cs and proper eip.
*
* We will be executing not from the copy in real mode platter,
* but from the original code where boot loaded us.
*
* By this time GDT and IDT are loaded as is cr3.
*/
mov $KFS_SEL, %ax
mov %eax, %fs
mov $KGS_SEL, %ax
mov %eax, %gs
mov $KDS_SEL, %ax
mov %eax, %ds
mov %eax, %es
mov %gs:CPU_TSS, %esi
mov %eax, %ss
mov TSS_ESP0(%esi), %esp
mov $(KTSS_SEL), %ax
ltr %ax
xorw %ax, %ax /* clear LDTR */
lldt %ax
mov %cr0, %edx
and $~(CR0_TS|CR0_EM), %edx /* clear emulate math chip bit */
or $(CR0_MP|CR0_NE), %edx
mov %edx, %cr0 /* set machine status word */
/*
* Before going any farther, enable usage of page table NX bit if
* that's how our page tables are set up.
*/
bt $X86FSET_NX, x86_featureset
jnc 1f
movl %cr4, %ecx
andl $CR4_PAE, %ecx
jz 1f
movl $MSR_AMD_EFER, %ecx
rdmsr
orl $AMD_EFER_NXE, %eax
wrmsr
1:
mov %gs:CPU_THREAD, %eax /* get thread ptr */
call *T_PC(%eax) /* call mp_startup */
/* not reached */
int $20 /* whoops, returned somehow! */
#endif
SET_SIZE(real_mode_start_cpu)
#endif /* __amd64 */
#if defined(__amd64)
ENTRY_NP(real_mode_stop_cpu_stage1)
#if !defined(__GNUC_AS__)
/*
* For vulcan as we need to do a .code32 and mentally invert the
* meaning of the addr16 and data16 prefixes to get 32-bit access when
* generating code to be executed in 16-bit mode (sigh...)
*/
.code32
cli
movw %cs, %ax
movw %ax, %ds /* load cs into ds */
movw %ax, %ss /* and into ss */
/*
* Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
*/
movw $CPUHALTCODEOFF, %ax
.byte 0xff, 0xe0 /* jmp *%ax */
#else /* __GNUC_AS__ */
/*
* NOTE: The GNU assembler automatically does the right thing to
* generate data size operand prefixes based on the code size
* generation mode (e.g. .code16, .code32, .code64) and as such
* prefixes need not be used on instructions EXCEPT in the case
* of address prefixes for code for which the reference is not
* automatically of the default operand size.
*/
.code16
cli
movw %cs, %ax
movw %ax, %ds /* load cs into ds */
movw %ax, %ss /* and into ss */
/*
* Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
*/
movw $CPUHALTCODEOFF, %ax
jmp *%ax
#endif /* !__GNUC_AS__ */
.globl real_mode_stop_cpu_stage1_end
real_mode_stop_cpu_stage1_end:
nop
SET_SIZE(real_mode_stop_cpu_stage1)
#elif defined(__i386)
ENTRY_NP(real_mode_stop_cpu_stage1)
#if !defined(__GNUC_AS__)
cli
D16 movw %cs, %eax
movw %eax, %ds /* load cs into ds */
movw %eax, %ss /* and into ss */
/*
* Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
*/
movw $CPUHALTCODEOFF, %ax
.byte 0xff, 0xe0 /* jmp *%ax */
#else /* __GNUC_AS__ */
cli
mov %cs, %ax
mov %eax, %ds /* load cs into ds */
mov %eax, %ss /* and into ss */
/*
* Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
*/
movw $CPUHALTCODEOFF, %ax
jmp *%ax
#endif /* !__GNUC_AS__ */
.globl real_mode_stop_cpu_stage1_end
real_mode_stop_cpu_stage1_end:
nop
SET_SIZE(real_mode_stop_cpu_stage1)
#endif /* __amd64 */
ENTRY_NP(real_mode_stop_cpu_stage2)
movw $0xdead, %ax
movw %ax, CPUHALTEDOFF
real_mode_stop_cpu_loop:
/*
* Put CPU into halted state.
* Only INIT, SMI, NMI could break the loop.
*/
hlt
jmp real_mode_stop_cpu_loop
.globl real_mode_stop_cpu_stage2_end
real_mode_stop_cpu_stage2_end:
nop
SET_SIZE(real_mode_stop_cpu_stage2)
#endif /* __lint */