locore.s revision ae115bc77f6fcde83175c75b4206dc2e50747966
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
/* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
/* All Rights Reserved */
/* Copyright (c) 1987, 1988 Microsoft Corporation */
/* All Rights Reserved */
#pragma ident "%Z%%M% %I% %E% SMI"
#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/privregs.h>
#include <sys/x86_archext.h>
#include <sys/machparam.h>
#if defined(__lint)
#include <sys/bootconf.h>
#include <sys/bootsvcs.h>
#else /* __lint */
#include <sys/segments.h>
#include <sys/traptrace.h>
#include "assym.h"
/*
* Our assumptions:
* - We are running in protected-paged mode.
* - Interrupts are disabled.
* - The GDT and IDT are the callers; we need our copies.
* - The kernel's text, initialized data and bss are mapped.
*
* Our actions:
* - Save arguments
* - Initialize our stack pointer to the thread 0 stack (t0stack)
* and leave room for a phony "struct regs".
* - Our GDT and IDT need to get munged.
* - Since we are using the boot's GDT descriptors, we need
* to copy them into our GDT before we switch to ours.
* - We start using our GDT by loading correct values in the
* selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
* gs=KGS_SEL).
* - The default LDT entry for syscall is set.
* - We load the default LDT into the hardware LDT register.
* - We load the default TSS into the hardware task register.
* - Check for cpu type, i.e. 486 vs. P5 vs. P6 etc.
* - mlsetup(%esp) gets called.
* - We change our appearance to look like the real thread 0.
* (NOTE: making ourselves to be a real thread may be a noop)
* - main() gets called. (NOTE: main() never returns).
*
* NOW, the real code!
*/
/*
* The very first thing in the kernel's text segment must be a jump
*/
.text
/*
* Globals:
*/
/*
* call back into boot - sysp (bootsvcs.h) and bootops (bootconf.h)
*/
/*
* NOTE: t0stack should be the first thing in the data section so that
* if it ever overflows, it will fault on the last kernel text page.
*/
.data
#ifdef TRAPTRACE
/*
* trap_trace_ctl must agree with the structure definition in
* <sys/traptrace.h>
*/
.long 0
.long 0
.NWORD 0 /* pad */
/*
* Enough padding for 31 more CPUs (no .skip on x86 -- grrrr).
*/
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0
#if defined(__amd64)
#if NCPU != 64
#error "NCPU != 64, Expand padding for trap_trace_ctl"
#endif
/*
* Enough padding for 32 more CPUs (no .skip on x86 -- grrrr).
*/
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
.NWORD 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
#else /* __amd64 */
#if NCPU != 32
#error "NCPU != 32, Expand padding for trap_trace_ctl"
#endif
#endif /* __amd64 */
/*
* CPU 0's preallocated TRAPTRACE buffer.
*/
/*
* A dummy TRAPTRACE entry to use after death.
*/
#endif /* TRAPTRACE */
#endif /* __lint */
#if defined(__amd64)
#if defined(__lint)
/* ARGSUSED */
void
{}
#else /* __lint */
/*
* kobj_init() vectors us back to here with (note) a slightly different
* set of arguments than _start is given (see lint prototypes above).
*/
/*
* %rdi = boot services (should die someday)
* %rdx = bootops
* end
*/
/*
* Initialize our stack pointer to the thread 0 stack (t0stack)
* and leave room for a "struct regs" for lwp0. Note that the
* stack doesn't actually align to a 16-byte boundary until just
* before we call mlsetup because we want to use %rsp to point at
* our regs structure.
*/
#if (REGSIZE & 15) == 0
#endif
/*
* Save call back for special x86 boot services vector
*/
/*
* Save arguments and flags, if only for debugging ..
*/
/*
* Enable write protect and alignment check faults.
*/
/*
* (We just assert this works by virtue of being here)
*/
/*
* mlsetup() gets called with a struct regs as argument, while
* main takes no args and should never return.
*/
/* (stack pointer now aligned on 16-byte boundary right here) */
/* NOTREACHED */
#endif /* __amd64 */
#endif /* __lint */
#if !defined(__lint)
.string "main() returned"
.string "486 style cpu detected - no longer supported!"
#endif /* !__lint */
#if !defined(__amd64)
#if defined(__lint)
/* ARGSUSED */
void
{}
#else /* __lint */
/*
* kobj_init() vectors us back to here with (note) a slightly different
* set of arguments than _start is given (see lint prototypes above).
*/
/*
* %ecx = boot services (should die someday)
* %ebx = bootops
*/
/*
* Initialize our stack pointer to the thread 0 stack (t0stack)
* and leave room for a phony "struct regs".
*/
/*
* Save call back for special x86 boot services vector
*/
/*
* Save all registers and flags
*/
/*
* Override bios settings and enable write protect and
* alignment check faults.
*/
/*
* enable WP for detecting faults, and enable alignment checking.
*/
/*
* If bit 21 of eflags can be flipped, then cpuid is present
* and enabled.
*/
/*
* cpuid may be disabled on Cyrix, try to detect Cyrix by the 5/2 test
* div does not modify the cc flags on Cyrix, even though this may
* also be true for other vendors, this is generally true only for
* newer models from those vendors that support and do not disable
* cpuid (usually because cpuid cannot be disabled)
*/
/*
* clear cc flags
*/
/*
* perform 5/2 test
*/
/*
* div did not modify the cc flags, chances are the vendor is Cyrix
* assume the vendor is Cyrix and use the CCR's to enable cpuid
*/
/*
* even if the cpu vendor is Cyrix and the motherboard/chipset
* vendor decided to ignore lines A1-A4 for I/O addresses, I/O port
* 0x21 corresponds with 0x23 and since 0x22 is still untouched,
* the reads and writes of 0x21 are guaranteed to be off-chip of
* the cpu
*/
/*
* enable read of ISR at I/O port 0x20
*/
/*
* read IMR and store in %bl
*/
/*
* mask out all interrupts so that ISR will not change
*/
/*
* reads of I/O port 0x22 on Cyrix are always directed off-chip
* make use of I/O pull-up to test for an unknown device on 0x22
*/
/*
* motherboard/chipset vendor may be ignoring line A1 of I/O address
*/
/*
* if the ISR and the value read from 0x22 do not match then we have
* detected some unknown device, probably a chipset, at 0x22
*/
/*
* now test to see if some unknown device is using I/O port 0x23
*
* read the external I/O port at 0x23
*/
/*
* Test for pull-up at 0x23 or if I/O address line A1 is being ignored.
* IMR is 0xff so both tests are performed simultaneously.
*/
/*
* We are a Cyrix part. In case we are some model of Cx486 or a Cx586,
* record the type and fix it later if not.
*/
/*
* Try to read CCR3. All Cyrix cpu's which support cpuid have CCR3.
*
* load CCR3 index into CCR index register
*/
/*
* If we are not a Cyrix cpu, then we have performed an external I/O
* cycle. If the CCR index was not valid for this Cyrix model, we may
* have performed an external I/O cycle as well. In these cases and
* if the motherboard/chipset vendor ignores I/O address line A1,
* then the PIC will have IRQ3 set at the lowest priority as a side
* effect of the above outb. We are reasonalbly confident that there
* is not an unknown device on I/O port 0x22, so there should have been
* no unpredictable side-effect of the above outb.
*/
/*
* read CCR3
*/
/*
* If we are not a Cyrix cpu the inb above produced an external I/O
* cycle. If we are a Cyrix model that does not support CCR3 wex
* produced an external I/O cycle. In all known Cyrix models 6x86 and
* above, bit 3 of CCR3 is reserved and cannot be set to 1. In all
* Cyrix models prior to the 6x86 that supported CCR3, bits 4-7 are
* reserved as well. It is highly unlikely that CCR3 contains the value
* 0xff. We test to see if I/O port 0x23 is pull-up or the IMR and
* deduce we are not a Cyrix with support for cpuid if so.
*/
/*
* There exist 486 ISA Cyrix chips that support CCR3 but do not support
* DIR0 and DIR1. If we try to read DIR0, we may generate external I/O
* cycles, the exact behavior is model specific and undocumented.
* Unfortunately these external I/O cycles may confuse some PIC's beyond
* recovery. Fortunatetly we can use the following undocumented trick:
* if bit 4 of CCR3 can be toggled, then DIR0 and DIR1 are supported.
* Pleasantly MAPEN contains bit 4 of CCR3, so this trick is guaranteed
* to work on all Cyrix cpu's which support cpuid.
*/
/*
* write back CRR3 with toggled bit 4 to CCR3
*/
/*
* read CCR3
*/
/*
* restore CCR3
*/
/*
* if bit 4 was not toggled DIR0 and DIR1 are not supported in which
* case we do not have cpuid anyway
*/
/*
* read DIR0
*/
/*
* test for pull-up
*/
/*
* Values of 0x20-0x27 in DIR0 are currently reserved by Cyrix for
* future use. If Cyrix ever produces a cpu that supports cpuid with
* these ids, the following test will have to change. For now we remain
* pessimistic since the formats of the CRR's may be different then.
*
* test for at least a 6x86, to see if we support both MAPEN and CPUID
*/
/*
* enable MAPEN
*/
/*
* select CCR4
*/
/*
* read CCR4
*/
/*
* enable cpuid
*/
/*
* select CCR4
*/
/*
* write CCR4
*/
/*
* select CCR3
*/
/*
* disable MAPEN and write CCR3
*/
/*
* restore IMR
*/
/*
* test to see if cpuid available
*/
/*
* In case the motherboard/chipset vendor is ignoring line A1 of the
* I/O address, we set the PIC priorities to sane values.
*/
/*
* cpuid instruction present
*/
/*
* early cyrix cpus are somewhat strange and need to be
* probed in curious ways to determine their identity
*/
/*
* test to see if a useful cpuid
*/
/* AMD Software Optimization Guide - Section 6.2 */
/*
* lose the return address
*/
/*
* Processor signature and feature flags for Cyrix are insane.
* BIOS can play with semi-documented registers, so cpuid must be used
* cautiously. Since we are Cyrix that has cpuid, we have DIR0 and DIR1
* Keep the family in %ebx and feature flags in %edx until not needed
*/
/*
* read DIR0
*/
/*
* First we handle the cases where we are a 6x86 or 6x86L.
* The 6x86 is basically a 486, the only reliable bit in the
* feature flags is for FPU. The 6x86L is better, unfortunately
* there is no really good way to distinguish between these two
* cpu's. We are pessimistic and when in doubt assume 6x86.
*/
/*
* We are an M1, either a 6x86 or 6x86L.
*/
/*
* although it is possible that we are a 6x86L, the cpu and
* documentation are so buggy, we just do not care.
*/
/*
* read DIR1
*/
/*
* We are a 6x86L, or at least a 6x86 with honest cpuid feature flags
*/
/*
* We are likely a 6x86, or a 6x86L without a way of knowing
*
* The 6x86 has NO Pentium or Pentium Pro compatible features even
* though it claims to be a Pentium Pro compatible!
*
* The 6x86 core used in the 6x86 may have most of the Pentium system
* registers and largely conform to the Pentium System Programming
* Reference. Documentation on these parts is long gone. Treat it as
* a crippled Pentium and hope for the best.
*/
/*
* Now we check whether we are a MediaGX or GXm. We have particular
* reason for concern here. Even though most of the GXm's
* report having TSC in the cpuid feature flags, the TSC may be
* horribly broken. What is worse, is that MediaGX's are basically
* 486's while the good GXm's are more like Pentium Pro's!
*/
/*
* We are either a MediaGX (sometimes called a Gx86) or GXm
*/
/*
* We do not honestly know what we are, so assume a MediaGX
*/
/*
* It is still possible we are either a MediaGX or GXm, trust cpuid
* family should be 5 on a GXm
*/
/*
*/
GXm:
/*
* read DIR1
*/
/*
* we are a MediaGX for which we do not trust cpuid
*/
/*
* Now we check whether we are a 6x86MX or MII. These cpu's are
* virtually identical, but we care because for the 6x86MX, we
* must work around the coma bug. Also for 6x86MX prior to revision
* 1.4, the TSC may have serious bugs.
*/
/*
*/
/*
* read DIR1
*/
/*
* It is altogether unclear how the revision stamped on the cpu
* maps to the values in DIR0 and DIR1. Just assume TSC is broken.
*/
MII:
/*
* We are some chip that we cannot identify yet, an MIII perhaps.
* We will be optimistic and hope that the chip is much like an MII,
* and that cpuid is sane. Cyrix seemed to have gotten it right in
* time for the MII, we can only hope it stayed that way.
* Maybe the BIOS or Cyrix is trying to hint at something
*/
/*
* Just test for the features Cyrix is known for
*/
/*
* DIR0 with values from 0x80 to 0x8f indicates a VIA Cyrix III, aka
* the Cyrix MIII. There may be parts later that use the same ranges
* for DIR0 with special values in DIR1, maybe the VIA CIII, but for
* now we will call anything with a DIR0 of 0x80 or higher an MIII.
* The MIII is supposed to support large pages, but we will believe
* it when we see it. For now we just enable and test for MII features.
*/
/*
* With NO_LOCK set to 0 in CCR1, the usual state that BIOS enforces, some
* bus cycles are issued with LOCK# asserted. With NO_LOCK set to 1, all bus
* cycles except page table accesses and interrupt ACK cycles do not assert
* LOCK#. xchgl is an instruction that asserts LOCK# if NO_LOCK is set to 0.
* Due to a bug in the cpu core involving over-optimization of branch
* prediction, register renaming, and execution of instructions down both the
* X and Y pipes for the xchgl instruction, short loops can be written that
* never de-assert LOCK# from one invocation of the loop to the next, ad
* infinitum. The undesirable effect of this situation is that interrupts are
* not serviced. The ideal workaround to this bug would be to set NO_LOCK to
* 1. Unfortunately bus cycles that would otherwise have asserted LOCK# no
* longer do, unless they are page table accesses or interrupt ACK cycles.
* With LOCK# not asserted, these bus cycles are now cached. This can cause
* undesirable behaviour if the ARR's are not configured correctly. Solaris
* does not configure the ARR's, nor does it provide any useful mechanism for
* doing so, thus the ideal workaround is not viable. Fortunately, the only
* known exploits for this bug involve the xchgl instruction specifically.
* There is a group of undocumented registers on Cyrix 6x86, 6x86L, and
* 6x86MX cpu's which can be used to specify one instruction as a serializing
* instruction. With the xchgl instruction serialized, LOCK# is still
* asserted, but it is the sole instruction for which LOCK# is asserted.
* There is now some added penalty for the xchgl instruction, but the usual
* bus locking is preserved. This ingenious workaround was discovered by
* disassembling a binary provided by Cyrix as a workaround for this bug on
* Windows, but its not documented anywhere by Cyrix, nor is the bug actually
* mentioned in any public errata! The only concern for this workaround is
* that there may be similar undiscovered bugs with other instructions that
* assert LOCK# that may be leveraged to similar ends. The fact that Cyrix
* fixed this bug sometime late in 1997 and no other exploits other than
* xchgl have been discovered is good indication that this workaround is
* reasonable.
*/
/*
* What is known about DBR1, DBR2, DBR3, and DOR is that for normal
* cpu execution DBR1, DBR2, and DBR3 are set to 0. To obtain opcode
* serialization, DBR1, DBR2, and DBR3 are loaded with 0xb8, 0x7f,
* and 0xff. Then, DOR is loaded with the one byte opcode.
*/
/*
* select CCR3
*/
/*
* read CCR3 and mask out MAPEN
*/
/*
* save masked CCR3 in %ah
*/
/*
* select CCR3
*/
/*
* enable MAPEN
*/
/*
* read DBR0
*/
/*
* disable MATCH and save in %bh
*/
/*
* write DBR0
*/
/*
* write DBR1
*/
/*
* write DBR2
*/
/*
* write DBR3
*/
/*
* write DOR
*/
/*
* enable MATCH
*/
/*
* disable MAPEN
*/
popfl /* Restore original FLAGS */
popal /* Restore all registers */
/*
* mlsetup(%esp) gets called.
*/
/*
* We change our appearance to look like the real thread 0.
* (NOTE: making ourselves to be a real thread may be a noop)
* main() gets called. (NOTE: main() never returns).
*/
/* NOTREACHED */
/* NOTREACHED */
#endif /* __lint */
#endif /* !__amd64 */
/*
* For stack layout, see privregs.h
* When cmntrap gets called, the error code and trap number have been pushed.
* When cmntrap_pushed gets called, the entire struct regs has been pushed.
*/
#if defined(__lint)
/* ARGSUSED */
void
cmntrap()
{}
#else /* __lint */
#if defined(__amd64)
/*
* - if this is a #pf i.e. T_PGFLT, %r15 is live
* and contains the faulting address i.e. a copy of %cr2
*
* - if this is a #db i.e. T_SGLSTP, %r15 is live
* and contains the value of %db6
*/
/*
* We must first check if DTrace has set its NOFAULT bit. This
* regrettably must happen before the trap stack is recorded, because
* this requires a call to getpcstack() and may induce recursion if an
* fbt::getpcstack: enabling is inducing the bad load.
*/
/*
* We know that this isn't a DTrace non-faulting load; we can now safely
* reenable interrupts. (In the case of pagefaults, we enter through an
* interrupt gate.)
*/
je 0f
/*
* If we've taken a GPF, we don't (unfortunately) have the address that
* induced the fault. So instead of setting the fault to BADADDR,
* we'll set the fault to ILLOP.
*/
jmp 1f
0:
/* fault addr is illegal value */
1:
/*NOTREACHED*/
2:
3:
/*
* - if this is a #pf i.e. T_PGFLT, %esi is live
* and contains the faulting address i.e. a copy of %cr2
*
* - if this is a #db i.e. T_SGLSTP, %esi is live
* and contains the value of %db6
*/
/*
* We must first check if DTrace has set its NOFAULT bit. This
* regrettably must happen before the trap stack is recorded, because
* this requires a call to getpcstack() and may induce recursion if an
* fbt::getpcstack: enabling is inducing the bad load.
*/
/*
* We know that this isn't a DTrace non-faulting load; we can now safely
* reenable interrupts. (In the case of pagefaults, we enter through an
* interrupt gate.)
*/
je 0f
/*
* If we've taken a GPF, we don't (unfortunately) have the address that
* induced the fault. So instead of setting the fault to BADADDR,
* we'll set the fault to ILLOP.
*/
jmp 1f
0:
/* fault addr is illegal value */
1:
2:
3:
#endif /* __i386 */
/*
* Declare a uintptr_t which has the size of _cmntrap to enable stack
* traceback code to know when a regs structure is on the stack.
*/
.string "bad DTrace flags"
.string "bad DTrace trap"
#endif /* __lint */
#if defined(__lint)
/* ARGSUSED */
void
{}
void
bop_trap_handler(void)
{}
#else /* __lint */
#if defined(__amd64)
/*
* Handle traps early in boot. Just revectors into C quickly as
* these are always fatal errors.
*/
pushl $0
/*
* Handle traps early in boot. Just revectors into C quickly as
* these are always fatal errors.
*/
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/* ARGSUSED */
void
{}
#else /* __lint */
#if defined(__amd64)
#endif /* __i386 */
#endif /* __lint */
/*
* Return from _sys_trap routine.
*/
#if defined(__lint)
void
lwp_rtt_initial(void)
{}
void
lwp_rtt(void)
{}
void
_sys_rtt(void)
{}
#else /* __lint */
#if defined(__amd64)
/*
* r14 lwp
* rdx lwp->lwp_procp
* r15 curthread
*/
/*
* XX64 Is the stack misaligned correctly at this point?
* If not, we need to do a push before calling anything ..
*/
#if defined(DEBUG)
/*
* If we were to run lwp_savectx at this point -without-
* RUPDATE_PENDING being set, we'd end up sampling the hardware
* state left by the previous running lwp, rather than setting
* the values requested by the lwp creator. Bad.
*/
jne 1f
1:
#endif
/*
* If agent lwp, clear %fs and %gs
*/
jne 1f
1:
/*
* set up to take fault on first use of fp
*/
/*
* XXX - may want a fast path that avoids sys_rtt_common in the
* most common case.
*/
/*
* Return to user
*/
/*
* Return to 32-bit userland
*/
/*NOTREACHED*/
/*
* Return to 64-bit userland
*/
/*NOTREACHED*/
/*
* Return to supervisor
* NOTE: to make the check in trap() that tests if we are executing
* after _sys_rtt .
*/
/*
* Restore regs before doing iretq to kernel mode
*/
/*NOTREACHED*/
/*
* If agent lwp, clear %fs and %gs.
*/
jne 1f
1:
/*
* set up to take fault on first use of fp
*/
/*
* XXX - may want a fast path that avoids sys_rtt_common in the
* most common case.
*/
/*
* Return to User.
*/
/*
* Return to supervisor
*/
/*
* Restore regs before doing iret to kernel mode
*/
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/*
* So why do we have to deal with all this crud in the world of ia32?
*
* Basically there are four classes of ia32 implementations, those that do not
* have a TSC, those that have a marginal TSC that is broken to the extent
* that it is useless, those that have a marginal TSC that is not quite so
* horribly broken and can be used with some care, and those that have a
* reliable TSC. This crud has to be here in order to sift through all the
* variants.
*/
/*ARGSUSED*/
{
return (0);
}
#else /* __lint */
#if defined(__amd64)
/*
* XX64 quick and dirty port from the i386 version. Since we
* believe the amd64 tsc is more reliable, could this code be
* simpler?
*/
/
/ cause the calculated cpu frequency to vary slightly from boot to boot.
/
/ In all cases even if this constant is set inappropriately, the algorithm
/ will still work and the caller should be able to handle variances in the
/ calculation of cpu frequency, but the calculation will be inefficient and
/ take a disproportionate amount of time relative to a well selected value.
/ As the slowest supported cpu becomes faster, this constant should be
/ carefully increased.
movl $0x8000, %ecx
/ to make sure the instruction cache has been warmed
clc
jmp freq_tsc_loop
/ The following block of code up to and including the latching of the PIT
/ counter after freq_tsc_perf_loop is very critical and very carefully
/ written, it should only be modified with great care. freq_tsc_loop to
/ freq_tsc_perf_loop fits exactly in 16 bytes as do the instructions in
/ freq_tsc_perf_loop up to the unlatching of the PIT counter.
.align 32
freq_tsc_loop:
/ save the loop count in %ebx
movl %ecx, %ebx
/ initialize the PIT counter and start a count down
movb $PIT_LOADMODE, %al
outb $PITCTL_PORT
movb $0xff, %al
outb $PITCTR0_PORT
outb $PITCTR0_PORT
/ read the TSC and store the TS in %edi:%esi
rdtsc
movl %eax, %esi
freq_tsc_perf_loop:
movl %edx, %edi
movl %eax, %esi
movl %edx, %edi
loop freq_tsc_perf_loop
/ read the TSC and store the LSW in %ecx
rdtsc
movl %eax, %ecx
/ latch the PIT counter and status
movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
outb $PITCTL_PORT
/ remember if the icache has been warmed
setc %ah
/ read the PIT status
inb $PITCTR0_PORT
shll $8, %eax
/ read PIT count
inb $PITCTR0_PORT
shll $8, %eax
inb $PITCTR0_PORT
bswap %eax
/ check to see if the PIT count was loaded into the CE
btw $_CONST(PITSTAT_NULLCNT+8), %ax
jc freq_tsc_increase_count
/ check to see if PIT counter wrapped
btw $_CONST(PITSTAT_OUTPUT+8), %ax
jnc freq_tsc_pit_did_not_wrap
/ halve count
shrl $1, %ebx
movl %ebx, %ecx
/ the instruction cache has been warmed
stc
jmp freq_tsc_loop
freq_tsc_increase_count:
shll $1, %ebx
jc freq_tsc_too_fast
movl %ebx, %ecx
/ the instruction cache has been warmed
stc
jmp freq_tsc_loop
freq_tsc_pit_did_not_wrap:
roll $16, %eax
cmpw $0x2000, %ax
notw %ax
jb freq_tsc_sufficient_duration
freq_tsc_calculate:
/ in mode 0, the PIT loads the count into the CE on the first CLK pulse,
/ then on the second CLK pulse the CE is decremented, therefore mode 0
/ is really a (count + 1) counter, ugh
xorl %esi, %esi
movw %ax, %si
incl %esi
movl $0xf000, %eax
mull %ebx
/ tuck away (target_pit_count * loop_count)
movl %edx, %ecx
movl %eax, %ebx
movl %esi, %eax
movl $0xffffffff, %edx
mull %edx
addl %esi, %eax
adcl $0, %edx
cmpl %ecx, %edx
ja freq_tsc_div_safe
jb freq_tsc_too_fast
cmpl %ebx, %eax
jbe freq_tsc_too_fast
freq_tsc_div_safe:
movl %ecx, %edx
movl %ebx, %eax
movl %esi, %ecx
divl %ecx
movl %eax, %ecx
/ the instruction cache has been warmed
stc
jmp freq_tsc_loop
freq_tsc_sufficient_duration:
/ test to see if the icache has been warmed
btl $16, %eax
jnc freq_tsc_calculate
/ recall mode 0 is a (count + 1) counter
andl $0xffff, %eax
incl %eax
/ save the number of PIT counts
movl %eax, (%r9)
/
/ cause the calculated cpu frequency to vary slightly from boot to boot.
/
/ In all cases even if this constant is set inappropriately, the algorithm
/ will still work and the caller should be able to handle variances in the
/ calculation of cpu frequency, but the calculation will be inefficient and
/ take a disproportionate amount of time relative to a well selected value.
/ As the slowest supported cpu becomes faster, this constant should be
/ carefully increased.
movl $0x8000, %ecx
/ to make sure the instruction cache has been warmed
clc
jmp freq_tsc_loop
/ The following block of code up to and including the latching of the PIT
/ counter after freq_tsc_perf_loop is very critical and very carefully
/ written, it should only be modified with great care. freq_tsc_loop to
/ freq_tsc_perf_loop fits exactly in 16 bytes as do the instructions in
/ freq_tsc_perf_loop up to the unlatching of the PIT counter.
.align 32
freq_tsc_loop:
/ save the loop count in %ebx
movl %ecx, %ebx
/ initialize the PIT counter and start a count down
movb $PIT_LOADMODE, %al
outb $PITCTL_PORT
movb $0xff, %al
outb $PITCTR0_PORT
outb $PITCTR0_PORT
/ read the TSC and store the TS in %edi:%esi
rdtsc
movl %eax, %esi
freq_tsc_perf_loop:
movl %edx, %edi
movl %eax, %esi
movl %edx, %edi
loop freq_tsc_perf_loop
/ read the TSC and store the LSW in %ecx
rdtsc
movl %eax, %ecx
/ latch the PIT counter and status
movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
outb $PITCTL_PORT
/ remember if the icache has been warmed
setc %ah
/ read the PIT status
inb $PITCTR0_PORT
shll $8, %eax
/ read PIT count
inb $PITCTR0_PORT
shll $8, %eax
inb $PITCTR0_PORT
bswap %eax
/ check to see if the PIT count was loaded into the CE
btw $_CONST(PITSTAT_NULLCNT+8), %ax
jc freq_tsc_increase_count
/ check to see if PIT counter wrapped
btw $_CONST(PITSTAT_OUTPUT+8), %ax
jnc freq_tsc_pit_did_not_wrap
/ halve count
shrl $1, %ebx
movl %ebx, %ecx
/ the instruction cache has been warmed
stc
jmp freq_tsc_loop
freq_tsc_increase_count:
shll $1, %ebx
jc freq_tsc_too_fast
movl %ebx, %ecx
/ the instruction cache has been warmed
stc
jmp freq_tsc_loop
freq_tsc_pit_did_not_wrap:
roll $16, %eax
cmpw $0x2000, %ax
notw %ax
jb freq_tsc_sufficient_duration
freq_tsc_calculate:
/ in mode 0, the PIT loads the count into the CE on the first CLK pulse,
/ then on the second CLK pulse the CE is decremented, therefore mode 0
/ is really a (count + 1) counter, ugh
xorl %esi, %esi
movw %ax, %si
incl %esi
movl $0xf000, %eax
mull %ebx
/ tuck away (target_pit_count * loop_count)
movl %edx, %ecx
movl %eax, %ebx
movl %esi, %eax
movl $0xffffffff, %edx
mull %edx
addl %esi, %eax
adcl $0, %edx
cmpl %ecx, %edx
ja freq_tsc_div_safe
jb freq_tsc_too_fast
cmpl %ebx, %eax
jbe freq_tsc_too_fast
freq_tsc_div_safe:
movl %ecx, %edx
movl %ebx, %eax
movl %esi, %ecx
divl %ecx
movl %eax, %ecx
/ the instruction cache has been warmed
stc
jmp freq_tsc_loop
freq_tsc_sufficient_duration:
/ test to see if the icache has been warmed
btl $16, %eax
jnc freq_tsc_calculate
/ recall mode 0 is a (count + 1) counter
andl $0xffff, %eax
incl %eax
/ save the number of PIT counts
movl 8(%ebp), %ebx
movl %eax, (%ebx)
#endif /* __i386 */
#endif /* __lint */
#if !defined(__amd64)
#if defined(__lint)
/*
* We do not have a TSC so we use a block of instructions with well known
* timings.
*/
/*ARGSUSED*/
{
return (0);
}
#else /* __lint */
.align 16
/ cpu's this instruction takes more or less clock ticks depending on its
/ arguments.
freq_notsc_perf_loop:
idivl %ebx
idivl %ebx
idivl %ebx
idivl %ebx
idivl %ebx
loop freq_notsc_perf_loop
/ latch the PIT counter and status
movb $_CONST(PIT_READBACK|PIT_READBACKC0), %al
outb $PITCTL_PORT
/ read the PIT status
inb $PITCTR0_PORT
shll $8, %eax
/ read PIT count
inb $PITCTR0_PORT
shll $8, %eax
inb $PITCTR0_PORT
bswap %eax
/ check to see if the PIT count was loaded into the CE
btw $_CONST(PITSTAT_NULLCNT+8), %ax
jc freq_notsc_increase_count
/ check to see if PIT counter wrapped
btw $_CONST(PITSTAT_OUTPUT+8), %ax
jnc freq_notsc_pit_did_not_wrap
/ halve count
shrl $1, %edi
movl %edi, %ecx
jmp freq_notsc_loop
freq_notsc_increase_count:
shll $1, %edi
jc freq_notsc_too_fast
movl %edi, %ecx
jmp freq_notsc_loop
freq_notsc_pit_did_not_wrap:
shrl $16, %eax
cmpw $0x2000, %ax
notw %ax
jb freq_notsc_sufficient_duration
freq_notsc_calculate:
/ in mode 0, the PIT loads the count into the CE on the first CLK pulse,
/ then on the second CLK pulse the CE is decremented, therefore mode 0
/ is really a (count + 1) counter, ugh
xorl %esi, %esi
movw %ax, %si
incl %esi
movl %edi, %eax
movl $0xf000, %ecx
mull %ecx
/ tuck away (target_pit_count * loop_count)
movl %edx, %edi
movl %eax, %ecx
movl %esi, %eax
movl $0xffffffff, %edx
mull %edx
addl %esi, %eax
adcl $0, %edx
cmpl %edi, %edx
ja freq_notsc_div_safe
jb freq_notsc_too_fast
cmpl %ecx, %eax
jbe freq_notsc_too_fast
freq_notsc_div_safe:
movl %edi, %edx
movl %ecx, %eax
movl %esi, %ecx
divl %ecx
movl %eax, %ecx
jmp freq_notsc_loop
freq_notsc_sufficient_duration:
/ recall mode 0 is a (count + 1) counter
incl %eax
/ save the number of PIT counts
movl 8(%ebp), %ebx
movl %eax, (%ebx)
/ calculate the number of cpu clock ticks that elapsed
cmpl $X86_VENDOR_Cyrix, x86_vendor
jz freq_notsc_notcyrix
/ freq_notsc_perf_loop takes 86 clock cycles on Cyrix 6x86 cores
movl $86, %eax
jmp freq_notsc_calculate_tsc
freq_notsc_notcyrix:
/ freq_notsc_perf_loop takes 237 clock cycles on Intel Pentiums
movl $237, %eax
freq_notsc_calculate_tsc:
mull %edi
jmp freq_notsc_end
freq_notsc_too_fast:
/ return 0 as a 64 bit quantity
xorl %eax, %eax
xorl %edx, %edx
freq_notsc_end:
popl %ebx
popl %esi
popl %edi
popl %ebp
ret
SET_SIZE(freq_notsc)
#endif /* __lint */
#endif /* !__amd64 */
#if !defined(__lint)
.data
#if !defined(__amd64)
.align 4
cpu_vendor:
.long 0, 0, 0 /* Vendor ID string returned */
.globl CyrixInstead
.globl x86_feature
.globl x86_type
.globl x86_vendor
#endif
#endif /* __lint */