/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
*/
/*
* Process switching routines.
*/
#if !defined(lint)
#include "assym.h"
#else /* lint */
#include <sys/thread.h>
#endif /* lint */
#include <sys/param.h>
#include <sys/asm_linkage.h>
#include <sys/mmu.h>
#include <sys/pcb.h>
#include <sys/machthread.h>
#include <sys/machclock.h>
#include <sys/privregs.h>
#include <sys/vtrace.h>
#include <vm/hat_sfmmu.h>
/*
* resume(kthread_id_t)
*
* a thread can only run on one processor at a time. there
* exists a window on MPs where the current thread on one
* processor is capable of being dispatched by another processor.
* some overlap between outgoing and incoming threads can happen
* when they are the same thread. in this case where the threads
* are the same, resume() on one processor will spin on the incoming
* thread until resume() on the other processor has finished with
* the outgoing thread.
*
* The MMU context changes when the resuming thread resides in a different
* process. Kernel threads are known by resume to reside in process 0.
* The MMU context, therefore, only changes when resuming a thread in
* a process different from curproc.
*
* resume_from_intr() is called when the thread being resumed was not
* passivated by resume (e.g. was interrupted). This means that the
* resume lock is already held and that a restore context is not needed.
* Also, the MMU context is not changed on the resume in this case.
*
* resume_from_zombie() is the same as resume except the calling thread
* is a zombie and must be put on the deathrow list after the CPU is
* off the stack.
*/
#if defined(lint)
/* ARGSUSED */
void
resume(kthread_id_t t)
{}
#else /* lint */
ENTRY(resume)
save %sp, -SA(MINFRAME), %sp ! save ins and locals
call __dtrace_probe___sched_off__cpu ! DTrace probe
mov %i0, %o0 ! arg for DTrace probe
membar #Sync ! flush writebuffers
flushw ! flushes all but this window
stn %i7, [THREAD_REG + T_PC] ! save return address
stn %fp, [THREAD_REG + T_SP] ! save sp
!
! Save GSR (Graphics Status Register).
!
! Read fprs, call fp_save if FPRS_FEF set.
! This handles floating-point state saving.
! The fprs could be turned on by hw bcopy software,
! *or* by fp_disabled. Handle it either way.
!
ldn [THREAD_REG + T_LWP], %o4 ! get lwp pointer
rd %fprs, %g4 ! read fprs
brnz,pt %o4, 0f ! if user thread skip
ldn [THREAD_REG + T_CPU], %i1 ! get CPU pointer
!
! kernel thread
!
! we save fprs at the beginning the stack so we know
! where to check at resume time
ldn [THREAD_REG + T_STACK], %i2
ldn [THREAD_REG + T_CTX], %g3 ! get ctx pointer
andcc %g4, FPRS_FEF, %g0 ! is FPRS_FEF set
bz,pt %icc, 1f ! nope, skip
st %g4, [%i2 + SA(MINFRAME) + FPU_FPRS] ! save fprs
! save kernel fp state in stack
add %i2, SA(MINFRAME), %o0 ! o0 = kfpu_t ptr
rd %gsr, %g5
call fp_save
stx %g5, [%o0 + FPU_GSR] ! store GSR
ba,a,pt %icc, 1f
nop
0:
! user thread
! o4 = lwp ptr
! g4 = fprs
! i1 = CPU ptr
ldn [%o4 + LWP_FPU], %o0 ! fp pointer
stn %fp, [THREAD_REG + T_SP] ! save sp
andcc %g4, FPRS_FEF, %g0 ! is FPRS_FEF set
st %g4, [%o0 + FPU_FPRS] ! store FPRS
#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
sethi %hi(fpu_exists), %g5
ld [%g5 + %lo(fpu_exists)], %g5
brz,pn %g5, 1f
ldn [THREAD_REG + T_CTX], %g3 ! get ctx pointer
#endif
bz,pt %icc, 1f ! most apps don't use fp
ldn [THREAD_REG + T_CTX], %g3 ! get ctx pointer
ldn [%o4 + LWP_FPU], %o0 ! fp pointer
rd %gsr, %g5
call fp_save ! doesn't touch globals
stx %g5, [%o0 + FPU_GSR] ! store GSR
1:
!
! Perform context switch callback if set.
! This handles coprocessor state saving.
! i1 = cpu ptr
! g3 = ctx pointer
!
wr %g0, %g0, %fprs ! disable fpu and clear fprs
brz,pt %g3, 2f ! skip call when zero
ldn [%i0 + T_PROCP], %i3 ! delay slot - get proc pointer
call savectx
mov THREAD_REG, %o0 ! delay - arg = thread pointer
2:
ldn [THREAD_REG + T_PROCP], %i2 ! load old curproc - for mmu
!
! Temporarily switch to idle thread's stack
!
ldn [%i1 + CPU_IDLE_THREAD], %o0 ! idle thread pointer
ldn [%o0 + T_SP], %o1 ! get onto idle thread stack
sub %o1, SA(MINFRAME), %sp ! save room for ins and locals
clr %fp
!
! Set the idle thread as the current thread
!
mov THREAD_REG, %l3 ! save %g7 (current thread)
mov %o0, THREAD_REG ! set %g7 to idle
stn %o0, [%i1 + CPU_THREAD] ! set CPU's thread to idle
!
! Clear and unlock previous thread's t_lock
! to allow it to be dispatched by another processor.
!
clrb [%l3 + T_LOCK] ! clear tp->t_lock
!
! IMPORTANT: Registers at this point must be:
! %i0 = new thread
! %i1 = cpu pointer
! %i2 = old proc pointer
! %i3 = new proc pointer
!
! Here we are in the idle thread, have dropped the old thread.
!
ALTENTRY(_resume_from_idle)
! SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3)
SET_KCONTEXTREG(%o0, %g1, %g2, %g3, %o3, l1, l2, l3)
cmp %i2, %i3 ! resuming the same process?
be,pt %xcc, 5f ! yes.
nop
ldx [%i3 + P_AS], %o0 ! load p->p_as
ldx [%o0 + A_HAT], %i5 ! %i5 = new proc hat
!
! update cpusran field
!
ld [%i1 + CPU_ID], %o4
add %i5, SFMMU_CPUSRAN, %o5
CPU_INDEXTOSET(%o5, %o4, %g1)
ldx [%o5], %o2 ! %o2 = cpusran field
mov 1, %g2
sllx %g2, %o4, %o4 ! %o4 = bit for this cpu
andcc %o4, %o2, %g0
bnz,pn %xcc, 0f ! bit already set, go to 0
nop
3:
or %o2, %o4, %o1 ! or in this cpu's bit mask
casx [%o5], %o2, %o1
cmp %o2, %o1
bne,a,pn %xcc, 3b
ldx [%o5], %o2 ! o2 = cpusran field
membar #LoadLoad|#StoreLoad
0:
!
! disable interrupts
!
! if resume from user to kernel thread
! call sfmmu_setctx_sec
! if resume from kernel (or a different user) thread to user thread
! call sfmmu_alloc_ctx
! sfmmu_load_mmustate
!
! enable interrupts
!
! %i5 = new proc hat
!
sethi %hi(ksfmmup), %o2
ldx [%o2 + %lo(ksfmmup)], %o2
rdpr %pstate, %i4
cmp %i5, %o2 ! new proc hat == ksfmmup ?
bne,pt %xcc, 3f ! new proc is not kernel as, go to 3
wrpr %i4, PSTATE_IE, %pstate
SET_KAS_CTXSEC_ARGS(%i5, %o0, %o1)
! new proc is kernel as
call sfmmu_setctx_sec ! switch to kernel context
or %o0, %o1, %o0
ba,a,pt %icc, 4f
!
! Switch to user address space.
!
3:
mov %i5, %o0 ! %o0 = sfmmup
mov %i1, %o2 ! %o2 = CPU
set SFMMU_PRIVATE, %o3 ! %o3 = sfmmu private flag
call sfmmu_alloc_ctx
mov %g0, %o1 ! %o1 = allocate flag = 0
brz,a,pt %o0, 4f ! %o0 == 0, no private alloc'ed
nop
ldn [%i5 + SFMMU_SCDP], %o0 ! using shared contexts?
brz,a,pt %o0, 4f
nop
ldn [%o0 + SCD_SFMMUP], %o0 ! %o0 = scdp->scd_sfmmup
mov %i1, %o2 ! %o2 = CPU
set SFMMU_SHARED, %o3 ! %o3 = sfmmu shared flag
call sfmmu_alloc_ctx
mov 1, %o1 ! %o1 = allocate flag = 1
4:
call sfmmu_load_mmustate ! program MMU registers
mov %i5, %o0
wrpr %g0, %i4, %pstate ! enable interrupts
5:
!
! spin until dispatched thread's mutex has
! been unlocked. this mutex is unlocked when
! it becomes safe for the thread to run.
!
ldstub [%i0 + T_LOCK], %o0 ! lock curthread's t_lock
6:
brnz,pn %o0, 7f ! lock failed
ldx [%i0 + T_PC], %i7 ! delay - restore resuming thread's pc
!
! Fix CPU structure to indicate new running thread.
! Set pointer in new thread to the CPU structure.
! XXX - Move migration statistic out of here
!
ldx [%i0 + T_CPU], %g2 ! last CPU to run the new thread
cmp %g2, %i1 ! test for migration
be,pt %xcc, 4f ! no migration
ldn [%i0 + T_LWP], %o1 ! delay - get associated lwp (if any)
ldx [%i1 + CPU_STATS_SYS_CPUMIGRATE], %g2
inc %g2
stx %g2, [%i1 + CPU_STATS_SYS_CPUMIGRATE]
stx %i1, [%i0 + T_CPU] ! set new thread's CPU pointer
4:
stx %i0, [%i1 + CPU_THREAD] ! set CPU's thread pointer
membar #StoreLoad ! synchronize with mutex_exit()
mov %i0, THREAD_REG ! update global thread register
stx %o1, [%i1 + CPU_LWP] ! set CPU's lwp ptr
brz,a,pn %o1, 1f ! if no lwp, branch and clr mpcb
stx %g0, [%i1 + CPU_MPCB]
!
! user thread
! o1 = lwp
! i0 = new thread
!
ldx [%i0 + T_STACK], %o0
stx %o0, [%i1 + CPU_MPCB] ! set CPU's mpcb pointer
#ifdef CPU_MPCB_PA
ldx [%o0 + MPCB_PA], %o0
stx %o0, [%i1 + CPU_MPCB_PA]
#endif
! Switch to new thread's stack
ldx [%i0 + T_SP], %o0 ! restore resuming thread's sp
sub %o0, SA(MINFRAME), %sp ! in case of intr or trap before restore
mov %o0, %fp
!
! Restore resuming thread's GSR reg and floating-point regs
! Note that the ld to the gsr register ensures that the loading of
! the floating point saved state has completed without necessity
! of a membar #Sync.
!
#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
sethi %hi(fpu_exists), %g3
ld [%g3 + %lo(fpu_exists)], %g3
brz,pn %g3, 2f
ldx [%i0 + T_CTX], %i5 ! should resumed thread restorectx?
#endif
ldx [%o1 + LWP_FPU], %o0 ! fp pointer
ld [%o0 + FPU_FPRS], %g5 ! get fpu_fprs
andcc %g5, FPRS_FEF, %g0 ! is FPRS_FEF set?
bz,a,pt %icc, 9f ! no, skip fp_restore
wr %g0, FPRS_FEF, %fprs ! enable fprs so fp_zero works
ldx [THREAD_REG + T_CPU], %o4 ! cpu pointer
call fp_restore
wr %g5, %g0, %fprs ! enable fpu and restore fprs
ldx [%o0 + FPU_GSR], %g5 ! load saved GSR data
wr %g5, %g0, %gsr ! restore %gsr data
ba,pt %icc,2f
ldx [%i0 + T_CTX], %i5 ! should resumed thread restorectx?
9:
!
! Zero resuming thread's fp registers, for *all* non-fp program
! Remove all possibility of using the fp regs as a "covert channel".
!
call fp_zero
wr %g0, %g0, %gsr
ldx [%i0 + T_CTX], %i5 ! should resumed thread restorectx?
ba,pt %icc, 2f
wr %g0, %g0, %fprs ! disable fprs
1:
#ifdef CPU_MPCB_PA
mov -1, %o1
stx %o1, [%i1 + CPU_MPCB_PA]
#endif
!
! kernel thread
! i0 = new thread
!
! Switch to new thread's stack
!
ldx [%i0 + T_SP], %o0 ! restore resuming thread's sp
sub %o0, SA(MINFRAME), %sp ! in case of intr or trap before restore
mov %o0, %fp
!
! Restore resuming thread's GSR reg and floating-point regs
! Note that the ld to the gsr register ensures that the loading of
! the floating point saved state has completed without necessity
! of a membar #Sync.
!
ldx [%i0 + T_STACK], %o0
ld [%o0 + SA(MINFRAME) + FPU_FPRS], %g5 ! load fprs
ldx [%i0 + T_CTX], %i5 ! should thread restorectx?
andcc %g5, FPRS_FEF, %g0 ! did we save fp in stack?
bz,a,pt %icc, 2f
wr %g0, %g0, %fprs ! clr fprs
wr %g5, %g0, %fprs ! enable fpu and restore fprs
call fp_restore
add %o0, SA(MINFRAME), %o0 ! o0 = kpu_t ptr
ldx [%o0 + FPU_GSR], %g5 ! load saved GSR data
wr %g5, %g0, %gsr ! restore %gsr data
2:
!
! Restore resuming thread's context
! i5 = ctx ptr
!
brz,a,pt %i5, 8f ! skip restorectx() when zero
ld [%i1 + CPU_BASE_SPL], %o0
call restorectx ! thread can not sleep on temp stack
mov THREAD_REG, %o0 ! delay slot - arg = thread pointer
!
! Set priority as low as possible, blocking all interrupt threads
! that may be active.
!
ld [%i1 + CPU_BASE_SPL], %o0
8:
wrpr %o0, 0, %pil
wrpr %g0, WSTATE_KERN, %wstate
!
! If we are resuming an interrupt thread, store a starting timestamp
! in the thread structure.
!
lduh [THREAD_REG + T_FLAGS], %o0
andcc %o0, T_INTR_THREAD, %g0
bnz,pn %xcc, 0f
nop
5:
call __dtrace_probe___sched_on__cpu ! DTrace probe
nop
ret ! resume curthread
restore
0:
add THREAD_REG, T_INTR_START, %o2
1:
ldx [%o2], %o1
RD_CLOCK_TICK(%o0,%o3,%g5,__LINE__)
casx [%o2], %o1, %o0
cmp %o0, %o1
be,pt %xcc, 5b
nop
! If an interrupt occurred while we were attempting to store
! the timestamp, try again.
ba,pt %xcc, 1b
nop
!
! lock failed - spin with regular load to avoid cache-thrashing.
!
7:
brnz,a,pt %o0, 7b ! spin while locked
ldub [%i0 + T_LOCK], %o0
ba %xcc, 6b
ldstub [%i0 + T_LOCK], %o0 ! delay - lock curthread's mutex
SET_SIZE(_resume_from_idle)
SET_SIZE(resume)
#endif /* lint */
#if defined(lint)
/* ARGSUSED */
void
resume_from_zombie(kthread_id_t t)
{}
#else /* lint */
ENTRY(resume_from_zombie)
save %sp, -SA(MINFRAME), %sp ! save ins and locals
call __dtrace_probe___sched_off__cpu ! DTrace probe
mov %i0, %o0 ! arg for DTrace probe
ldn [THREAD_REG + T_CPU], %i1 ! cpu pointer
flushw ! flushes all but this window
ldn [THREAD_REG + T_PROCP], %i2 ! old procp for mmu ctx
!
! Temporarily switch to the idle thread's stack so that
! the zombie thread's stack can be reclaimed by the reaper.
!
ldn [%i1 + CPU_IDLE_THREAD], %o2 ! idle thread pointer
ldn [%o2 + T_SP], %o1 ! get onto idle thread stack
sub %o1, SA(MINFRAME), %sp ! save room for ins and locals
clr %fp
!
! Set the idle thread as the current thread.
! Put the zombie on death-row.
!
mov THREAD_REG, %o0 ! save %g7 = curthread for arg
mov %o2, THREAD_REG ! set %g7 to idle
stn %g0, [%i1 + CPU_MPCB] ! clear mpcb
#ifdef CPU_MPCB_PA
mov -1, %o1
stx %o1, [%i1 + CPU_MPCB_PA]
#endif
call reapq_add ! reapq_add(old_thread);
stn %o2, [%i1 + CPU_THREAD] ! delay - CPU's thread = idle
!
! resume_from_idle args:
! %i0 = new thread
! %i1 = cpu
! %i2 = old proc
! %i3 = new proc
!
b _resume_from_idle ! finish job of resume
ldn [%i0 + T_PROCP], %i3 ! new process
SET_SIZE(resume_from_zombie)
#endif /* lint */
#if defined(lint)
/* ARGSUSED */
void
resume_from_intr(kthread_id_t t)
{}
#else /* lint */
ENTRY(resume_from_intr)
save %sp, -SA(MINFRAME), %sp ! save ins and locals
!
! We read in the fprs and call fp_save if FPRS_FEF is set
! to save the floating-point state if fprs has been
! modified by operations such as hw bcopy or fp_disabled.
! This is to resolve an issue where an interrupting thread
! doesn't retain their floating-point registers when
! switching out of the interrupt context.
!
rd %fprs, %g4
ldn [THREAD_REG + T_STACK], %i2
andcc %g4, FPRS_FEF, %g0 ! is FPRS_FEF set
bz,pt %icc, 4f
st %g4, [%i2 + SA(MINFRAME) + FPU_FPRS] ! save fprs
! save kernel fp state in stack
add %i2, SA(MINFRAME), %o0 ! %o0 = kfpu_t ptr
rd %gsr, %g5
call fp_save
stx %g5, [%o0 + FPU_GSR] ! store GSR
4:
flushw ! flushes all but this window
stn %fp, [THREAD_REG + T_SP] ! delay - save sp
stn %i7, [THREAD_REG + T_PC] ! save return address
ldn [%i0 + T_PC], %i7 ! restore resuming thread's pc
ldn [THREAD_REG + T_CPU], %i1 ! cpu pointer
!
! Fix CPU structure to indicate new running thread.
! The pinned thread we're resuming already has the CPU pointer set.
!
mov THREAD_REG, %l3 ! save old thread
stn %i0, [%i1 + CPU_THREAD] ! set CPU's thread pointer
membar #StoreLoad ! synchronize with mutex_exit()
mov %i0, THREAD_REG ! update global thread register
!
! Switch to new thread's stack
!
ldn [THREAD_REG + T_SP], %o0 ! restore resuming thread's sp
sub %o0, SA(MINFRAME), %sp ! in case of intr or trap before restore
mov %o0, %fp
clrb [%l3 + T_LOCK] ! clear intr thread's tp->t_lock
!
! If we are resuming an interrupt thread, store a timestamp in the
! thread structure.
!
lduh [THREAD_REG + T_FLAGS], %o0
andcc %o0, T_INTR_THREAD, %g0
bnz,pn %xcc, 0f
!
! We're resuming a non-interrupt thread.
! Clear CPU_INTRCNT and check if cpu_kprunrun set?
!
ldub [%i1 + CPU_KPRUNRUN], %o5 ! delay
brnz,pn %o5, 3f ! call kpreempt(KPREEMPT_SYNC);
stub %g0, [%i1 + CPU_INTRCNT]
1:
ret ! resume curthread
restore
0:
!
! We're an interrupt thread. Update t_intr_start and cpu_intrcnt
!
add THREAD_REG, T_INTR_START, %o2
2:
ldx [%o2], %o1
RD_CLOCK_TICK(%o0,%o3,%l1,__LINE__)
casx [%o2], %o1, %o0
cmp %o0, %o1
bne,pn %xcc, 2b
ldn [THREAD_REG + T_INTR], %l1 ! delay
! Reset cpu_intrcnt if we aren't pinning anyone
brz,a,pt %l1, 2f
stub %g0, [%i1 + CPU_INTRCNT]
2:
ba,pt %xcc, 1b
nop
3:
!
! We're a non-interrupt thread and cpu_kprunrun is set. call kpreempt.
!
call kpreempt
mov KPREEMPT_SYNC, %o0
ba,pt %xcc, 1b
nop
SET_SIZE(resume_from_intr)
#endif /* lint */
/*
* thread_start()
*
* the current register window was crafted by thread_run() to contain
* an address of a procedure (in register %i7), and its args in registers
* %i0 through %i5. a stack trace of this thread will show the procedure
* that thread_start() invoked at the bottom of the stack. an exit routine
* is stored in %l0 and called when started thread returns from its called
* procedure.
*/
#if defined(lint)
void
thread_start(void)
{}
#else /* lint */
ENTRY(thread_start)
mov %i0, %o0
jmpl %i7, %o7 ! call thread_run()'s start() procedure.
mov %i1, %o1
call thread_exit ! destroy thread if it returns.
nop
unimp 0
SET_SIZE(thread_start)
#endif /* lint */