/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
/* All Rights Reserved */
#include <sys/param.h>
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/cpuvar.h>
#include <sys/var.h>
#include <sys/tuneable.h>
#include <sys/cmn_err.h>
#include <sys/buf.h>
#include <sys/disp.h>
#include <sys/vmsystm.h>
#include <sys/vmparam.h>
#include <sys/class.h>
#include <sys/vtrace.h>
#include <sys/modctl.h>
#include <sys/debug.h>
#include <sys/tnf_probe.h>
#include <sys/procfs.h>
#include <vm/seg.h>
#include <vm/seg_kp.h>
#include <vm/as.h>
#include <vm/rm.h>
#include <vm/seg_kmem.h>
#include <sys/callb.h>
/*
* The swapper sleeps on runout when there is no one to swap in.
* It sleeps on runin when it could not find space to swap someone
* in or after swapping someone in.
*/
char runout;
char runin;
char wake_sched; /* flag tells clock to wake swapper on next tick */
char wake_sched_sec; /* flag tells clock to wake swapper after a second */
/*
* The swapper swaps processes to reduce memory demand and runs
* when avefree < desfree. The swapper resorts to SOFTSWAP when
* avefree < desfree which results in swapping out all processes
* sleeping for more than maxslp seconds. HARDSWAP occurs when the
* system is on the verge of thrashing and this results in swapping
* out runnable threads or threads sleeping for less than maxslp secs.
*
* The swapper runs through all the active processes in the system
* and invokes the scheduling class specific swapin/swapout routine
* for every thread in the process to obtain an effective priority
* for the process. A priority of -1 implies that the thread isn't
* swappable. This effective priority is used to find the most
* eligible process to swapout or swapin.
*
* NOTE: Threads which have been swapped are not linked on any
* queue and their dispatcher lock points at the "swapped_lock".
*
* Processes containing threads with the TS_DONT_SWAP flag set cannot be
* swapped out immediately by the swapper. This is due to the fact that
* such threads may be holding locks which may be needed by the swapper
* to push its pages out. The TS_SWAPENQ flag is set on such threads
* to prevent them running in user mode. When such threads reach a
* safe point (i.e., are not holding any locks - CL_TRAPRET), they
* queue themseleves onto the swap queue which is processed by the
* swapper. This results in reducing memory demand when the system
* is desparate for memory as the thread can't run in user mode.
*
* The swap queue consists of threads, linked via t_link, which are
* haven't been swapped, are runnable but not on the run queue. The
* swap queue is protected by the "swapped_lock". The dispatcher
* lock (t_lockp) of all threads on the swap queue points at the
* "swapped_lock". Thus, the entire queue and/or threads on the
* queue can be locked by acquiring "swapped_lock".
*/
static kthread_t *tswap_queue;
extern disp_lock_t swapped_lock; /* protects swap queue and threads on it */
int maxslp = 0;
pgcnt_t avefree; /* 5 sec moving average of free memory */
pgcnt_t avefree30; /* 30 sec moving average of free memory */
/*
* Minimum size used to decide if sufficient memory is available
* before a process is swapped in. This is necessary since in most
* cases the actual size of a process (p_swrss) being swapped in
* is usually 2 pages (kernel stack pages). This is due to the fact
* almost all user pages of a process are stolen by pageout before
* the swapper decides to swapout it out.
*/
int min_procsize = 12;
static int swapin(proc_t *);
static int swapout(proc_t *, uint_t *, int);
static void process_swap_queue();
#ifdef __sparc
extern void lwp_swapin(kthread_t *);
#endif /* __sparc */
/*
* Counters to keep track of the number of swapins or swapouts.
*/
uint_t tot_swapped_in, tot_swapped_out;
uint_t softswap, hardswap, swapqswap;
/*
* Macro to determine if a process is eligble to be swapped.
*/
#define not_swappable(p) \
(((p)->p_flag & SSYS) || (p)->p_stat == SIDL || \
(p)->p_stat == SZOMB || (p)->p_as == NULL || \
(p)->p_as == &kas)
/*
* Memory scheduler.
*/
void
sched()
{
kthread_id_t t;
pri_t proc_pri;
pri_t thread_pri;
pri_t swapin_pri;
int desperate;
pgcnt_t needs;
int divisor;
proc_t *prp;
proc_t *swapout_prp;
proc_t *swapin_prp;
spgcnt_t avail;
int chosen_pri;
time_t swapout_time;
time_t swapin_proc_time;
callb_cpr_t cprinfo;
kmutex_t swap_cpr_lock;
mutex_init(&swap_cpr_lock, NULL, MUTEX_DEFAULT, NULL);
CALLB_CPR_INIT(&cprinfo, &swap_cpr_lock, callb_generic_cpr, "sched");
if (maxslp == 0)
maxslp = MAXSLP;
loop:
needs = 0;
desperate = 0;
swapin_pri = v.v_nglobpris;
swapin_prp = NULL;
chosen_pri = -1;
process_swap_queue();
/*
* Set desperate if
* 1. At least 2 runnable processes (on average).
* 2. Short (5 sec) and longer (30 sec) average is less
* than minfree and desfree respectively.
* 3. Pagein + pageout rate is excessive.
*/
if (avenrun[0] >= 2 * FSCALE &&
(MAX(avefree, avefree30) < desfree) &&
(pginrate + pgoutrate > maxpgio || avefree < minfree)) {
TRACE_4(TR_FAC_SCHED, TR_DESPERATE,
"desp:avefree: %d, avefree30: %d, freemem: %d"
" pginrate: %d\n", avefree, avefree30, freemem, pginrate);
desperate = 1;
goto unload;
}
/*
* Search list of processes to swapin and swapout deadwood.
*/
swapin_proc_time = 0;
top:
mutex_enter(&pidlock);
for (prp = practive; prp != NULL; prp = prp->p_next) {
if (not_swappable(prp))
continue;
/*
* Look at processes with at least one swapped lwp.
*/
if (prp->p_swapcnt) {
time_t proc_time;
/*
* Higher priority processes are good candidates
* to swapin.
*/
mutex_enter(&prp->p_lock);
proc_pri = -1;
t = prp->p_tlist;
proc_time = 0;
do {
if (t->t_schedflag & TS_LOAD)
continue;
thread_lock(t);
thread_pri = CL_SWAPIN(t, 0);
thread_unlock(t);
if (t->t_stime - proc_time > 0)
proc_time = t->t_stime;
if (thread_pri > proc_pri)
proc_pri = thread_pri;
} while ((t = t->t_forw) != prp->p_tlist);
mutex_exit(&prp->p_lock);
if (proc_pri == -1)
continue;
TRACE_3(TR_FAC_SCHED, TR_CHOOSE_SWAPIN,
"prp %p epri %d proc_time %d",
prp, proc_pri, proc_time);
/*
* Swapin processes with a high effective priority.
*/
if (swapin_prp == NULL || proc_pri > chosen_pri) {
swapin_prp = prp;
chosen_pri = proc_pri;
swapin_pri = proc_pri;
swapin_proc_time = proc_time;
}
} else {
/*
* No need to soft swap if we have sufficient
* memory.
*/
if (avefree > desfree ||
avefree < desfree && freemem > desfree)
continue;
/*
* Skip processes that are exiting
* or whose address spaces are locked.
*/
mutex_enter(&prp->p_lock);
if ((prp->p_flag & SEXITING) ||
(prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) {
mutex_exit(&prp->p_lock);
continue;
}
/*
* Softswapping to kick out deadwood.
*/
proc_pri = -1;
t = prp->p_tlist;
do {
if ((t->t_schedflag & (TS_SWAPENQ |
TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD)
continue;
thread_lock(t);
thread_pri = CL_SWAPOUT(t, SOFTSWAP);
thread_unlock(t);
if (thread_pri > proc_pri)
proc_pri = thread_pri;
} while ((t = t->t_forw) != prp->p_tlist);
if (proc_pri != -1) {
uint_t swrss;
mutex_exit(&pidlock);
TRACE_1(TR_FAC_SCHED, TR_SOFTSWAP,
"softswap:prp %p", prp);
(void) swapout(prp, &swrss, SOFTSWAP);
softswap++;
prp->p_swrss += swrss;
mutex_exit(&prp->p_lock);
goto top;
}
mutex_exit(&prp->p_lock);
}
}
if (swapin_prp != NULL)
mutex_enter(&swapin_prp->p_lock);
mutex_exit(&pidlock);
if (swapin_prp == NULL) {
TRACE_3(TR_FAC_SCHED, TR_RUNOUT,
"schedrunout:runout nswapped: %d, avefree: %ld freemem: %ld",
nswapped, avefree, freemem);
t = curthread;
thread_lock(t);
runout++;
t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART);
t->t_whystop = PR_SUSPENDED;
t->t_whatstop = SUSPEND_NORMAL;
(void) new_mstate(t, LMS_SLEEP);
mutex_enter(&swap_cpr_lock);
CALLB_CPR_SAFE_BEGIN(&cprinfo);
mutex_exit(&swap_cpr_lock);
thread_stop(t); /* change state and drop lock */
swtch();
mutex_enter(&swap_cpr_lock);
CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock);
mutex_exit(&swap_cpr_lock);
goto loop;
}
/*
* Decide how deserving this process is to be brought in.
* Needs is an estimate of how much core the process will
* need. If the process has been out for a while, then we
* will bring it in with 1/2 the core needed, otherwise
* we are conservative.
*/
divisor = 1;
swapout_time = (ddi_get_lbolt() - swapin_proc_time) / hz;
if (swapout_time > maxslp / 2)
divisor = 2;
needs = MIN(swapin_prp->p_swrss, lotsfree);
needs = MAX(needs, min_procsize);
needs = needs / divisor;
/*
* Use freemem, since we want processes to be swapped
* in quickly.
*/
avail = freemem - deficit;
if (avail > (spgcnt_t)needs) {
deficit += needs;
TRACE_2(TR_FAC_SCHED, TR_SWAPIN_VALUES,
"swapin_values: prp %p needs %lu", swapin_prp, needs);
if (swapin(swapin_prp)) {
mutex_exit(&swapin_prp->p_lock);
goto loop;
}
deficit -= MIN(needs, deficit);
mutex_exit(&swapin_prp->p_lock);
} else {
mutex_exit(&swapin_prp->p_lock);
/*
* If deficit is high, too many processes have been
* swapped in so wait a sec before attempting to
* swapin more.
*/
if (freemem > needs) {
TRACE_2(TR_FAC_SCHED, TR_HIGH_DEFICIT,
"deficit: prp %p needs %lu", swapin_prp, needs);
goto block;
}
}
TRACE_2(TR_FAC_SCHED, TR_UNLOAD,
"unload: prp %p needs %lu", swapin_prp, needs);
unload:
/*
* Unload all unloadable modules, free all other memory
* resources we can find, then look for a thread to hardswap.
*/
modreap();
segkp_cache_free();
swapout_prp = NULL;
mutex_enter(&pidlock);
for (prp = practive; prp != NULL; prp = prp->p_next) {
/*
* No need to soft swap if we have sufficient
* memory.
*/
if (not_swappable(prp))
continue;
if (avefree > minfree ||
avefree < minfree && freemem > desfree) {
swapout_prp = NULL;
break;
}
/*
* Skip processes that are exiting
* or whose address spaces are locked.
*/
mutex_enter(&prp->p_lock);
if ((prp->p_flag & SEXITING) ||
(prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) {
mutex_exit(&prp->p_lock);
continue;
}
proc_pri = -1;
t = prp->p_tlist;
do {
if ((t->t_schedflag & (TS_SWAPENQ |
TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD)
continue;
thread_lock(t);
thread_pri = CL_SWAPOUT(t, HARDSWAP);
thread_unlock(t);
if (thread_pri > proc_pri)
proc_pri = thread_pri;
} while ((t = t->t_forw) != prp->p_tlist);
mutex_exit(&prp->p_lock);
if (proc_pri == -1)
continue;
/*
* Swapout processes sleeping with a lower priority
* than the one currently being swapped in, if any.
*/
if (swapin_prp == NULL || swapin_pri > proc_pri) {
TRACE_2(TR_FAC_SCHED, TR_CHOOSE_SWAPOUT,
"hardswap: prp %p needs %lu", prp, needs);
if (swapout_prp == NULL || proc_pri < chosen_pri) {
swapout_prp = prp;
chosen_pri = proc_pri;
}
}
}
/*
* Acquire the "p_lock" before dropping "pidlock"
* to prevent the proc structure from being freed
* if the process exits before swapout completes.
*/
if (swapout_prp != NULL)
mutex_enter(&swapout_prp->p_lock);
mutex_exit(&pidlock);
if ((prp = swapout_prp) != NULL) {
uint_t swrss = 0;
int swapped;
swapped = swapout(prp, &swrss, HARDSWAP);
if (swapped) {
/*
* If desperate, we want to give the space obtained
* by swapping this process out to processes in core,
* so we give them a chance by increasing deficit.
*/
prp->p_swrss += swrss;
if (desperate)
deficit += MIN(prp->p_swrss, lotsfree);
hardswap++;
}
mutex_exit(&swapout_prp->p_lock);
if (swapped)
goto loop;
}
/*
* Delay for 1 second and look again later.
*/
TRACE_3(TR_FAC_SCHED, TR_RUNIN,
"schedrunin:runin nswapped: %d, avefree: %ld freemem: %ld",
nswapped, avefree, freemem);
block:
t = curthread;
thread_lock(t);
runin++;
t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART);
t->t_whystop = PR_SUSPENDED;
t->t_whatstop = SUSPEND_NORMAL;
(void) new_mstate(t, LMS_SLEEP);
mutex_enter(&swap_cpr_lock);
CALLB_CPR_SAFE_BEGIN(&cprinfo);
mutex_exit(&swap_cpr_lock);
thread_stop(t); /* change to stop state and drop lock */
swtch();
mutex_enter(&swap_cpr_lock);
CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock);
mutex_exit(&swap_cpr_lock);
goto loop;
}
/*
* Remove the specified thread from the swap queue.
*/
static void
swapdeq(kthread_id_t tp)
{
kthread_id_t *tpp;
ASSERT(THREAD_LOCK_HELD(tp));
ASSERT(tp->t_schedflag & TS_ON_SWAPQ);
tpp = &tswap_queue;
for (;;) {
ASSERT(*tpp != NULL);
if (*tpp == tp)
break;
tpp = &(*tpp)->t_link;
}
*tpp = tp->t_link;
tp->t_schedflag &= ~TS_ON_SWAPQ;
}
/*
* Swap in lwps. Returns nonzero on success (i.e., if at least one lwp is
* swapped in) and 0 on failure.
*/
static int
swapin(proc_t *pp)
{
kthread_id_t tp;
int err;
int num_swapped_in = 0;
struct cpu *cpup = CPU;
pri_t thread_pri;
ASSERT(MUTEX_HELD(&pp->p_lock));
ASSERT(pp->p_swapcnt);
top:
tp = pp->p_tlist;
do {
/*
* Only swapin eligible lwps (specified by the scheduling
* class) which are unloaded and ready to run.
*/
thread_lock(tp);
thread_pri = CL_SWAPIN(tp, 0);
if (thread_pri != -1 && tp->t_state == TS_RUN &&
(tp->t_schedflag & TS_LOAD) == 0) {
size_t stack_size;
pgcnt_t stack_pages;
ASSERT((tp->t_schedflag & TS_ON_SWAPQ) == 0);
thread_unlock(tp);
/*
* Now drop the p_lock since the stack needs
* to brought in.
*/
mutex_exit(&pp->p_lock);
stack_size = swapsize(tp->t_swap);
stack_pages = btopr(stack_size);
/* Kernel probe */
TNF_PROBE_4(swapin_lwp, "vm swap swapin", /* CSTYLED */,
tnf_pid, pid, pp->p_pid,
tnf_lwpid, lwpid, tp->t_tid,
tnf_kthread_id, tid, tp,
tnf_ulong, page_count, stack_pages);
rw_enter(&kas.a_lock, RW_READER);
err = segkp_fault(segkp->s_as->a_hat, segkp,
tp->t_swap, stack_size, F_SOFTLOCK, S_OTHER);
rw_exit(&kas.a_lock);
/*
* Re-acquire the p_lock.
*/
mutex_enter(&pp->p_lock);
if (err) {
num_swapped_in = 0;
break;
} else {
#ifdef __sparc
lwp_swapin(tp);
#endif /* __sparc */
CPU_STATS_ADDQ(cpup, vm, swapin, 1);
CPU_STATS_ADDQ(cpup, vm, pgswapin,
stack_pages);
pp->p_swapcnt--;
pp->p_swrss -= stack_pages;
thread_lock(tp);
tp->t_schedflag |= TS_LOAD;
dq_sruninc(tp);
/* set swapin time */
tp->t_stime = ddi_get_lbolt();
thread_unlock(tp);
nswapped--;
tot_swapped_in++;
num_swapped_in++;
TRACE_2(TR_FAC_SCHED, TR_SWAPIN,
"swapin: pp %p stack_pages %lu",
pp, stack_pages);
goto top;
}
}
thread_unlock(tp);
} while ((tp = tp->t_forw) != pp->p_tlist);
return (num_swapped_in);
}
/*
* Swap out lwps. Returns nonzero on success (i.e., if at least one lwp is
* swapped out) and 0 on failure.
*/
static int
swapout(proc_t *pp, uint_t *swrss, int swapflags)
{
kthread_id_t tp;
pgcnt_t ws_pages = 0;
int err;
int swapped_lwps = 0;
struct as *as = pp->p_as;
struct cpu *cpup = CPU;
pri_t thread_pri;
ASSERT(MUTEX_HELD(&pp->p_lock));
if (pp->p_flag & SEXITING)
return (0);
top:
tp = pp->p_tlist;
do {
klwp_t *lwp = ttolwp(tp);
/*
* Swapout eligible lwps (specified by the scheduling
* class) which don't have TS_DONT_SWAP set. Set the
* "intent to swap" flag (TS_SWAPENQ) on threads
* which have TS_DONT_SWAP set so that they can be
* swapped if and when they reach a safe point.
*/
thread_lock(tp);
thread_pri = CL_SWAPOUT(tp, swapflags);
if (thread_pri != -1) {
if (tp->t_schedflag & TS_DONT_SWAP) {
tp->t_schedflag |= TS_SWAPENQ;
tp->t_trapret = 1;
aston(tp);
} else {
pgcnt_t stack_pages;
size_t stack_size;
ASSERT((tp->t_schedflag &
(TS_DONT_SWAP | TS_LOAD)) == TS_LOAD);
if (lock_try(&tp->t_lock)) {
/*
* Remove thread from the swap_queue.
*/
if (tp->t_schedflag & TS_ON_SWAPQ) {
ASSERT(!(tp->t_schedflag &
TS_SWAPENQ));
swapdeq(tp);
} else if (tp->t_state == TS_RUN)
dq_srundec(tp);
tp->t_schedflag &=
~(TS_LOAD | TS_SWAPENQ);
lock_clear(&tp->t_lock);
/*
* Set swapout time if the thread isn't
* sleeping.
*/
if (tp->t_state != TS_SLEEP)
tp->t_stime = ddi_get_lbolt();
thread_unlock(tp);
nswapped++;
tot_swapped_out++;
lwp->lwp_ru.nswap++;
/*
* Now drop the p_lock since the
* stack needs to pushed out.
*/
mutex_exit(&pp->p_lock);
stack_size = swapsize(tp->t_swap);
stack_pages = btopr(stack_size);
ws_pages += stack_pages;
/* Kernel probe */
TNF_PROBE_4(swapout_lwp,
"vm swap swapout",
/* CSTYLED */,
tnf_pid, pid, pp->p_pid,
tnf_lwpid, lwpid, tp->t_tid,
tnf_kthread_id, tid, tp,
tnf_ulong, page_count,
stack_pages);
rw_enter(&kas.a_lock, RW_READER);
err = segkp_fault(segkp->s_as->a_hat,
segkp, tp->t_swap, stack_size,
F_SOFTUNLOCK, S_WRITE);
rw_exit(&kas.a_lock);
if (err) {
cmn_err(CE_PANIC,
"swapout: segkp_fault "
"failed err: %d", err);
}
CPU_STATS_ADDQ(cpup,
vm, pgswapout, stack_pages);
mutex_enter(&pp->p_lock);
pp->p_swapcnt++;
swapped_lwps++;
goto top;
}
}
}
thread_unlock(tp);
} while ((tp = tp->t_forw) != pp->p_tlist);
/*
* Unload address space when all lwps are swapped out.
*/
if (pp->p_swapcnt == pp->p_lwpcnt) {
size_t as_size = 0;
/*
* Avoid invoking as_swapout() if the process has
* no MMU resources since pageout will eventually
* steal pages belonging to this address space. This
* saves CPU cycles as the number of pages that are
* potentially freed or pushed out by the segment
* swapout operation is very small.
*/
if (rm_asrss(pp->p_as) != 0)
as_size = as_swapout(as);
CPU_STATS_ADDQ(cpup, vm, pgswapout, btop(as_size));
CPU_STATS_ADDQ(cpup, vm, swapout, 1);
ws_pages += btop(as_size);
TRACE_2(TR_FAC_SCHED, TR_SWAPOUT,
"swapout: pp %p pages_pushed %lu", pp, ws_pages);
/* Kernel probe */
TNF_PROBE_2(swapout_process, "vm swap swapout", /* CSTYLED */,
tnf_pid, pid, pp->p_pid,
tnf_ulong, page_count, ws_pages);
}
*swrss = ws_pages;
return (swapped_lwps);
}
void
swapout_lwp(klwp_t *lwp)
{
kthread_id_t tp = curthread;
ASSERT(curthread == lwptot(lwp));
/*
* Don't insert the thread onto the swap queue if
* sufficient memory is available.
*/
if (avefree > desfree || avefree < desfree && freemem > desfree) {
thread_lock(tp);
tp->t_schedflag &= ~TS_SWAPENQ;
thread_unlock(tp);
return;
}
/*
* Lock the thread, then move it to the swapped queue from the
* onproc queue and set its state to be TS_RUN.
*/
thread_lock(tp);
ASSERT(tp->t_state == TS_ONPROC);
if (tp->t_schedflag & TS_SWAPENQ) {
tp->t_schedflag &= ~TS_SWAPENQ;
/*
* Set the state of this thread to be runnable
* and move it from the onproc queue to the swap queue.
*/
disp_swapped_enq(tp);
/*
* Insert the thread onto the swap queue.
*/
tp->t_link = tswap_queue;
tswap_queue = tp;
tp->t_schedflag |= TS_ON_SWAPQ;
thread_unlock_nopreempt(tp);
TRACE_1(TR_FAC_SCHED, TR_SWAPOUT_LWP, "swapout_lwp:%x", lwp);
swtch();
} else {
thread_unlock(tp);
}
}
/*
* Swap all threads on the swap queue.
*/
static void
process_swap_queue(void)
{
kthread_id_t tp;
uint_t ws_pages;
proc_t *pp;
struct cpu *cpup = CPU;
klwp_t *lwp;
int err;
if (tswap_queue == NULL)
return;
/*
* Acquire the "swapped_lock" which locks the swap queue,
* and unload the stacks of all threads on it.
*/
disp_lock_enter(&swapped_lock);
while ((tp = tswap_queue) != NULL) {
pgcnt_t stack_pages;
size_t stack_size;
tswap_queue = tp->t_link;
tp->t_link = NULL;
/*
* Drop the "dispatcher lock" before acquiring "t_lock"
* to avoid spinning on it since the thread at the front
* of the swap queue could be pinned before giving up
* its "t_lock" in resume.
*/
disp_lock_exit(&swapped_lock);
lock_set(&tp->t_lock);
/*
* Now, re-acquire the "swapped_lock". Acquiring this lock
* results in locking the thread since its dispatcher lock
* (t_lockp) is the "swapped_lock".
*/
disp_lock_enter(&swapped_lock);
ASSERT(tp->t_state == TS_RUN);
ASSERT(tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ));
tp->t_schedflag &= ~(TS_LOAD | TS_ON_SWAPQ);
tp->t_stime = ddi_get_lbolt(); /* swapout time */
disp_lock_exit(&swapped_lock);
lock_clear(&tp->t_lock);
lwp = ttolwp(tp);
lwp->lwp_ru.nswap++;
pp = ttoproc(tp);
stack_size = swapsize(tp->t_swap);
stack_pages = btopr(stack_size);
/* Kernel probe */
TNF_PROBE_4(swapout_lwp, "vm swap swapout", /* CSTYLED */,
tnf_pid, pid, pp->p_pid,
tnf_lwpid, lwpid, tp->t_tid,
tnf_kthread_id, tid, tp,
tnf_ulong, page_count, stack_pages);
rw_enter(&kas.a_lock, RW_READER);
err = segkp_fault(segkp->s_as->a_hat, segkp, tp->t_swap,
stack_size, F_SOFTUNLOCK, S_WRITE);
rw_exit(&kas.a_lock);
if (err) {
cmn_err(CE_PANIC,
"process_swap_list: segkp_fault failed err: %d", err);
}
CPU_STATS_ADDQ(cpup, vm, pgswapout, stack_pages);
nswapped++;
tot_swapped_out++;
swapqswap++;
/*
* Don't need p_lock since the swapper is the only
* thread which increments/decrements p_swapcnt and p_swrss.
*/
ws_pages = stack_pages;
pp->p_swapcnt++;
TRACE_1(TR_FAC_SCHED, TR_SWAPQ_LWP, "swaplist: pp %p", pp);
/*
* Unload address space when all lwps are swapped out.
*/
if (pp->p_swapcnt == pp->p_lwpcnt) {
size_t as_size = 0;
if (rm_asrss(pp->p_as) != 0)
as_size = as_swapout(pp->p_as);
CPU_STATS_ADDQ(cpup, vm, pgswapout,
btop(as_size));
CPU_STATS_ADDQ(cpup, vm, swapout, 1);
ws_pages += btop(as_size);
TRACE_2(TR_FAC_SCHED, TR_SWAPQ_PROC,
"swaplist_proc: pp %p pages_pushed: %lu",
pp, ws_pages);
/* Kernel probe */
TNF_PROBE_2(swapout_process, "vm swap swapout",
/* CSTYLED */,
tnf_pid, pid, pp->p_pid,
tnf_ulong, page_count, ws_pages);
}
pp->p_swrss += ws_pages;
disp_lock_enter(&swapped_lock);
}
disp_lock_exit(&swapped_lock);
}