intr.c revision 636dfb4b6ac0749387c883053011a3afb4b4893b
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2012, Joyent, Inc. All rights reserverd.
*/
/*
* To understand the present state of interrupt handling on i86pc, we must
* first consider the history of interrupt controllers and our way of handling
* interrupts.
*
* History of Interrupt Controllers on i86pc
* -----------------------------------------
*
* Intel 8259 and 8259A
*
* The first interrupt controller that attained widespread use on i86pc was
* the Intel 8259(A) Programmable Interrupt Controller that first saw use with
* the 8086. It took up to 8 interrupt sources and combined them into one
* output wire. Up to 8 8259s could be slaved together providing up to 64 IRQs.
* With the switch to the 8259A, level mode interrupts became possible. For a
* long time on i86pc the 8259A was the only way to handle interrupts and it
* had its own set of quirks. The 8259A and its corresponding interval timer
* the 8254 are programmed using outb and inb instructions.
*
* Intel Advanced Programmable Interrupt Controller (APIC)
*
* Starting around the time of the introduction of the P6 family
* microarchitecture (i686) Intel introduced a new interrupt controller.
* Instead of having the series of slaved 8259A devices, Intel opted to outfit
* each processor with a Local APIC (lapic) and to outfit the system with at
* least one, but potentially more, I/O APICs (ioapic). The lapics and ioapics
* initially communicated over a dedicated bus, but this has since been
* replaced. Each physical core and even hyperthread currently contains its
* own local apic, which is not shared. There are a few exceptions for
* hyperthreads, but that does not usually concern us.
*
* Instead of talking directly to 8259 for status, sending End Of Interrupt
* (EOI), etc. a microprocessor now communicates directly to the lapic. This
* also allows for each microprocessor to be able to have independent controls.
* The programming method is different from the 8259. Consumers map the lapic
* registers into uncacheable memory to read and manipulate the state.
*
* The number of addressable interrupt vectors was increased to 256. However
* vectors 0-31 are reserved for the processor exception handling, leaving the
* remaining vectors for general use. In addition to hardware generated
* interrupts, the lapic provides a way for generating inter-processor
* interrupts (IPI) which are the basis for CPU cross calls and CPU pokes.
*
* AMD ended up implementing the Intel APIC architecture in lieu of their work
* with Cyrix.
*
* Intel x2apic
*
* The x2apic is an extension to the lapic which started showing up around the
* same time as the Sandy Bridge chipsets. It provides a new programming mode
* as well as new features. The goal of the x2apic is to solve a few problems
* with the previous generation of lapic and the x2apic is backwards compatible
* with the previous programming and model. The only downsides to using the
* backwards compatibility is that you are not able to take advantage of the new
* x2apic features.
*
* o The APIC ID is increased from an 8-bit value to a 32-bit value. This
* increases the maximum number of addressable physical processors beyond
* 256. This new ID is assembled in a similar manner as the information that
* is obtainable by the extended cpuid topology leaves.
*
* o A new means of generating IPIs was introduced.
*
* o Instead of memory mapping the registers, the x2apic only allows for
* programming it through a series of wrmsrs. This has important semantic
* side effects. Recall that the registers were previously all mapped to
* uncachable memory which meant that all operations to the local apic were
* serializing instructions. With the switch to using wrmsrs this has been
* relaxed and these operations can no longer be assumed to be serializing
* instructions.
*
* Note for the rest of this we are only going to concern ourselves with the
* apic and x2apic which practically all of i86pc has been using now for
* quite some time.
*
* Interrupt Priority Levels
* -------------------------
*
* On i86pc systems there are a total of fifteen interrupt priority levels
* (ipls) which range from 1-15. Level 0 is for normal processing and
* non-interrupt processing. To manipulate these values the family of spl
* functions (which date back to UNIX on the PDP-11) are used. Specifically,
* splr() to raise the priority level and splx() to lower it. One should not
* generally call setspl() directly.
*
* Both i86pc and the supported SPARC platforms honor the same conventions for
* the meaning behind these IPLs. The most important IPL is the platform's
* LOCK_LEVEL (0xa on i86pc). If a thread is above LOCK_LEVEL it _must_ not
* sleep on any synchronization object. The only allowed synchronization
* primitive is a mutex that has been specifically initialized to be a spin
* lock (see mutex_init(9F)). Another important level is DISP_LEVEL (0xb on
* i86pc). You must be at DISP_LEVEL if you want to control the dispatcher.
* The XC_HI_PIL is the highest level (0xf) and is used during cross-calls.
*
* Each interrupt that is registered in the system fires at a specific IPL.
* Generally most interrupts fire below LOCK_LEVEL.
*
* PSM Drivers
* -----------
*
* We currently have three sets of PSM (platform specific module) drivers
* available. uppc, pcplusmp, and apix. uppc (uni-processor PC) is the original
* driver that interacts with the 8259A and 8254. In general, it is not used
* anymore given the prevalence of the apic.
*
* The system prefers to use the apix driver over the pcplusmp driver. The apix
* driver requires HW support for an x2apic. If there is no x2apic HW, apix
* will not be used. In general we prefer using the apix driver over the
* pcplusmp driver because it gives us much more flexibility with respect to
* interrupts. In the apix driver each local apic has its own independent set
* of interrupts, whereas the pcplusmp driver only has a single global set of
* interrupts. This is why pcplusmp only supports a finite number of interrupts
* per IPL -- generally 16, often less. The apix driver supports using either
* the x2apic or the local apic programing modes. The programming mode does not
* change the number of interrupts available, just the number of processors
* that we can address. For the apix driver, the x2apic mode is enabled if the
* system supports interrupt re-mapping, otherwise the module manages the
* x2apic in local mode.
*
* When there is no x2apic present, we default back to the pcplusmp PSM driver.
* In general, this is not problematic unless you have more than 256
* processors in the machine or you do not have enough interrupts available.
*
* Controlling Interrupt Generation on i86pc
* -----------------------------------------
*
* There are two different ways to manipulate which interrupts will be
* generated on i86pc. Each offers different degrees of control.
*
* The first is through the flags register (eflags and rflags on i386 and amd64
* respectively). The IF bit determines whether or not interrupts are enabled
* or disabled. This is manipulated in one of several ways. The most common way
* is through the cli and sti instructions. These clear the IF flag and set it,
* respectively, for the current processor. The other common way is through the
* use of the intr_clear and intr_restore functions.
*
* Assuming interrupts are not blocked by the IF flag, then the second form is
* through the Processor-Priority Register (PPR). The PPR is used to determine
* whether or not a pending interrupt should be delivered. If the ipl of the
* new interrupt is higher than the current value in the PPR, then the lapic
* will either deliver it immediately (if interrupts are not in progress) or it
* will deliver it once the current interrupt processing has issued an EOI. The
* highest unmasked interrupt will be the one delivered.
*
* The PPR register is based upon the max of the following two registers in the
* lapic, the TPR register (also known as CR8 on amd64) that can be used to
* mask interrupt levels, and the current vector. Because the pcplusmp module
* always sets TPR appropriately early in the do_interrupt path, we can usually
* just think that the PPR is the TPR. The pcplusmp module also issues an EOI
* once it has set the TPR, so higher priority interrupts can come in while
* we're servicing a lower priority interrupt.
*
* Handling Interrupts
* -------------------
*
* Interrupts can be broken down into three categories based on priority and
* source:
*
* o High level interrupts
* o Low level hardware interrupts
* o Low level software interrupts
*
* High Level Interrupts
*
* High level interrupts encompasses both hardware-sourced and software-sourced
* interrupts. Examples of high level hardware interrupts include the serial
* console. High level software-sourced interrupts are still delivered through
* the local apic through IPIs. This is primarily cross calls.
*
* When a high level interrupt comes in, we will raise the SPL and then pin the
* current lwp to the processor. We will use its lwp, but our own interrupt
* stack and process the high level interrupt in-situ. These handlers are
* designed to be very short in nature and cannot go to sleep, only block on a
* spin lock. If the interrupt has a lot of work to do, it must generate a
* low-priority software interrupt that will be processed later.
*
* Low level hardware interrupts
*
* Low level hardware interrupts start off like their high-level cousins. The
* current CPU contains a number of kernel threads (kthread_t) that can be used
* to process low level interrupts. These are shared between both low level
* hardware and software interrupts. Note that while we run with our
* kthread_t, we borrow the pinned threads lwp_t until such a time as we hit a
* synchronization object. If we hit one and need to sleep, then the scheduler
* will instead create the rest of what we need.
*
* Low level software interrupts
*
* Low level software interrupts are handled in a similar way as hardware
* interrupts, but the notification vector is different. Each CPU has a bitmask
* of pending software interrupts. We can notify a CPU to process software
* interrupts through a specific trap vector as well as through several
* checks that are performed throughout the code. These checks will look at
* processing software interrupts as we lower our spl.
*
* We attempt to process the highest pending software interrupt that we can
* which is greater than our current IPL. If none currently exist, then we move
* on. We process a software interrupt in a similar fashion to a hardware
* interrupt.
*
* Traditional Interrupt Flow
* --------------------------
*
* The following diagram tracks the flow of the traditional uppc and pcplusmp
* interrupt handlers. The apix driver has its own version of do_interrupt().
* We come into the interrupt handler with all interrupts masked by the IF
* flag. This is because we set up the handler using an interrupt-gate, which
* is defined architecturally to have cleared the IF flag for us.
*
* +--------------+ +----------------+ +-----------+
* | _interrupt() |--->| do_interrupt() |--->| *setlvl() |
* +--------------+ +----------------+ +-----------+
* | | |
* | | |
* low-level| | | softint
* HW int | | +---------------------------------------+
* +--------------+ | | |
* | intr_thread_ |<-----+ | hi-level int |
* | prolog() | | +----------+ |
* +--------------+ +--->| hilevel_ | Not on intr stack |
* | | intr_ |-----------------+ |
* | | prolog() | | |
* +------------+ +----------+ | |
* | switch_sp_ | | On intr v |
* | and_call() | | Stack +------------+ |
* +------------+ | | switch_sp_ | |
* | v | and_call() | |
* v +-----------+ +------------+ |
* +-----------+ | dispatch_ | | |
* | dispatch_ | +-------------------| hilevel() |<------------+ |
* | hardint() | | +-----------+ |
* +-----------+ | |
* | v |
* | +-----+ +----------------------+ +-----+ hi-level |
* +---->| sti |->| av_dispatch_autovect |->| cli |---------+ |
* +-----+ +----------------------+ +-----+ | |
* | | | |
* v | | |
* +----------+ | | |
* | for each | | | |
* | handler | | | |
* | *intr() | | v |
* +--------------+ +----------+ | +----------------+ |
* | intr_thread_ | low-level | | hilevel_intr_ | |
* | epilog() |<-------------------------------+ | epilog() | |
* +--------------+ +----------------+ |
* | | | |
* | +----------------------v v---------------------+ |
* | +------------+ |
* | +---------------------->| *setlvlx() | |
* | | +------------+ |
* | | | |
* | | v |
* | | +--------+ +------------------+ +-------------+ |
* | | | return |<----| softint pending? |----->| dosoftint() |<-----+
* | | +--------+ no +------------------+ yes +-------------+
* | | ^ | |
* | | | softint pil too low | |
* | | +--------------------------------------+ |
* | | v
* | | +-----------+ +------------+ +-----------+
* | | | dispatch_ |<-----| switch_sp_ |<---------| *setspl() |
* | | | softint() | | and_call() | +-----------+
* | | +-----------+ +------------+
* | | |
* | | v
* | | +-----+ +----------------------+ +-----+ +------------+
* | | | sti |->| av_dispatch_autovect |->| cli |->| dosoftint_ |
* | | +-----+ +----------------------+ +-----+ | epilog() |
* | | +------------+
* | | | |
* | +----------------------------------------------------+ |
* v |
* +-----------+ |
* | interrupt | |
* | thread |<---------------------------------------------------+
* | blocked |
* +-----------+
* |
* v
* +----------------+ +------------+ +-----------+ +-------+ +---------+
* | set_base_spl() |->| *setlvlx() |->| splhigh() |->| sti() |->| swtch() |
* +----------------+ +------------+ +-----------+ +-------+ +---------+
*
* Calls made on Interrupt Stacks and Epilogue routines
*
* We use the switch_sp_and_call() assembly routine to switch our sp to the
* interrupt stacks and then call the appropriate dispatch function. In the
* case of interrupts which may block, softints and hardints, we always ensure
* that we are still on the interrupt thread when we call the epilog routine.
* This is not just important, it's necessary. If the interrupt thread blocked,
* we won't return from our switch_sp_and_call() function and instead we'll go
* through and set ourselves up to swtch() directly.
*
* New Interrupt Flow
* ------------------
*
* The apix module has its own interrupt path. This is done for various
* reasons. The first is that rather than having global interrupt vectors, we
* now have per-cpu vectors.
*
* The other substantial change is that the apix design does not use the TPR to
* mask interrupts below the current level. In fact, except for one special
* case, it does not use the TPR at all. Instead, it only uses the IF flag
* The design is such that when interrupts are allowed to come in, if we are
* currently servicing a higher priority interupt, the new interrupt is treated
* as pending and serviced later. Specifically, in the pcplusmp module's
* apic_intr_enter() the code masks interrupts at or below the current
* IPL using the TPR before sending EOI, whereas the apix module's
* apix_intr_enter() simply sends EOI.
*
* The one special case where the apix code uses the TPR is when it calls
* through the apic_reg_ops function pointer apic_write_task_reg in
* apix_init_intr() to initially mask all levels and then finally to enable all
* levels.
*
* Recall that we come into the interrupt handler with all interrupts masked
* by the IF flag. This is because we set up the handler using an
* interrupt-gate which is defined architecturally to have cleared the IF flag
* for us.
*
* +--------------+ +---------------------+
* | _interrupt() |--->| apix_do_interrupt() |
* +--------------+ +---------------------+
* |
* hard int? +----+--------+ softint?
* | | (but no low-level looping)
* +-----------+ |
* | *setlvl() | |
* +---------+ +-----------+ +----------------------------------+
* |apix_add_| check IPL | |
* |pending_ |<-------------+------+----------------------+ |
* |hardint()| low-level int| hi-level int| |
* +---------+ v v |
* | check IPL +-----------------+ +---------------+ |
* +--+-----+ | apix_intr_ | | apix_hilevel_ | |
* | | | thread_prolog() | | intr_prolog() | |
* | return +-----------------+ +---------------+ |
* | | | On intr |
* | +------------+ | stack? +------------+ |
* | | switch_sp_ | +---------| switch_sp_ | |
* | | and_call() | | | and_call() | |
* | +------------+ | +------------+ |
* | | | | |
* | +----------------+ +----------------+ |
* | | apix_dispatch_ | | apix_dispatch_ | |
* | | lowlevel() | | hilevel() | |
* | +----------------+ +----------------+ |
* | | | |
* | v v |
* | +-------------------------+ |
* | |apix_dispatch_by_vector()|----+ |
* | +-------------------------+ | |
* | !XC_HI_PIL| | | | |
* | +---+ +-------+ +---+ | |
* | |sti| |*intr()| |cli| | |
* | +---+ +-------+ +---+ | hi-level? |
* | +---------------------------+----+ |
* | v low-level? v |
* | +----------------+ +----------------+ |
* | | apix_intr_ | | apix_hilevel_ | |
* | | thread_epilog()| | intr_epilog() | |
* | +----------------+ +----------------+ |
* | | | |
* | v-----------------+--------------------------------+ |
* | +------------+ |
* | | *setlvlx() | +----------------------------------------------------+
* | +------------+ |
* | | | +--------------------------------+ low
* v v v------+ v | level
* +------------------+ +------------------+ +-----------+ | pending?
* | apix_do_pending_ |----->| apix_do_pending_ |----->| apix_do_ |--+
* | hilevel() | | hardint() | | softint() | |
* +------------------+ +------------------+ +-----------+ return
* | | |
* | while pending | while pending | while pending
* | hi-level | low-level | softint
* | | |
* +---------------+ +-----------------+ +-----------------+
* | apix_hilevel_ | | apix_intr_ | | apix_do_ |
* | intr_prolog() | | thread_prolog() | | softint_prolog()|
* +---------------+ +-----------------+ +-----------------+
* | On intr | |
* | stack? +------------+ +------------+ +------------+
* +--------| switch_sp_ | | switch_sp_ | | switch_sp_ |
* | | and_call() | | and_call() | | and_call() |
* | +------------+ +------------+ +------------+
* | | | |
* +------------------+ +------------------+ +------------------------+
* | apix_dispatch_ | | apix_dispatch_ | | apix_dispatch_softint()|
* | pending_hilevel()| | pending_hardint()| +------------------------+
* +------------------+ +------------------+ | | | |
* | | | | | | | |
* | +----------------+ | +----------------+ | | | |
* | | apix_hilevel_ | | | apix_intr_ | | | | |
* | | intr_epilog() | | | thread_epilog()| | | | |
* | +----------------+ | +----------------+ | | | |
* | | | | | | | |
* | +------------+ | +----------+ +------+ | | |
* | | *setlvlx() | | |*setlvlx()| | | | |
* | +------------+ | +----------+ | +----------+ | +---------+
* | | +---+ |av_ | +---+ |apix_do_ |
* +---------------------------------+ |sti| |dispatch_ | |cli| |softint_ |
* | apix_dispatch_pending_autovect()| +---+ |softvect()| +---+ |epilog() |
* +---------------------------------+ +----------+ +---------+
* |!XC_HI_PIL | | | |
* +---+ +-------+ +---+ +----------+ +-------+
* |sti| |*intr()| |cli| |apix_post_| |*intr()|
* +---+ +-------+ +---+ |hardint() | +-------+
* +----------+
*/
#include <sys/cpu_event.h>
#include <sys/segments.h>
#include <sys/traptrace.h>
#include <sys/sysmacros.h>
#include <sys/smp_impldefs.h>
#include <sys/pool_pset.h>
#include <sys/archsystm.h>
#include <sys/machsystm.h>
#include <sys/x86_archext.h>
#if defined(__xpv)
#include <sys/hypervisor.h>
#endif
/*
* This panic message is intended as an aid to interrupt debugging.
*
* The associated assertion tests the condition of enabling
* events when events are already enabled. The implication
* being that whatever code the programmer thought was
* protected by having events disabled until the second
* enable happened really wasn't protected at all ..
*/
const char *stistimsg = "stisti";
/*
* This variable tracks the last place events were disabled on each cpu
* it assists in debugging when asserts that interrupts are enabled trip.
*/
#endif
/*
* Set cpu's base SPL level to the highest active interrupt level
*/
void
set_base_spl(void)
{
}
/*
* Do all the work necessary to set up the cpu and thread structures
* to dispatch a high-level interrupt.
*
* Returns 0 if we're -not- already on the high-level interrupt stack,
* (and *must* switch to it), non-zero if we are already on that stack.
*
* Called with interrupts masked.
* The 'pil' is already set to the appropriate level for rp->r_trapno.
*/
static int
{
if (pil == CBE_HIGH_PIL) {
cpu->cpu_profile_pc = 0;
cpu->cpu_cpcprofile_pc = 0;
} else {
cpu->cpu_profile_upc = 0;
cpu->cpu_cpcprofile_upc = 0;
}
}
if (mask != 0) {
int nestpil;
/*
* We have interrupted another high-level interrupt.
* Load starting timestamp, compute interval, update
* cumulative counter.
*/
/*
* Another high-level interrupt is active below this one, so
* there is no need to check for an interrupt thread. That
* will be done by the lowest priority high-level interrupt
* active.
*/
} else {
/*
* See if we are interrupting a low-level interrupt thread.
* If so, account for its time slice only if its time stamp
* is non-zero.
*/
t->t_intr_start = 0;
}
}
/*
* Store starting timestamp in CPU structure for this PIL.
*/
if (pil == 15) {
/*
* To support reentrant level 15 interrupts, we maintain a
* recursion count in the top half of cpu_intr_actv. Only
* when this count hits zero do we clear the PIL 15 bit from
* the lower half of cpu_intr_actv.
*/
(*refcntp)++;
}
return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
}
/*
* Does most of the work of returning from a high level interrupt.
*
* Returns 0 if there are no more high level interrupts (in which
* case we must switch back to the interrupted thread stack) or
* non-zero if there are more (in which case we should stay on it).
*
* Called with interrupts masked
*/
static int
{
if (pil == 15) {
/*
* To support reentrant level 15 interrupts, we maintain a
* recursion count in the top half of cpu_intr_actv. Only
* when this count hits zero do we clear the PIL 15 bit from
* the lower half of cpu_intr_actv.
*/
if (--(*refcntp) == 0)
} else {
}
/*
* Check for lower-pil nested high-level interrupt beneath
* current one. If so, place a starting timestamp in its
* pil_high_start entry.
*/
if (mask != 0) {
int nestpil;
/*
* find PIL of nested interrupt
*/
/*
* (Another high-level interrupt is active below this one,
* so there is no need to check for an interrupt
* thread. That will be done by the lowest priority
* high-level interrupt active.)
*/
} else {
/*
* Check to see if there is a low-level interrupt active.
* If so, place a starting timestamp in the thread
* structure.
*/
if (t->t_flag & T_INTR_THREAD)
t->t_intr_start = now;
}
}
/*
* Set up the cpu, thread and interrupt thread structures for
* executing an interrupt thread. The new stack pointer of the
* interrupt thread (which *must* be switched to) is returned.
*/
static caddr_t
{
/*
* Get set to run an interrupt thread.
* There should always be an interrupt thread, since we
* allocate one for each level on each CPU.
*
* t_intr_start could be zero due to cpu_intr_swtch_enter.
*/
t = cpu->cpu_thread;
t->t_intr_start = 0;
}
/*
* unlink the interrupt thread off the cpu
*
* Note that the code in kcpc_overflow_intr -relies- on the
* ordering of events here - in particular that t->t_lwp of
* the interrupt thread is set to the pinned thread *before*
* curthread is changed.
*/
/*
* (threads on the interrupt thread free list could have state
* preset to TS_ONPROC, but it helps in debugging if
* they're TS_FREE.)
*/
}
#ifdef DEBUG
int intr_thread_cnt;
#endif
/*
* Called with interrupts disabled
*/
static void
{
kthread_t *t;
/*
* If there is still an interrupted thread underneath this one
* then the interrupt was never blocked and the return is
* fairly simple. Otherwise it isn't.
*/
/*
* The interrupted thread is no longer pinned underneath
* the interrupt thread. This means the interrupt must
* have blocked, and the interrupted thread has been
* unpinned, and has probably been running around the
* system for a while.
*
* Since there is no longer a thread under this one, put
* this interrupt thread back on the CPU's free list and
* resume the idle thread which will dispatch the next
* thread to run.
*/
#ifdef DEBUG
#endif
/*
* Set CPU's base SPL based on active interrupts bitmask
*/
set_base_spl();
(void) splhigh();
sti();
/*
* Return interrupt thread to pool
*/
swtch();
panic("intr_thread_epilog: swtch returned");
/*NOTREACHED*/
}
/*
* Return interrupt thread to the pool
*/
t->t_intr_start = now;
cpu->cpu_thread = t;
}
/*
* intr_get_time() is a resource for interrupt handlers to determine how
* much time has been spent handling the current interrupt. Such a function
* is needed because higher level interrupts can arrive during the
* processing of an interrupt. intr_get_time() only returns time spent in the
* current interrupt handler.
*
* The caller must be calling from an interrupt handler running at a pil
* below or at lock level. Timings are not provided for high-level
* interrupts.
*
* The first time intr_get_time() is called while handling an interrupt,
* it returns the time since the interrupt handler was invoked. Subsequent
* calls will return the time since the prior call to intr_get_time(). Time
* is returned as ticks. Use scalehrtimef() to convert ticks to nsec.
*
* Theory Of Intrstat[][]:
*
* uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
* uint64_ts per pil.
*
* intrstat[pil][0] is a cumulative count of the number of ticks spent
* handling all interrupts at the specified pil on this CPU. It is
* exported via kstats to the user.
*
* intrstat[pil][1] is always a count of ticks less than or equal to the
* value in [0]. The difference between [1] and [0] is the value returned
* by a call to intr_get_time(). At the start of interrupt processing,
* [0] and [1] will be equal (or nearly so). As the interrupt consumes
* time, [0] will increase, but [1] will remain the same. A call to
* intr_get_time() will return the difference, then update [1] to be the
* same as [0]. Future calls will return the time since the last call.
* Finally, when the interrupt completes, [1] is updated to the same as [0].
*
* Implementation:
*
* intr_get_time() works much like a higher level interrupt arriving. It
* "checkpoints" the timing information by incrementing intrstat[pil][0]
* to include elapsed running time, and by setting t_intr_start to rdtsc.
* It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
* and updates intrstat[pil][1] to be the same as the new value of
* intrstat[pil][0].
*
* In the normal handling of interrupts, after an interrupt handler returns
* and the code in intr_thread() updates intrstat[pil][0], it then sets
* intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
* the timings are reset, i.e. intr_get_time() will return [0] - [1] which
* is 0.
*
* Whenever interrupts arrive on a CPU which is handling a lower pil
* interrupt, they update the lower pil's [0] to show time spent in the
* handler that they've interrupted. This results in a growing discrepancy
* between [0] and [1], which is returned the next time intr_get_time() is
* called. Time spent in the higher-pil interrupt will not be returned in
* the next intr_get_time() call from the original interrupt, because
* the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
*/
intr_get_time(void)
{
kthread_t *t;
cli();
t = cpu->cpu_thread;
ASSERT(t->t_intr_start != 0);
t->t_intr_start = time;
sti();
return (ret);
}
static caddr_t
{
top:
return (0);
/*
* XX64 Sigh.
*
* This is a transliteration of the i386 assembler code for
* soft interrupts. One question is "why does this need
* to be atomic?" One possible race is -other- processors
* posting soft interrupts to us in set_pending() i.e. the
* CPU might get preempted just after the address computation,
* but just before the atomic transaction, so another CPU would
* actually set the original CPU's st_pending bit. However,
* it looks like it would be simpler to disable preemption there.
* Are there other races for which preemption control doesn't work?
*
* The i386 assembler version -also- checks to see if the bit
* being cleared was actually set; if it wasn't, it rechecks
* for more. This seems a bit strange, as the only code that
* ever clears the bit is -this- code running with interrupts
* disabled on -this- CPU. This code would probably be cheaper:
*
* atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending,
* ~(1 << pil));
*
* and t->t_preempt--/++ around set_pending() even cheaper,
* but at this point, correctness is critical, so we slavishly
* emulate the i386 port.
*/
if (atomic_btr32((uint32_t *)
goto top;
}
/*
* Get set to run interrupt thread.
* There should always be an interrupt thread since we
* allocate one for each level on the CPU.
*/
/* t_intr_start could be zero due to cpu_intr_swtch_enter. */
t = cpu->cpu_thread;
t->t_intr_start = 0;
}
/*
* Note that the code in kcpc_overflow_intr -relies- on the
* ordering of events here - in particular that t->t_lwp of
* the interrupt thread is set to the pinned thread *before*
* curthread is changed.
*/
/*
* Push interrupted thread onto list from new thread.
* Set the new thread as the current one.
* Set interrupted thread's T_SP because if it is the idle thread,
* resume() may use that stack between threads.
*/
/*
* Set bit for this pil in CPU's interrupt active bitmask.
*/
/*
* Initialize thread priority level from intr_pri
*/
}
static void
{
/*
* If there is still an interrupted thread underneath this one
* then the interrupt was never blocked and the return is
* fairly simple. Otherwise it isn't.
*/
/*
* Put thread back on the interrupt thread list.
* This was an interrupt thread, so set CPU's base SPL.
*/
set_base_spl();
(void) splhigh();
sti();
swtch();
/*NOTREACHED*/
panic("dosoftint_epilog: swtch returned");
}
cpu->cpu_thread = t;
if (t->t_flag & T_INTR_THREAD)
t->t_intr_start = now;
}
/*
* Make the interrupted thread 'to' be runnable.
*
* Since t->t_sp has already been saved, t->t_pc is all
* that needs to be set in this function.
*
* Returns the interrupt level of the interrupt thread.
*/
int
kthread_t *t) /* interrupted thread */
{
extern void _sys_rtt();
}
/*
* Create interrupt kstats for this CPU.
*/
void
{
int i;
char name[KSTAT_STRLEN];
if (pool_pset_enabled())
else
/*
* Initialize each PIL's named kstat
*/
for (i = 0; i < PIL_MAX; i++) {
i + 1);
i + 1);
}
}
}
/*
* Delete interrupt kstats for this CPU.
*/
void
{
}
/*
* Convert interrupt statistics from CPU ticks to nanoseconds and
* update kstat.
*/
int
{
int i;
if (rw == KSTAT_WRITE)
return (EACCES);
for (i = 0; i < PIL_MAX; i++) {
scalehrtimef(&hrt);
}
return (0);
}
/*
* An interrupt thread is ending a time slice, so compute the interval it
* ran for and update the statistic for its PIL.
*/
void
{
/*
* We could be here with a zero timestamp. This could happen if:
* an interrupt thread which no longer has a pinned thread underneath
* it (i.e. it blocked at some point in its past) has finished running
* its handler. intr_thread() updated the interrupt statistic for its
* PIL and zeroed its timestamp. Since there was no pinned thread to
* return to, swtch() gets called and we end up here.
*
* Note that we use atomic ops below (cas64 and atomic_add_64), which
* we don't use in the functions above, because we're not called
*/
if (t->t_intr_start) {
do {
start = t->t_intr_start;
interval);
} else
}
/*
* An interrupt thread is returning from swtch(). Place a starting timestamp
* in its thread structure.
*/
void
{
do {
ts = t->t_intr_start;
}
/*
* Dispatch a hilevel interrupt (one above LOCK_LEVEL)
*/
/*ARGSUSED*/
static void
{
sti();
cli();
}
/*
* Dispatch a soft interrupt
*/
/*ARGSUSED*/
static void
{
sti();
cli();
/*
* Must run softint_epilog() on the interrupt thread stack, since
* there may not be a return from it if the interrupt thread blocked.
*/
}
/*
* Dispatch a normal interrupt
*/
static void
{
sti();
cli();
/*
* Must run intr_thread_epilog() on the interrupt thread stack, since
* there may not be a return from it if the interrupt thread blocked.
*/
}
/*
* Deliver any softints the current interrupt priority allows.
* Called with interrupts disabled.
*/
void
{
int oldipl;
/*
* If returned stack pointer is NULL, priority is too high
* to run any of the pending softints now.
* Break out and they will be run later.
*/
break;
}
}
/*
* Interrupt service routine, called with interrupts disabled.
*/
/*ARGSUSED*/
void
{
#ifdef TRAPTRACE
#endif /* TRAPTRACE */
/*
* If it's a softint go do it now.
*/
ASSERT(!interrupts_enabled());
return;
}
/*
* Raise the interrupt priority.
*/
#ifdef TRAPTRACE
#endif /* TRAPTRACE */
/*
* Bail if it is a spurious interrupt
*/
if (newipl == -1)
return;
#ifdef TRAPTRACE
#endif /* TRAPTRACE */
if (newipl > LOCK_LEVEL) {
/*
* High priority interrupts run on this cpu's interrupt stack.
*/
} else { /* already on the interrupt stack */
dispatch_hilevel(vector, 0);
}
} else {
/*
* Run this interrupt in a separate thread.
*/
}
#if !defined(__xpv)
/*
* Deliver any pending soft interrupts.
*/
#endif /* !__xpv */
}
/*
* Common tasks always done by _sys_rtt, called with interrupts disabled.
* Returns 1 if returning to userland, 0 if returning to system mode.
*/
int
{
extern void mutex_exit_critical_start();
extern long mutex_exit_critical_size;
extern void mutex_owner_running_critical_start();
extern long mutex_owner_running_critical_size;
loop:
/*
* Check if returning to user
*/
/*
* Check if AST pending.
*/
/*
* Let trap() handle the AST
*/
sti();
cli();
goto loop;
}
#if defined(__amd64)
/*
* We are done if segment registers do not need updating.
*/
return (1);
/*
* 1 or more of the selectors is bad.
* Deliver a SIGSEGV.
*/
sti();
mutex_enter(&p->p_lock);
mutex_exit(&p->p_lock);
psig();
cli();
}
#endif /* __amd64 */
return (1);
}
/*
* Here if we are returning to supervisor mode.
* Check for a kernel preemption request.
*/
/*
* Do nothing if already in kpreempt
*/
if (!tp->t_preempt_lk) {
sti();
cli();
tp->t_preempt_lk = 0;
}
}
/*
* If we interrupted the mutex_exit() critical region we must
* reset the PC back to the beginning to prevent missed wakeups
* See the comments in mutex_exit() for details.
*/
}
/*
* If we interrupted the mutex_owner_running() critical region we
* must reset the PC back to the beginning to prevent dereferencing
* of a freed thread pointer. See the comments in mutex_owner_running
* for details.
*/
}
return (0);
}
void
{
}
(((*get_pending_spl)() > (newpri)) || \
/*
* do_splx routine, takes new ipl to set
* returns the old ipl.
* We are careful not to set priority lower than CPU->cpu_base_pri,
* even though it seems we're raising the priority, it could be set
* higher at any time by an interrupt routine, so we must block interrupts
* and look at CPU->cpu_base_pri
*/
int
{
flag = intr_clear();
/*
* If we are going to reenable interrupts see if new priority level
* allows pending softint delivery.
*/
fakesoftint();
ASSERT(!interrupts_enabled());
return (curpri);
}
/*
* Common spl raise routine, takes new ipl to set
* returns the old ipl, will not lower ipl.
*/
int
{
flag = intr_clear();
/*
* Only do something if new priority is larger
*/
/*
* See if new priority level allows pending softint delivery
*/
fakesoftint();
}
return (curpri);
}
int
getpil(void)
{
}
int
spl_xcall(void)
{
}
int
interrupts_enabled(void)
{
}
#ifdef DEBUG
void
assert_ints_enabled(void)
{
}
#endif /* DEBUG */