ddi_intr_irm.c revision 7ff178cd8db129d385d3177eb20744d3b6efc59b
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
*/
#include <sys/sysmacros.h>
/*
* MSI-X allocation limit.
*/
extern uint_t ddi_msix_alloc_limit;
#endif
/*
* Interrupt Resource Management (IRM).
*/
(r->ireq_type == DDI_INTR_TYPE_MSIX)) || \
(r->ireq_flags & DDI_IRM_FLAG_NEW))
extern pri_t minclsyspri;
/* Global policies */
int irm_enable = 1;
/* Global list of interrupt pools */
/* Global debug tunables */
#ifdef DEBUG
int irm_debug_policy = 0;
uint_t irm_debug_size = 0;
#endif /* DEBUG */
static void irm_balance_thread(ddi_irm_pool_t *);
static void i_ddi_irm_balance(ddi_irm_pool_t *);
static int i_ddi_irm_reduce_by_policy(ddi_irm_pool_t *, int, int);
static void i_ddi_irm_reduce_new(ddi_irm_pool_t *, int);
static int i_ddi_irm_modify_increase(ddi_irm_req_t *, int);
/*
* OS Initialization Routines
*/
/*
* irm_init()
*
* Initialize IRM subsystem before any drivers are attached.
*/
void
irm_init(void)
{
/* Do nothing if IRM is disabled */
if (!irm_enable)
return;
/* Verify that the default balancing policy is valid */
/* Initialize the global list of interrupt pools */
}
/*
* i_ddi_irm_poststartup()
*
* IRM is not activated until after the IO subsystem is initialized.
* When activated, per-pool balancing threads are spawned and a flag
* is set so that all future pools will be activated when created.
*
* NOTE: the global variable 'irm_enable' disables IRM if zero.
*/
void
i_ddi_irm_poststartup(void)
{
/* Do nothing if IRM is disabled */
if (!irm_enable)
return;
/* Lock the global list */
/* Activate all defined pools */
/* Set future pools to be active */
irm_active = B_TRUE;
/* Unlock the global list */
}
/*
* NDI interfaces for creating/destroying IRM pools.
*/
/*
* ndi_irm_create()
*
* Nexus interface to create an IRM pool. Create the new
* pool and add it to the global list of interrupt pools.
*/
int
{
/* Check if IRM is enabled */
if (!irm_enable)
return (NDI_FAILURE);
/* Validate parameters */
return (NDI_FAILURE);
/* Allocate and initialize the pool */
/* Add to global list of pools */
/* If IRM is active, then activate the pool */
if (irm_active)
return (NDI_SUCCESS);
}
/*
* ndi_irm_resize_pool()
*
* Nexus interface to resize IRM pool. If the pool size drops
* below the allocated number of vectors then initiate rebalance
* operation before resizing the pool. If rebalance operation fails
* then return NDI_FAILURE.
*/
int
{
" current-size 0x%x new-size 0x%x\n",
return (NDI_EINVAL);
/* Check if IRM is enabled */
if (!irm_enable)
return (NDI_FAILURE);
/*
* If we are increasing the pool size or if the reserved
* number of vectors is <= the new pool size then simply
* update the pool size and enqueue a reblance operation
* if necessary to use the new vectors.
*/
/* set new pool size */
/* adjust the default allocation limit */
/* queue a rebalance operation to use the new vectors */
return (NDI_SUCCESS);
}
" needs a rebalance operation\n", (void *)pool_p));
/*
* requires a rebalance operation
*/
/* save the current pool size */
/* set the pool size to the desired new value */
/* perform the rebalance operation */
/*
* If rebalance operation couldn't free up enough
* vectors then fail the resize operation.
*/
/* restore the pool size to the previous value */
/* enqueue a rebalance operation for the original pool size */
return (NDI_FAILURE);
} else { /* rebalance worked */
/* adjust the default allocation limit */
" resized from %x to %x\n",
return (NDI_SUCCESS);
}
}
/*
* ndi_irm_destroy()
*
* Nexus interface to destroy an IRM pool. Destroy the pool
* and remove it from the global list of interrupt pools.
*/
int
{
(void *)pool_p));
/* Validate parameters */
return (NDI_FAILURE);
/* Validate that pool is empty */
if (pool_p->ipool_resno != 0)
return (NDI_BUSY);
/* Remove the pool from the global list */
/* Terminate the balancing thread */
if (pool_p->ipool_thread &&
} else
/* Destroy the pool */
return (NDI_SUCCESS);
}
/*
*/
/*
* i_ddi_irm_insert()
*
* Insert a new request into an interrupt pool, and balance the pool.
*/
int
{
/* Validate parameters */
return (DDI_EINVAL);
}
/* Check for an existing request */
return (DDI_SUCCESS);
/* Check for IRM support from the system */
return (DDI_ENOTSUP);
}
/* Check for IRM support from the driver */
/* Determine request size */
/* Allocate and initialize the request */
if (irm_flag)
/* Lock the pool */
/* Check for minimal fit before inserting */
return (DDI_EAGAIN);
}
/* Insert the request into the pool */
/*
* Try to fulfill the request.
*
* If all the interrupts are available, and either the request
* is static or the pool is active, then just take them directly.
*
* If only some of the interrupts are available, and the request
* can receive future callbacks, then take some now but queue the
* pool to be rebalanced later.
*
* Otherwise, immediately rebalance the pool and wait.
*/
"request completely fulfilled.\n"));
} else if (irm_flag &&
"request partially fulfilled.\n"));
} else {
"request needs immediate rebalance.\n"));
}
/* Fail if the request cannot be fulfilled at all */
if (req_p->ireq_navail == 0) {
return (DDI_EAGAIN);
}
/* Unlock the pool */
return (DDI_SUCCESS);
}
/*
* i_ddi_irm_modify()
*
* Modify an existing request in an interrupt pool, and balance the pool.
*/
int
{
int type;
int retval = DDI_SUCCESS;
/* Validate parameters */
return (DDI_EINVAL);
}
/* Do nothing if not mapped to an IRM pool */
return (DDI_SUCCESS);
/* Do nothing if new size is the same */
return (DDI_SUCCESS);
/* Do not allow MSI requests to be resized */
return (DDI_ENOTSUP);
}
/* Select the pool */
return (DDI_FAILURE);
}
/* Validate request size is not too large */
return (DDI_EINVAL);
}
/* Lock the pool */
/*
* Process the modification.
*
* - To increase a non-IRM request, call the implementation in
* i_ddi_irm_modify_increase().
*
* - To decrease a non-IRM request, directly update the pool and
* request, then queue the pool for later rebalancing.
*
* - To modify an IRM request, always queue the pool for later
* rebalancing. IRM consumers rely upon callbacks for changes.
*/
} else {
/* Update pool and request */
}
/* Re-sort request into the pool */
/* Queue pool for asynchronous rebalance */
}
/* Unlock the pool */
return (retval);
}
/*
* i_ddi_irm_modify_increase()
*
* Increase a non-IRM request. The additional interrupts are
* directly taken from the pool when possible. Otherwise, an
* immediate, synchronous rebalance is performed. A temporary
* proxy request is used for any rebalance operation to ensure
* the request is not reduced below its current allocation.
*
* NOTE: pool must already be locked.
*/
static int
{
/* Compute number of additional vectors */
/* Check for minimal fit */
return (DDI_EAGAIN);
}
/* Update the pool */
/* Attempt direct implementation */
return (DDI_SUCCESS);
}
/* Rebalance required: fail if pool is not active */
return (DDI_EAGAIN);
}
/* Insert temporary proxy request */
/* Synchronously rebalance */
/* Remove proxy request, and merge into original request */
}
return (DDI_SUCCESS);
}
/*
* i_ddi_irm_remove()
*
* Remove a request from an interrupt pool, and balance the pool.
*/
int
{
/* Validate parameters */
return (DDI_EINVAL);
}
/* Check if the device has a request */
return (DDI_EINVAL);
}
/* Lock the pool */
/* Remove request */
/* Queue pool to be rebalanced */
/* Unlock the pool */
/* Destroy the request */
return (DDI_SUCCESS);
}
/*
* i_ddi_irm_set_cb()
*
* Change the callback flag for a request, in response to
* a change in its callback registration. Then rebalance
* the interrupt pool.
*
* NOTE: the request is not locked because the navail value
* is not directly affected. The balancing thread may
* modify the navail value in the background after it
* locks the request itself.
*/
void
{
(void *)dip, (int)has_cb_flag));
/* Validate parameters */
return;
/* Check for association with interrupt pool */
return;
}
/* Lock the pool */
/*
* Update the request and the pool
*/
if (has_cb_flag) {
/* Update pool statistics */
/* Update request */
/* Rebalance in background */
} else {
/* Determine new request size */
/* Use the default static limit for non-IRM drivers */
#endif
/* Update pool statistics */
} else {
}
/* Update request size, and re-sort in pool */
/* Rebalance synchronously, before losing callback */
/* Remove callback flag */
}
/* Unlock the pool */
}
/*
* i_ddi_irm_supported()
*
* Query if IRM is supported by a driver using a specific interrupt type.
* Notice that IRM is limited to MSI-X users with registered callbacks.
*/
int
{
}
/*
* Interrupt Pool Balancing
*/
/*
* irm_balance_thread()
*
* One instance of this thread operates per each defined IRM pool.
* It does the initial activation of the pool, as well as balancing
* any requests that were queued up before the pool was active.
* Once active, it waits forever to service balance operations.
*/
static void
{
(void *)pool_p));
/* Lock the pool */
/* Perform initial balance if required */
/* Activate the pool */
/* Main loop */
for (;;) {
/* Compute the delay interval */
/* Sleep until queued */
/* Wait one interval, or until there are waiters */
if ((interval > 0) &&
}
/* Check if awakened to exit */
"irm_balance_thread: exiting...\n"));
thread_exit();
}
/* Balance the pool */
/* Notify waiters */
}
/* Clear QUEUED condition */
}
}
/*
* i_ddi_irm_balance()
*
* Balance a pool. The general algorithm is to first reset all
* requests to their maximum size, use reduction algorithms to
* solve any imbalance, and then notify affected drivers.
*/
static void
{
#ifdef DEBUG
uint_t debug_totsz = 0;
int debug_policy = 0;
#endif /* DEBUG */
(void *)pool_p));
#ifdef DEBUG /* Adjust size and policy settings */
}
if (DDI_IRM_POLICY_VALID(irm_debug_policy)) {
"i_ddi_irm_balance: debug policy %d\n", irm_debug_policy));
}
#endif /* DEBUG */
/* Lock the availability lock */
/*
* Put all of the reducible requests into a scratch list.
* Reset each one of them to their maximum availability.
*/
if (DDI_IRM_IS_REDUCIBLE(req_p)) {
}
}
/* Balance the requests */
/* Unlock the availability lock */
/*
* Process REMOVE notifications.
*
* If a driver fails to release interrupts: exclude it from
* further processing, correct the resulting imbalance, and
* start over again at the head of the scratch list.
*/
while (req_p) {
} else {
}
}
/*
* Process ADD notifications.
*
* This is the last use of the scratch list, so empty it.
*/
}
}
#ifdef DEBUG /* Restore size and policy settings */
if (debug_totsz != 0)
if (debug_policy != 0)
#endif /* DEBUG */
}
/*
* i_ddi_irm_reduce()
*
* Use reduction algorithms to correct an imbalance in a pool.
*/
static void
{
int imbalance;
(void *)pool_p));
/* Compute the imbalance. Do nothing if already balanced. */
return;
/*
* Try policy based reduction first. If it failed, then
* possibly reduce new requests as a last resort.
*/
!= DDI_SUCCESS) {
"i_ddi_irm_reduce: policy reductions failed.\n"));
/* Compute remaining imbalance */
}
}
/*
* i_ddi_irm_enqueue()
*
* Queue a pool to be balanced. Signals the balancing thread to wake
* up and process the pool. If 'wait_flag' is true, then the current
* thread becomes a waiter and blocks until the balance is completed.
*/
static void
{
/* Do nothing if pool is already balanced */
#ifndef DEBUG
#else
#endif /* DEBUG */
"i_ddi_irm_enqueue: pool already balanced\n"));
return;
}
/* Avoid deadlocks when IRM is not active */
if (!irm_active && wait_flag) {
"i_ddi_irm_enqueue: pool not active.\n"));
return;
}
if (wait_flag)
}
if (wait_flag) {
}
}
/*
* i_ddi_irm_reduce_by_policy()
*
* Reduces requests based on reduction policies.
*
* For the DDI_IRM_POLICY_LARGE reduction policy, the algorithm
* generally reduces larger requests first, before advancing
* to smaller requests.
* For the DDI_IRM_POLICY_EVEN reduction policy, the algorithm
* reduces requests evenly, without giving a specific preference
* to smaller or larger requests. Each iteration reduces all
* reducible requests by the same amount until the imbalance is
* corrected.
*
* The scratch list is initially sorted in descending order by current
* navail values, which are maximized prior to reduction. This sorted
* order is preserved. It avoids reducing requests below the threshold
* of the interrupt pool's default allocation size.
*
* Optimizations in this algorithm include trying to reduce multiple
* requests together. And the algorithm attempts to reduce in larger
* increments when possible to minimize the total number of iterations.
*/
static int
{
while (imbalance > 0) {
/* Fail if none are reducible */
"i_ddi_irm_reduce_by_policy: Failure. "
"All requests have downsized to low limit.\n"));
return (DDI_FAILURE);
}
/* Count reducible requests */
break;
nreduce++;
}
/* Compute reduction */
else
imbalance = 0;
} else {
}
/* Reduce */
}
req_p->ireq_navail--;
}
}
return (DDI_SUCCESS);
}
/*
* i_ddi_irm_reduce_new()
*
* Reduces new requests. This is only used as a last resort
* after another reduction algorithm failed.
*
* NOTE: The pool locking in i_ddi_irm_insert() ensures
* there can be only one new request at a time in a pool.
*/
static void
{
"i_ddi_irm_reduce_new: pool_p %p imbalance %d\n",
return;
}
}
/* should never go here */
}
/*
* Miscellaneous Helper Functions
*/
/*
* i_ddi_intr_get_pool()
*
* Get an IRM pool that supplies interrupts of a specified type.
* Invokes a DDI_INTROP_GETPOOL to the bus nexus driver. Fails
* if no pool exists.
*/
{
return (pool_p);
}
return (pool_p);
return (NULL);
}
/*
* i_ddi_irm_insertion_sort()
*
* Use the insertion sort method to insert a request into a list.
* The list is sorted in descending order by request size.
*/
static void
{
}
/*
* i_ddi_irm_notify()
*
* Notify a driver of changes to its interrupt request using the
* generic callback mechanism. Checks for errors in processing.
*/
static int
{
/* Do not notify new or unchanged requests */
return (DDI_SUCCESS);
/* Determine action and count */
count));
} else {
count));
}
/* Lookup driver callback */
return (DDI_FAILURE);
}
/* Do callback */
/* Log callback errors */
if (ret != DDI_SUCCESS) {
}
/* Check if the driver exceeds its availability */
"(nintrs=%d, navail=%d).\n",
req_p->ireq_navail);
return (DDI_FAILURE);
}
/* Update request */
return (DDI_SUCCESS);
}
/*
* i_ddi_irm_debug_balance()
*
* synchronous rebalancing of an interrupt pool.
*/
#ifdef DEBUG
void
{
int type;
}
}
#endif