7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * CDDL HEADER START
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * The contents of this file are subject to the terms of the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Common Development and Distribution License (the "License").
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * You may not use this file except in compliance with the License.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * or http://www.opensolaris.org/os/licensing.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * See the License for the specific language governing permissions
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * and limitations under the License.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * When distributing Covered Code, include this CDDL HEADER in each
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If applicable, add the following below this CDDL HEADER, with the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * fields enclosed by brackets "[]" replaced with your own identifying
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * information: Portions Copyright [yyyy] [name of copyright owner]
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * CDDL HEADER END
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Copyright (c) 2010, Intel Corporation.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * All rights reserved.
a9ba5504895c94d55fd9eb409fdb92f2385c9515Richard PALO * Copyright 2016 PALO, Richard.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * PSMI 1.1 extensions are supported only in 2.6 and later versions.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * PSMI 1.2 extensions are supported only in 2.7 and later versions.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * PSMI 1.5 extensions are supported in Solaris Nevada.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * PSMI 1.6 extensions are supported in Solaris Nevada.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * PSMI 1.7 extensions are supported in Solaris Nevada.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Local Function Prototypes
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesstatic void apic_mark_vector(uchar_t oldvector, uchar_t newvector);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesstatic void apic_xlate_vector_free_timeout_handler(void *arg);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesstatic int apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases int new_bind_cpu, int apicindex, int intin_no, int which_irq,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesstatic int apic_setup_irq_table(dev_info_t *dip, int irqno,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *intr_flagp,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesstatic void apic_try_deferred_reprogram(int ipl, int vect);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesstatic void delete_defer_repro_ent(int which_irq);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesstatic void apic_ioapic_wait_pending_clear(int ioapicindex,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesextern int apic_acpi_translate_pci_irq(dev_info_t *dip, int busid, int devid,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases int ipin, int *pci_irqp, iflag_t *intr_flagp);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesextern int apic_handle_pci_pci_bridge(dev_info_t *idip, int child_devno,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases int child_ipin, struct apic_io_intr **intrp);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesextern struct apic_io_intr *apic_find_io_intr_w_busid(int irqno, int busid);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesextern int apic_find_intin(uchar_t ioapic, uchar_t intin);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesextern void apic_record_rdt_entry(apic_irq_t *irqptr, int irq);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * number of bits per byte, from <sys/param.h>
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases/* Max wait time (in repetitions) for flags to clear in an RDT entry. */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases/* The irq # is implicit in the array index: */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesstruct ioapic_reprogram_data apic_reprogram_info[APIC_MAX_VECTOR+1];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * APIC_MAX_VECTOR + 1 is the maximum # of IRQs as well. ioapic_reprogram_info
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * is indexed by IRQ number, NOT by vector number.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesextern int apic_sample_factor_redistribution;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases/* timeout for xlate_vector, mark_vector */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesint apic_revector_timeout = 16 * 10000; /* 160 millisec */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesextern int apic_use_acpi_madt_only; /* 1=ONLY use MADT from ACPI */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesextern uchar_t apic_io_vectbase[MAX_IO_APIC];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesextern boolean_t ioapic_mask_workaround[MAX_IO_APIC];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * First available slot to be used as IRQ index into the apic_irq_table
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * for those interrupts (like MSI/X) that don't have a physical IRQ.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * apic_defer_reprogram_lock ensures that only one processor is handling
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * deferred interrupt programming at *_intr_exit time.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * The current number of deferred reprogrammings outstanding
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Counters that keep track of deferred reprogramming stats
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesuint_t apic_last_ditch_reprogram_failures = 0;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesuchar_t apic_vector_to_irq[APIC_MAX_VECTOR+1];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* At least MSB will be set if EISA bus */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Following declarations are for revectoring; used when ISRs at different
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * IPLs share an irq.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases/* ACPI Interrupt Source Override Structure ptr */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Auto-configuration routines
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Initialise vector->ipl and ipl->pri arrays. level_intr and irqtable
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * are also set to NULL. vector->irq is set to a value which cannot map
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * to a real irq to show that it is free.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Initialize apic_ipls from apic_vectortoipl. This array is
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * used in apic_intr_enter to determine the IPL to use for the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * corresponding vector. On some systems, due to hardware errata
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * and interrupt sharing, the IPL may not correspond to the IPL listed
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * in apic_vectortoipl (see apic_addspl and apic_delspl).
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (i = 0; i < (APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL); i++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (j = 0; j < APIC_VECTOR_PER_IPL; j++, indx++)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* cpu 0 is always up (for now) */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (i = 0; i <= APIC_MAX_VECTOR; i++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* These *must* be initted to B_TRUE! */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Allocate a dummy irq table entry for the reserved entry.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * This takes care of the race between removing an irq and
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * clock detecting a CPU in that irq during interrupt load
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases LOCK_INIT_CLEAR(&apic_defer_reprogram_lock);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* mask interrupt vectors */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (j = 0; j < apic_io_max && mask_apic; j++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Bits 23-16 define the maximum redirection entries */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases intin_max = (ioapic_read(ioapic_ix, APIC_VERS_CMD) >> 16)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (i = 0; i <= intin_max; i++)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * i, AV_MASK);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Hack alert: deal with ACPI SCI interrupt chicken/egg here
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * acpica has already done add_avintr(); we just
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * to finish the job by mimicing translate_irq()
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Fake up an intrspec and setup the tables
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_setup_irq_table(NULL, apic_sci_vect, NULL,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases &ispec, &apic_sci_flags, DDI_INTR_TYPE_FIXED) < 0) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases cmn_err(CE_WARN, "!apic: SCI setup failed");
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Program I/O APIC */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (void) apic_setup_io_intr(irqptr, apic_sci_vect, B_FALSE);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Add mask bits to disable interrupt vector from happening
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * at or above IPL. In addition, it should remove mask bits
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * to enable interrupt vectors below the given IPL.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Both add and delspl are complicated by the fact that different interrupts
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * may share IRQs. This can happen in two ways.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * 1. The same H/W line is shared by more than 1 device
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * 1a. with interrupts at different IPLs
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * 1b. with interrupts at same IPL
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * 2. We ran out of vectors at a given IPL and started sharing vectors.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * 1b and 2 should be handled gracefully, except for the fact some ISRs
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * will get called often when no interrupt is pending for the device.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * For 1a, we handle it at the higher IPL.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_addspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((irqindex == -1) || (!apic_irq_table[irqindex]))
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqptr = irqheadptr = apic_irq_table[irqindex];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_addspl: dip=0x%p type=%d irqno=0x%x "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* return if it is not hardware interrupt */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Or if there are more interupts at a higher IPL */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * if apic_picinit() has not been called yet, just return.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * At the end of apic_picinit(), we will call setup_io_intr().
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Upgrade vector if max_ipl is not earlier ipl. If we cannot allocate,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * return failure.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases vector = apic_allocate_vector(max_ipl, irqindex, 1);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_mark_vector(irqptr->airq_vector, vector);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * reprogram irq being added and every one else
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * who is not in the UNINIT state
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((VIRTIRQ(irqindex, irqptr->airq_share_id) ==
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqno) || (irqptr->airq_temp_cpu != IRQ_UNINIT)) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * We cannot upgrade the vector, but we can change
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * the IPL that this vector induces.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Note that we subtract APIC_BASE_VECT from the vector
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * here because this array is used in apic_intr_enter
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * (no need to add APIC_BASE_VECT in that hot code
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * path since we can do it in the rarely-executed path
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_ipls[irqptr->airq_vector - APIC_BASE_VECT] =
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (void) apic_setup_io_intr(irqptr, irqindex, B_FALSE);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Recompute mask bits for the given interrupt vector.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If there is no interrupt servicing routine for this
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * vector, this function should disable interrupt vector
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * from happening at all IPLs. If there are still
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * handlers using the given vector, this function should
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * disable the given vector from happening below the lowest
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * IPL of the remaining hadlers.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_delspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_irq_t *irqptr, *preirqptr, *irqheadptr, *irqp;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqptr = preirqptr = irqheadptr = apic_irq_table[irqindex];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_delspl: dip=0x%p type=%d irqno=0x%x "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If there are more interrupts at a higher IPL, we don't need
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * to disable anything.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* return if it is not hardware interrupt */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Clear irq_struct. If two devices shared an intpt
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * line & 1 unloaded before picinit, we are hosed. But, then
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * we hope the machine survive.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Downgrade vector to new max_ipl if needed. If we cannot allocate,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * use old IPL. Not very elegant, but it should work.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((irqptr->airq_ipl != max_ipl) && (max_ipl != PSM_INVALID_IPL) &&
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (vector = apic_allocate_vector(max_ipl, irqno, 1)) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_mark_vector(irqheadptr->airq_vector, vector);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * We cannot downgrade the IPL of the vector below the vector's
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * hardware priority. If we did, it would be possible for a
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * higher-priority hardware vector to interrupt a CPU running at an IPL
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * lower than the hardware priority of the interrupting vector (but
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * higher than the soft IPL of this IRQ). When this happens, we would
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * then try to drop the IPL BELOW what it was (effectively dropping
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * below base_spl) which would be potentially catastrophic.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * (e.g. Suppose the hardware vector associated with this IRQ is 0x40
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * (hardware IPL of 4). Further assume that the old IPL of this IRQ
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * was 4, but the new IPL is 1. If we forced vector 0x40 to result in
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * an IPL of 1, it would be possible for the processor to be executing
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * at IPL 3 and for an interrupt to come in on vector 0x40, interrupting
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * the currently-executing ISR. When apic_intr_enter consults
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * apic_irqs[], it will return 1, bringing the IPL of the CPU down to 1
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * so even though the processor was running at IPL 4, an IPL 1
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * interrupt will have interrupted it, which must not happen)).
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Effectively, this means that the hardware priority corresponding to
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * the IRQ's IPL (in apic_ipls[]) cannot be lower than the vector's
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * hardware priority.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * (In the above example, then, after removal of the IPL 4 device's
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * interrupt handler, the new IPL will continue to be 4 because the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * hardware priority that IPL 1 implies is lower than the hardware
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * priority of the vector used.)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* apic_ipls is indexed by vector, starting at APIC_BASE_VECT */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases const int apic_ipls_index = irqptr->airq_vector -
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases const int vect_inherent_hwpri = irqptr->airq_vector >>
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If there are still devices using this IRQ, determine the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * new ipl to use.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases vect_desired_hwpri = apic_ipltopri[max_ipl] >>
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If the desired IPL's hardware priority is lower
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * than that of the vector, use the hardware priority
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * of the vector to determine the new IPL.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases hwpri = (vect_desired_hwpri < vect_inherent_hwpri) ?
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Now, to get the right index for apic_vectortoipl,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * we need to subtract APIC_BASE_VECT from the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * hardware-vector-equivalent (in hwpri). Since hwpri
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * is already shifted, we shift APIC_BASE_VECT before
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * doing the subtraction.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases hwpri -= (APIC_BASE_VECT >> APIC_IPL_SHIFT);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * No more devices on this IRQ, so reset this vector's
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * element in apic_ipls to the original IPL for this
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If there are still active interrupts, we are done.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (irqptr->airq_mps_intr_index == MSI_INDEX) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Disable the MSI vector
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Make sure we only disable on the last
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * of the multi-MSI support
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases } else if (irqptr->airq_mps_intr_index == MSIX_INDEX) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Disable the MSI-X vector
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * needs to clear its mask and addr/data for each MSI-X
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_pci_msi_unconfigure(irqptr->airq_dip, DDI_INTR_TYPE_MSIX,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Make sure we only disable on the last MSI-X
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * The assumption here is that this is safe, even for
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * systems with IOAPICs that suffer from the hardware
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * erratum because all devices have been quiesced before
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * they unregister their interrupt handlers. If that
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * assumption turns out to be false, this mask operation
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * can induce the same erratum result we're trying to
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * intin, AV_MASK);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * This irq entry is the only one in the chain.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* If hardbound, temp_cpu == cpu */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If we get here, we are sharing the vector and there are more than
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * one active irq entries in the chain.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Remove the irq entry from the chain */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (irqptr == irqheadptr) { /* The irq entry is at the head */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_irq_table[irqindex] = irqptr->airq_next;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Free the irq entry */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * apic_introp_xlate() replaces apic_translate_irq() and is
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * called only from apic_intr_ops(). With the new ADII framework,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * the priority can no longer be retrieved through i_ddi_get_intrspec().
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * It has to be passed in from the caller.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Return value:
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Success: irqno for the given device
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Failure: -1
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_introp_xlate(dev_info_t *dip, struct intrspec *ispec, int type)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases int dev_len, pci_irq, newirq, bustype, devid, busid, i;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_introp_xlate: dip=0x%p name=%s "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "type=%d irqno=0x%x\n", (void *)dip, ddi_get_name(dip), type,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip),
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((airqp = apic_find_irq(dip, ispec, type)) != NULL) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases return (apic_vector_to_irq[airqp->airq_vector]);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases return (apic_setup_irq_table(dip, irqno, NULL, ispec,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* check if we have already translated this irq */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (; newirq <= apic_max_device_irq; newirq++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (airqp->airq_mps_intr_index != FREE_INDEX)) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases return (VIRTIRQ(newirq, airqp->airq_share_id));
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi))
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* pci device */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_enable_acpi && !apic_use_acpi_madt_only) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_acpi_translate_pci_irq(dip, busid, devid,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases intr_flag.bustype = child_is_pciex ? BUS_PCIE : BUS_PCI;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases return (apic_setup_irq_table(dip, pci_irq, NULL, ispec,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid))
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((pci_irq = apic_handle_pci_pci_bridge(dip,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases return (apic_setup_irq_table(dip, pci_irq, intrp, ispec,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_enable_acpi && !apic_use_acpi_madt_only) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* search iso entries first */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) +
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases return (apic_setup_irq_table(dip, irqno, NULL, ispec,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (i = 0; i < 2; i++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (((busid = apic_find_bus_id(bustype)) != -1) &&
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ((intrp = apic_find_io_intr_w_busid(irqno, busid))
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((newirq = apic_setup_irq_table(dip, irqno,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases/* MPS default configuration */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases newirq = apic_setup_irq_table(dip, irqno, NULL, ispec, NULL, type);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Attempt to share vector with someone else
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_share_vector(int irqno, iflag_t *intr_flagp, short intr_index, int ipl,
a9ba5504895c94d55fd9eb409fdb92f2385c9515Richard PALO uchar_t ioapicindex, uchar_t ipin, apic_irq_t **irqptrp)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases#endif /* DEBUG */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_share_vector: irqno=0x%x "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "intr_index=0x%x ipl=0x%x\n", irqno, intr_index, ipl));
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases lowest = apic_ipltopri[ipl-1] + APIC_VECTOR_PER_IPL;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (highest < lowest) /* Both ipl and ipl-1 map to same pri */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* not compatible */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Assign a share id which is free or which is larger
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * than the largest one.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (irqptr->airq_mps_intr_index == FREE_INDEX) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases#endif /* DEBUG */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_irq_table[chosen_irq]->airq_next = irqptr;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases#endif /* DEBUG */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqptr->airq_vector = apic_irq_table[chosen_irq]->airq_vector;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* shuffle the pointers to test apic_delspl path */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqptr->airq_next = apic_irq_table[chosen_irq];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases#endif /* DEBUG */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Allocate/Initialize the apic_irq_table[] entry for given irqno. If the entry
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * is used already, we will try to allocate a new irqno.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Return value:
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Success: irqno
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Failure: -1
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_setup_irq_table(dev_info_t *dip, int irqno, struct apic_io_intr *intrp,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases struct intrspec *ispec, iflag_t *intr_flagp, int type)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_setup_irq_table: dip=0x%p type=%d "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "irqno=0x%x origirq=0x%x\n", (void *)dip, type, irqno, origirq));
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases major = (dip != NULL) ? ddi_driver_major(dip) : 0;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* MSI/X doesn't need to setup ioapic stuffs */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases intr_index = (type == DDI_INTR_TYPE_MSI) ? MSI_INDEX :
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((irqno = apic_allocate_irq(apic_first_avail_irq)) == -1) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* need an irq for MSI/X to index into autovect[] */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases cmn_err(CE_WARN, "No interrupt irq: %s instance %d",
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Find ioapicindex. If destid was ALL, we will exit with 0. */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ASSERT((ioapic == apic_io_id[ioapicindex]) ||
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* check whether this intin# has been used by another irqno */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* ACPI case */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ipin = irqno - apic_io_vectbase[ioapicindex];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin &&
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* default configuration */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases APIC_VERBOSE_IOAPIC((CE_WARN, "No intrspec for irqno = %x\n",
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases } else if ((vector = apic_allocate_vector(ipl, irqno, 0)) == 0) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((newirq = apic_share_vector(irqno, intr_flagp, intr_index,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases sdip = apic_irq_table[IRQINDEX(newirq)]->airq_dip;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* This is OK to do really */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases " instance %d and SCI",
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases " instance %d and %s instance %d",
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* try high priority allocation now that share has failed */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((vector = apic_allocate_vector(ipl, irqno, 1)) == 0) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases cmn_err(CE_WARN, "No interrupt vector: %s instance %d",
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (irqptr->airq_mps_intr_index != FREE_INDEX) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * The slot is used by another irqno, so allocate
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * a free irqno for this interrupt
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases newirq = apic_allocate_irq(apic_first_avail_irq);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases vector = apic_modify_vector(vector, newirq);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_max_device_irq = max(irqno, apic_max_device_irq);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_min_device_irq = min(irqno, apic_min_device_irq);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqptr->airq_mps_intr_index = (short)intr_index;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irqptr->airq_cpu = apic_bind_intr(dip, irqno, ioapic, ipin);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* setup I/O APIC entry for non-MSI/X interrupts */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * return the cpu to which this intr should be bound.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Check properties or any other mechanism to see if user wants it
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * bound to a specific CPU. If so, return the cpu id with high bit set.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If not, use the policy to choose a cpu and return the id.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_bind_intr(dev_info_t *dip, int irq, uchar_t ioapicid, uchar_t intin)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases int instance, instno, prop_len, bind_cpu, count;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_intr_policy == INTR_LOWEST_PRIORITY)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (; i <= apic_max_device_irq; i++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((i == irq) || (apic_irq_table[i] == NULL) ||
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((apic_irq_table[i]->airq_major == major) &&
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "!%s: %s (%s) instance #%d "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "irq 0x%x vector 0x%x ioapic 0x%x "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "intin 0x%x is bound to cpu %d\n",
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * search for "drvname"_intpt_bind_cpus property first, the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * syntax of the property should be "a[,b,c,...]" where
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * instance 0 binds to cpu a, instance 1 binds to cpu b,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * instance 3 binds to cpu c...
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * ddi_getlongprop() will search /option first, then /
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * if "drvname"_intpt_bind_cpus doesn't exist, then find
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * intpt_bind_cpus property. The syntax is the same, and
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * it applies to all the devices if its "drvname" specific
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * property doesn't exist
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (void) strcat(prop_name, "_intpt_bind_cpus");
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (i = count = 0; i < (prop_len - 1); i++)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * if somehow the binding instances defined in the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * property are not enough for this instno., then
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * reuse the pattern for the next instance until
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * it reaches the requested instno
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* if specific CPU is bogus, then default to next cpu */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases cmn_err(CE_WARN, "%s: %s=%s: CPU %d not present",
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* indicate that we are bound at user request */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * no need to check apic_cpus[].aci_status, if specific CPU is
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * not up, then post_cpu_start will handle it.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases cmn_err(CE_CONT, "!%s: %s (%s) instance %d irq 0x%x "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_irq_table[irq]->airq_vector, ioapicid, intin,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases psm_name, irq, apic_irq_table[irq]->airq_vector, ioapicid,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Mark vector as being in the process of being deleted. Interrupts
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * may still come in on some CPU. The moment an interrupt comes with
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * the new vector, we know we can free the old one. Called only from
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * addspl and delspl with interrupts disabled. Because an interrupt
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * can be shared, but no interrupt from either device may come in,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * we also use a timeout mechanism, which we arbitrarily set to
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * apic_revector_timeout microseconds.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_mark_vector(uchar_t oldvector, uchar_t newvector)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases kmem_zalloc(sizeof (newvector) * APIC_MAX_VECTOR * 2,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * This failure is not catastrophic.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * But, the oldvec will never be freed.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_newvec_to_oldvec = &apic_oldvec_to_newvec[APIC_MAX_VECTOR];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* See if we already did this for drivers which do double addintrs */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_oldvec_to_newvec[oldvector] != newvector) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_oldvec_to_newvec[oldvector] = newvector;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_newvec_to_oldvec[newvector] = oldvector;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (void) timeout(apic_xlate_vector_free_timeout_handler,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (void *)(uintptr_t)oldvector, drv_usectohz(apic_revector_timeout));
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * xlate_vector is called from intr_enter if revector_pending is set.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * It will xlate it if needed and mark the old vector as free.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Do we really need to do this ? */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((newvector = apic_oldvec_to_newvec[vector]) != 0)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * The incoming vector is new . See if a stale entry is
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((oldvector = apic_newvec_to_oldvec[vector]) != 0)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* There could have been more than one reprogramming! */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_xlate_vector_free_timeout_handler(void *arg)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((newvector = apic_oldvec_to_newvec[oldvector]) != 0) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Bind interrupt corresponding to irq_ptr to bind_cpu.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Must be called with interrupts disabled and apic_ioapic_lock held
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_rebind(apic_irq_t *irq_ptr, int bind_cpu,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases which_irq = apic_vector_to_irq[irq_ptr->airq_vector];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu != IRQ_UNBOUND) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Mask off high bit so it can be used as array index */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Can't bind to a CPU that's not accepting interrupts:
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases cpu_infop = &apic_cpus[bind_cpu & ~IRQ_USER_BOUND];
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (!(cpu_infop->aci_status & APIC_CPU_INTR_ENABLE))
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If we are about to change the interrupt vector for this interrupt,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * and this interrupt is level-triggered, attached to an IOAPIC,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * has been delivered to a CPU and that CPU has not handled it
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * yet, we cannot reprogram the IOAPIC now.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((irq_ptr->airq_vector != RDT_VECTOR(rdt_entry)) &&
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_check_stuck_interrupt(irq_ptr, airq_temp_cpu,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases bind_cpu, ioapicindex, intin_no, which_irq, drep) != 0) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * NOTE: We do not unmask the RDT here, as an interrupt MAY
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * still come in before we have a chance to reprogram it below.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * The reprogramming below will simultaneously change and
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * unmask the RDT entry.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu !=
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Write the vector, trigger, and polarity portion of
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((airq_temp_cpu != IRQ_UNBOUND) && (airq_temp_cpu != IRQ_UNINIT)) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irdt.ir_lo = AV_PDEST | AV_FIXED | irq_ptr->airq_rdt_entry;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Write the RDT entry -- bind to a specific CPU: */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Write the vector, trigger, and polarity portion of the RDT */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases int type = (irq_ptr->airq_mps_intr_index == MSI_INDEX) ?
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* first one */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "apic_pci_msi_enable_vector\n"));
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "apic_pci_msi_enable_mode\n"));
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases } else { /* MSI-X */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irq_ptr->airq_origirq, irq_ptr->airq_vector, 1,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_pci_msi_enable_mode(irq_ptr->airq_dip, type,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases irq_ptr->airq_temp_cpu = (uint32_t)bind_cpu;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_redist_cpu_skip &= ~(1 << (bind_cpu & ~IRQ_USER_BOUND));
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_last_ditch_clear_remote_irr(int ioapic_ix, int intin_no)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Trying to clear the bit through normal
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * channels has failed. So as a last-ditch
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * effort, try to set the trigger mode to
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * edge, then to level. This has been
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * observed to work on many systems.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If the bit's STILL set, this interrupt may
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "not clear for IOAPIC %d intin %d.\n"
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "\tInterrupts to this pin may cease "
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * This function is protected by apic_ioapic_lock coupled with the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * fact that interrupts are disabled.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_reprogram_info[which_irq].done = B_TRUE;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Interrupts must be disabled during this function to prevent
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * self-deadlock. Interrupts are disabled because this function
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * is called from apic_check_stuck_interrupt(), which is called
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * from apic_rebind(), which requires its caller to disable interrupts.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesadd_defer_repro_ent(apic_irq_t *irq_ptr, int which_irq, int new_bind_cpu)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * On the off-chance that there's already a deferred
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * reprogramming on this irq, check, and if so, just update the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * CPU and irq pointer to which the interrupt is targeted, then return.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_reprogram_info[which_irq].irqp = irq_ptr;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_reprogram_info[which_irq].irqp = irq_ptr;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * This must be the last thing set, since we're not
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * grabbing any locks, apic_try_deferred_reprogram() will
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * make its decision about using this entry iff done
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_reprogram_info[which_irq].done = B_FALSE;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If there were previously no deferred reprogrammings, change
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * setlvlx to call apic_try_deferred_reprogram()
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_try_deferred_reprogram(int prev_ipl, int irq)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (!lock_try(&apic_defer_reprogram_lock)) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Acquire the apic_ioapic_lock so that any other operations that
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * may affect the apic_reprogram_info state are serialized.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * It's still possible for the last deferred reprogramming to clear
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * between the time we entered this function and the time we get to
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * the for loop below. In that case, *setlvlx will have been set
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * back to *_intr_exit and drep will be NULL. (There's no way to
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * stop that from happening -- we would need to grab a lock before
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * calling *setlvlx, which is neither realistic nor prudent).
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * For each deferred RDT entry, try to reprogram it now. Note that
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * there is no lock acquisition to read apic_reprogram_info because
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * '.done' is set only after the other fields in the structure are set.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (reproirq = 0; reproirq <= APIC_MAX_VECTOR; reproirq++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_reprogram_info[reproirq].done == B_FALSE) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Either we found a deferred action to perform, or
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * we entered this function spuriously, after *setlvlx
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * was restored to point to *_intr_exit. Any other
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * permutation is invalid.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases ASSERT(drep != NULL || *setlvlx == psm_intr_exit_fn());
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Though we can't really do anything about errors
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * at this point, keep track of them for reporting.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Note that it is very possible for apic_setup_io_intr
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * to re-register this very timeout if the Remote IRR bit
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * has not yet cleared.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_setup_io_intr(drep, reproirq, B_TRUE) != 0) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (void) apic_setup_io_intr(drep, reproirq, B_TRUE);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_ioapic_wait_pending_clear(int ioapic_ix, int intin_no)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Wait for the delivery pending bit to clear.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (AV_LEVEL|AV_PENDING)) == (AV_LEVEL|AV_PENDING)) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If we're still waiting on the delivery of this interrupt,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * continue to wait here until it is delivered (this should be
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * a very small amount of time, but include a timeout just in
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (waited = 0; waited < apic_max_reps_clear_pending;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Checks to see if the IOAPIC interrupt entry specified has its Remote IRR
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * bit set. Calls functions that modify the function that setlvlx points to,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * so that the reprogramming can be retried very shortly.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * This function will mask the RDT entry if the interrupt is level-triggered.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * (The caller is responsible for unmasking the RDT entry.)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Returns non-zero if the caller should defer IOAPIC reprogramming.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases int new_bind_cpu, int ioapic_ix, int intin_no, int which_irq,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Wait for the delivery pending bit to clear.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_ioapic_wait_pending_clear(ioapic_ix, intin_no);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Mask the RDT entry, but only if it's a level-triggered
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((rdt_entry & (AV_LEVEL|AV_MASK)) == AV_LEVEL) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Mask it */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If there was a race and an interrupt was injected
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * just before we masked, check for that case here.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Then, unmask the RDT entry and try again. If we're
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * on our last try, don't unmask (because we want the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * RDT entry to remain masked for the rest of the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Unmask it */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If the remote IRR bit is set, then the interrupt has been sent
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * to a CPU for processing. We have no choice but to wait for
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * that CPU to process the interrupt, at which point the remote IRR
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * bit will be cleared.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (AV_LEVEL|AV_REMOTE_IRR)) == (AV_LEVEL|AV_REMOTE_IRR)) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If the CPU that this RDT is bound to is NOT the current
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * CPU, wait until that CPU handles the interrupt and ACKs
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * it. If this interrupt is not bound to any CPU (that is,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * if it's bound to the logical destination of "anyone"), it
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * may have been delivered to the current CPU so handle that
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * case by deferring the reprogramming (below).
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (waited = 0; waited < apic_max_reps_clear_pending;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Remote IRR has cleared! */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If we waited and the Remote IRR bit is still not cleared,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * times for this interrupt, try the last-ditch workaround:
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (drep && drep->tries >= APIC_REPROGRAM_MAX_TRIES) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases apic_last_ditch_clear_remote_irr(ioapic_ix, intin_no);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Mark this one as reprogrammed: */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If waiting for the Remote IRR bit (above) didn't
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * allow it to clear, defer the reprogramming.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Add a new deferred-programming entry if the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * caller passed a NULL one (and update the existing one
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * in case anything changed).
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases add_defer_repro_ent(irq_ptr, which_irq, new_bind_cpu);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Inform caller to defer IOAPIC programming: */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Remote IRR is clear */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Called to migrate all interrupts at an irq to another cpu.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Must be called with interrupts disabled and apic_ioapic_lock held
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayasesapic_rebind_all(apic_irq_t *irq_ptr, int bind_cpu)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases retval |= apic_rebind(irqptr, bind_cpu, NULL);
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * apic_intr_redistribute does all the messy computations for identifying
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * which interrupt to move to which CPU. Currently we do just one interrupt
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * at a time. This reduces the time we spent doing all this within clock
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * interrupt. When it is done in idle, we could do more than 1.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * First we find the most busy and the most free CPU (time in ISR only)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * skipping those CPUs that has been identified as being ineligible (cpu_skip)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Then we look for IRQs which are closest to the difference between the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * most busy CPU and the average ISR load. We try to find one whose load
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * is less than difference.If none exists, then we chose one larger than the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * difference, provided it does not make the most idle CPU worse than the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * most busy one. In the end, we clear all the busy fields for CPUs. For
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * IRQs, they are cleared as they are scanned.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases cpu_free = cpu_busy = max_busy = average_busy = 0;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases min_free = apic_sample_factor_redistribution;
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Below we will check for CPU_INTR_ENABLE, bound, temp_bound, temp_cpu
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * without ioapic_lock. That is OK as we are just doing statistical
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * sampling anyway and any inaccuracy now will get corrected next time
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * The call to rebind which actually changes things will make sure
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * we are consistent.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (i = 0; i < apic_nproc; i++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (apic_cpus[i].aci_status & APIC_CPU_INTR_ENABLE)) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * If no unbound interrupts or only 1 total on this
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (cpu_infop->aci_bound + cpu_infop->aci_temp_bound)
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases (max_busy >= (min_free + apic_diff_for_redistribution))) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases "redistribute busy=%x free=%x max=%x min=%x",
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases#endif /* DEBUG */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases min_busy = max_busy; /* start with the max possible value */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (; i <= apic_max_device_irq; i++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Change to linked list per CPU ? */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Check for irq_busy & decide which one to move */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Also zero them for next round */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if ((irq_ptr->airq_temp_cpu == busiest_cpu) &&
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Check for least busy CPU,
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * best fit or what ?
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Most busy within the
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * required differential
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * least busy, but more than
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * the reqd diff
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * Making sure new cpu
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * will not end up
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases#endif /* DEBUG */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Make change permenant */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases#endif /* DEBUG */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases /* Make change permenant */
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * We leave cpu_skip set so that next time we
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * can choose another cpu
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * found nothing. Could be that we skipped over valid CPUs
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * or we have balanced everything. If we had a variable
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * ticks_for_redistribution, it could be increased here.
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases * apic_int_busy, int_free etc would also need to be
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (i = 0; i < apic_nproc; i++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (i = 0; i < apic_nproc; i++) {
7ff178cd8db129d385d3177eb20744d3b6efc59bJimmy Vetayases for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) {